Setup Supervisord for process management, log handling and log rotation

Note

This assumes the full path to your ~/devilrydeploy-directory is /home/devilryrunner/devilrydeploy — adjust accordingly.

Create a Supervisord configuration file

Create a file named ~/devilrydeploy/supervisord.conf and add the following:

[supervisord]
childlogdir = /home/devilryrunner/devilrydeploy/log
logfile = /home/devilryrunner/devilrydeploy/log/supervisord.log
logfile_maxbytes = 50MB
logfile_backups = 30
loglevel = info
pidfile = /home/devilryrunner/devilrydeploy/var/supervisord.pid
umask = 022
nodaemon = false
nocleanup = false

[inet_http_server]
port = 9001
username = devilryadmin
password = secret

[supervisorctl]
serverurl = http://localhost:9001
username = devilryadmin
password = secret

[rpcinterface:supervisor]
supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface

[program:gunicorn]
command = /home/devilryrunner/devilrydeploy/venv/bin/gunicorn devilry.project.production.wsgi -b 127.0.0.1:8002 -w 12 --timeout 300
environment = DJANGO_SETTINGS_MODULE=devilry_settings
process_name = gunicorn
directory = /home/devilryrunner/devilrydeploy
redirect_stderr = true
stdout_logfile = /home/devilryrunner/devilrydeploy/log/gunicorn.log
stdout_logfile_maxbytes = 150MB
stdout_logfile_backups = 15

[program:rqworker]
command = /home/devilryrunner/devilrydeploy/venv/bin/python manage.py rqworker default email highpriority
process_name = rqworker
directory = /home/devilryrunner/devilrydeploy
redirect_stderr = true
stdout_logfile = /home/devilryrunner/devilrydeploy/log/rq.log
stdout_logfile_maxbytes = 150MB
stdout_logfile_backups = 15

Password and security

Make sure you set some other password than secret in the [inet_http_server] and [supervisorctl] sections, and make sure ~/devilrydeploy/supervisord.conf is only accessible to the devilryrunner-user.

Create the var/ and log/ directories

The supervisord.conf file refers to the ~/devilrydeploy/var/ and ~/devilrydeploy/log/ directories. These must be created:

$ cd ~/devilrydeploy
$ mkdir var/ log/

Make sure all services work as excpected

To run supervisord in the foreground (for testing), run:

$ cd ~/devilrydeploy
$ venv/bin/supervisord -n -c supervisord.conf

You should now be able to open http://localhost:8002 in a browser and use Devilry. Use ctrl-c to kill supervisord and all the services it is running.

Run Supervisord for production

To run supervisord in the background with a PID, run:

$ cd ~/devilrydeploy
$ venv/bin/supervisord -c supervisord.conf

Warning

Do NOT run supervisord as root. Run it as the devilryrunner user.

Init script

The following init script works well. You need to adjust the DAEMON-variable:

#! /bin/sh
### BEGIN INIT INFO
# Provides:          supervisord
# Required-Start:    $remote_fs
# Required-Stop:     $remote_fs
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Example initscript
# Description:       This file should be used to construct scripts to be
#                    placed in /etc/init.d.
### END INIT INFO

# Author: Dan MacKinlay <danielm@phm.gov.au>
# Based on instructions by Bertrand Mathieu
# http://zebert.blogspot.com/2009/05/installing-django-solr-varnish-and.html
# See: https://gist.github.com/176149

# Do NOT "set -e"

# PATH should only include /usr/* if it runs after the mountnfs.sh script
PATH=/sbin:/usr/sbin:/bin:/usr/bin
DESC="Description of the service"
NAME=supervisord
DAEMON=/usr/local/bin/supervisord
DAEMON_ARGS=""
PIDFILE=/var/run/$NAME.pid
SCRIPTNAME=/etc/init.d/$NAME

# Exit if the package is not installed
[ -x "$DAEMON" ] || exit 0

# Read configuration variable file if it is present
[ -r /etc/default/$NAME ] && . /etc/default/$NAME

# Load the VERBOSE setting and other rcS variables
. /lib/init/vars.sh

# Define LSB log_* functions.
# Depend on lsb-base (>= 3.0-6) to ensure that this file is present.
. /lib/lsb/init-functions

#
# Function that starts the daemon/service
#
do_start()
{
	# Return
	#   0 if daemon has been started
	#   1 if daemon was already running
	#   2 if daemon could not be started
	start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON --test > /dev/null \
		|| return 1
	start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON -- \
		$DAEMON_ARGS \
		|| return 2
	# Add code here, if necessary, that waits for the process to be ready
	# to handle requests from services started subsequently which depend
	# on this one.  As a last resort, sleep for some time.
}

#
# Function that stops the daemon/service
#
do_stop()
{
	# Return
	#   0 if daemon has been stopped
	#   1 if daemon was already stopped
	#   2 if daemon could not be stopped
	#   other if a failure occurred
	start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME
	RETVAL="$?"
	[ "$RETVAL" = 2 ] && return 2
	# Wait for children to finish too if this is a daemon that forks
	# and if the daemon is only ever run from this initscript.
	# If the above conditions are not satisfied then add some other code
	# that waits for the process to drop all resources that could be
	# needed by services started subsequently.  A last resort is to
	# sleep for some time.
	start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --exec $DAEMON
	[ "$?" = 2 ] && return 2
	# Many daemons don't delete their pidfiles when they exit.
	rm -f $PIDFILE
	return "$RETVAL"
}

#
# Function that sends a SIGHUP to the daemon/service
#
do_reload() {
	#
	# If the daemon can reload its configuration without
	# restarting (for example, when it is sent a SIGHUP),
	# then implement that here.
	#
	start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name $NAME
	return 0
}

case "$1" in
  start)
	[ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME"
	do_start
	case "$?" in
		0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
		2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
	esac
	;;
  stop)
	[ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME"
	do_stop
	case "$?" in
		0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
		2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
	esac
	;;
  #reload|force-reload)
	#
	# If do_reload() is not implemented then leave this commented out
	# and leave 'force-reload' as an alias for 'restart'.
	#
	#log_daemon_msg "Reloading $DESC" "$NAME"
	#do_reload
	#log_end_msg $?
	#;;
  restart|force-reload)
	#
	# If the "reload" option is implemented then remove the
	# 'force-reload' alias
	#
	log_daemon_msg "Restarting $DESC" "$NAME"
	do_stop
	case "$?" in
	  0|1)
		do_start
		case "$?" in
			0) log_end_msg 0 ;;
			1) log_end_msg 1 ;; # Old process is still running
			*) log_end_msg 1 ;; # Failed to start
		esac
		;;
	  *)
	  	# Failed to stop
		log_end_msg 1
		;;
	esac
	;;
  *)
	#echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2
	echo "Usage: $SCRIPTNAME {start|stop|restart|force-reload}" >&2
	exit 3
	;;
esac

: