--- a/ccplugin.py Thu Sep 18 12:03:25 2014 +0200
+++ b/ccplugin.py Thu Sep 18 22:33:04 2014 +0200
@@ -1,6 +1,27 @@
+"""
+Provides a 'pyramid' command as a replacement to the 'start' command.
+
+The reloading strategy is heavily inspired by (and partially copied from)
+the pyramid script 'pserve'.
+"""
+import atexit
+import errno
+import os
+import signal
+import sys
+import time
+import threading
+import subprocess
+
+from cubicweb import BadCommandUsage, ExecutionError
from cubicweb.cwconfig import CubicWebConfiguration as cwcfg
from cubicweb.cwctl import CWCTL, InstanceCommand, init_cmdline_log_threshold
+from pyramid_cubicweb import make_cubicweb_application
+import waitress
+
+MAXFD = 1024
+
class PyramidStartHandler(InstanceCommand):
"""Start an interactive pyramid server.
@@ -15,7 +36,13 @@
options = (
("debug",
{'short': 'D', 'action': 'store_true',
- 'help': 'start server in debug mode.'}),
+ 'help': 'start server in the foreground.'}),
+ ('reload',
+ {'action': 'store_true',
+ 'help': 'Restart the server if any source file is changed'}),
+ ('reload-interval',
+ {'type': 'int', 'default': 1,
+ 'help': 'Interval, in seconds, between file modifications checks'}),
('loglevel',
{'short': 'l', 'type': 'choice', 'metavar': '<log level>',
'default': None, 'choices': ('debug', 'info', 'warning', 'error'),
@@ -23,10 +50,185 @@
}),
)
+ _reloader_environ_key = 'CW_RELOADER_SHOULD_RUN'
+
+ def debug(self, msg):
+ print('DEBUG - %s' % msg)
+
+ def info(self, msg):
+ print('INFO - %s' % msg)
+
+ def ordered_instances(self):
+ instances = super(PyramidStartHandler, self).ordered_instances()
+ if (self['debug'] or self['reload']) and len(instances) > 1:
+ raise BadCommandUsage(
+ '--debug and --reload can be used on a single instance only')
+ return instances
+
+ def quote_first_command_arg(self, arg):
+ """
+ There's a bug in Windows when running an executable that's
+ located inside a path with a space in it. This method handles
+ that case, or on non-Windows systems or an executable with no
+ spaces, it just leaves well enough alone.
+ """
+ if (sys.platform != 'win32' or ' ' not in arg):
+ # Problem does not apply:
+ return arg
+ try:
+ import win32api
+ except ImportError:
+ raise ValueError(
+ "The executable %r contains a space, and in order to "
+ "handle this issue you must have the win32api module "
+ "installed" % arg)
+ arg = win32api.GetShortPathName(arg)
+ return arg
+
+ def _remove_pid_file(self, written_pid, filename):
+ current_pid = os.getpid()
+ if written_pid != current_pid:
+ # A forked process must be exiting, not the process that
+ # wrote the PID file
+ return
+ if not os.path.exists(filename):
+ return
+ with open(filename) as f:
+ content = f.read().strip()
+ try:
+ pid_in_file = int(content)
+ except ValueError:
+ pass
+ else:
+ if pid_in_file != current_pid:
+ msg = "PID file %s contains %s, not expected PID %s"
+ self.out(msg % (filename, pid_in_file, current_pid))
+ return
+ self.info("Removing PID file %s" % filename)
+ try:
+ os.unlink(filename)
+ return
+ except OSError as e:
+ # Record, but don't give traceback
+ self.out("Cannot remove PID file: (%s)" % e)
+ # well, at least lets not leave the invalid PID around...
+ try:
+ with open(filename, 'w') as f:
+ f.write('')
+ except OSError as e:
+ self.out('Stale PID left in file: %s (%s)' % (filename, e))
+ else:
+ self.out('Stale PID removed')
+
+ def record_pid(self, pid_file):
+ pid = os.getpid()
+ self.debug('Writing PID %s to %s' % (pid, pid_file))
+ with open(pid_file, 'w') as f:
+ f.write(str(pid))
+ atexit.register(
+ self._remove_pid_file, pid, pid_file)
+
+ def daemonize(self, pid_file):
+ pid = live_pidfile(pid_file)
+ if pid:
+ raise ExecutionError(
+ "Daemon is already running (PID: %s from PID file %s)"
+ % (pid, pid_file))
+
+ self.debug('Entering daemon mode')
+ pid = os.fork()
+ if pid:
+ # The forked process also has a handle on resources, so we
+ # *don't* want proper termination of the process, we just
+ # want to exit quick (which os._exit() does)
+ os._exit(0)
+ # Make this the session leader
+ os.setsid()
+ # Fork again for good measure!
+ pid = os.fork()
+ if pid:
+ os._exit(0)
+
+ # @@: Should we set the umask and cwd now?
+
+ import resource # Resource usage information.
+ maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
+ if (maxfd == resource.RLIM_INFINITY):
+ maxfd = MAXFD
+ # Iterate through and close all file descriptors.
+ for fd in range(0, maxfd):
+ try:
+ os.close(fd)
+ except OSError: # ERROR, fd wasn't open to begin with (ignored)
+ pass
+
+ if (hasattr(os, "devnull")):
+ REDIRECT_TO = os.devnull
+ else:
+ REDIRECT_TO = "/dev/null"
+ os.open(REDIRECT_TO, os.O_RDWR) # standard input (0)
+ # Duplicate standard input to standard output and standard error.
+ os.dup2(0, 1) # standard output (1)
+ os.dup2(0, 2) # standard error (2)
+
+ def restart_with_reloader(self):
+ self.debug('Starting subprocess with file monitor')
+
+ while True:
+ args = [self.quote_first_command_arg(sys.executable)] + sys.argv
+ new_environ = os.environ.copy()
+ new_environ[self._reloader_environ_key] = 'true'
+ proc = None
+ try:
+ try:
+ proc = subprocess.Popen(args, env=new_environ)
+ exit_code = proc.wait()
+ proc = None
+ print "Process exited with ", exit_code
+ except KeyboardInterrupt:
+ self.info('^C caught in monitor process')
+ return 1
+ finally:
+ if proc is not None:
+ proc.terminate()
+ self.info(
+ 'Waiting for the server to stop. Hit CTRL-C to exit')
+ exit_code = proc.wait()
+
+ if exit_code != 3:
+ return exit_code
+
+ self.info('%s %s %s' % ('-' * 20, 'Restarting', '-' * 20))
+
+ def set_needreload(self):
+ self._needreload = True
+
+ def install_reloader(self, poll_interval, extra_files):
+ mon = Monitor(
+ poll_interval=poll_interval, extra_files=extra_files,
+ atexit=self.set_needreload)
+ mon_thread = threading.Thread(target=mon.periodic_reload)
+ mon_thread.daemon = True
+ mon_thread.start()
+
def pyramid_instance(self, appid):
- from pyramid_cubicweb import make_cubicweb_application
- from waitress import serve
+ self._needreload = False
+
+ if self['reload'] and not os.environ.get(self._reloader_environ_key):
+ return self.restart_with_reloader()
+
cwconfig = cwcfg.config_for(appid, debugmode=self['debug'])
+
+ if self['reload']:
+ _turn_sigterm_into_systemexit()
+ self.debug('Running reloading file monitor')
+ extra_files = [sys.argv[0], cwconfig.main_config_file()]
+ self.install_reloader(self['reload-interval'], extra_files)
+
+ if not self['reload'] and not self['debug']:
+ self.daemonize(cwconfig['pid-file'])
+ self.record_pid(cwconfig['pid-file'])
+
init_cmdline_log_threshold(cwconfig, self['loglevel'])
host = cwconfig['interface']
@@ -37,8 +239,114 @@
repo = cwconfig.repository()
try:
repo.start_looping_tasks()
- serve(pyramid_config.make_wsgi_app(), host=host, port=port)
+ waitress.serve(
+ pyramid_config.make_wsgi_app(), host=host, port=port)
finally:
repo.shutdown()
+ if self._needreload:
+ return 3
+ return 0
CWCTL.register(PyramidStartHandler)
+
+
+def live_pidfile(pidfile): # pragma: no cover
+ """(pidfile:str) -> int | None
+ Returns an int found in the named file, if there is one,
+ and if there is a running process with that process id.
+ Return None if no such process exists.
+ """
+ pid = read_pidfile(pidfile)
+ if pid:
+ try:
+ os.kill(int(pid), 0)
+ return pid
+ except OSError as e:
+ if e.errno == errno.EPERM:
+ return pid
+ return None
+
+
+def read_pidfile(filename):
+ if os.path.exists(filename):
+ try:
+ with open(filename) as f:
+ content = f.read()
+ return int(content.strip())
+ except (ValueError, IOError):
+ return None
+ else:
+ return None
+
+
+def _turn_sigterm_into_systemexit():
+ """
+ Attempts to turn a SIGTERM exception into a SystemExit exception.
+ """
+ try:
+ import signal
+ except ImportError:
+ return
+
+ def handle_term(signo, frame):
+ raise SystemExit
+ signal.signal(signal.SIGTERM, handle_term)
+
+
+class Monitor(object):
+ """
+ A file monitor and server stopper.
+
+ It is a simplified version of pyramid pserve.Monitor, with little changes:
+
+ - The constructor takes extra_files and atexit
+ - The process is stopped by auto-kill with signal SIGTERM
+ """
+ def __init__(self, poll_interval, extra_files=[], atexit=None):
+ self.module_mtimes = {}
+ self.keep_running = True
+ self.poll_interval = poll_interval
+ self.extra_files = extra_files
+ self.atexit = atexit
+
+ def _exit(self):
+ if self.atexit:
+ self.atexit()
+ os.kill(os.getpid(), signal.SIGTERM)
+
+ def periodic_reload(self):
+ while True:
+ if not self.check_reload():
+ self._exit()
+ break
+ time.sleep(self.poll_interval)
+
+ def check_reload(self):
+ filenames = list(self.extra_files)
+
+ for module in list(sys.modules.values()):
+ try:
+ filename = module.__file__
+ except (AttributeError, ImportError):
+ continue
+ if filename is not None:
+ filenames.append(filename)
+
+ for filename in filenames:
+ try:
+ stat = os.stat(filename)
+ if stat:
+ mtime = stat.st_mtime
+ else:
+ mtime = 0
+ except (OSError, IOError):
+ continue
+ if filename.endswith('.pyc') and os.path.exists(filename[:-1]):
+ mtime = max(os.stat(filename[:-1]).st_mtime, mtime)
+ if not filename in self.module_mtimes:
+ self.module_mtimes[filename] = mtime
+ elif self.module_mtimes[filename] < mtime:
+ print('%s changed; reloading...' % filename)
+ return False
+
+ return True