1 """

     2 Provides a 'pyramid' command as a replacement to the 'start' command.

     3 

     4 The reloading strategy is heavily inspired by (and partially copied from)

     5 the pyramid script 'pserve'.

     6 """

     7 from __future__ import print_function

     8 

     9 import atexit

    10 import errno

    11 import os

    12 import signal

    13 import sys

    14 import tempfile

    15 import time

    16 import threading

    17 import subprocess

    18 

    19 from cubicweb import BadCommandUsage, ExecutionError

    20 from cubicweb.__pkginfo__ import numversion as cwversion

    21 from cubicweb.cwconfig import CubicWebConfiguration as cwcfg

    22 from cubicweb.cwctl import CWCTL, InstanceCommand, init_cmdline_log_threshold

    23 from cubicweb.pyramid import wsgi_application_from_cwconfig

    24 from cubicweb.server import set_debug

    25 

    26 import waitress

    27 

    28 MAXFD = 1024

    29 

    30 DBG_FLAGS = ('RQL', 'SQL', 'REPO', 'HOOKS', 'OPS', 'SEC', 'MORE')

    31 LOG_LEVELS = ('debug', 'info', 'warning', 'error')

    32 

    33 

    34 class PyramidStartHandler(InstanceCommand):

    35     """Start an interactive pyramid server.

    36 

    37     This command requires http://hg.logilab.org/review/pyramid_cubicweb/

    38 

    39     <instance>

    40       identifier of the instance to configure.

    41     """

    42     name = 'pyramid'

    43 

    44     options = (

    45         ('no-daemon',

    46          {'action': 'store_true',

    47           'help': 'Run the server in the foreground.'}),

    48         ('debug-mode',

    49          {'action': 'store_true',

    50           'help': 'Activate the repository debug mode ('

    51                   'logs in the console and the debug toolbar).'

    52                   ' Implies --no-daemon'}),

    53         ('debug',

    54          {'short': 'D', 'action': 'store_true',

    55           'help': 'Equals to "--debug-mode --no-daemon --reload"'}),

    56         ('reload',

    57          {'action': 'store_true',

    58           'help': 'Restart the server if any source file is changed'}),

    59         ('reload-interval',

    60          {'type': 'int', 'default': 1,

    61           'help': 'Interval, in seconds, between file modifications checks'}),

    62         ('loglevel',

    63          {'short': 'l', 'type': 'choice', 'metavar': '<log level>',

    64           'default': None, 'choices': LOG_LEVELS,

    65           'help': 'debug if -D is set, error otherwise; '

    66                   'one of %s' % (LOG_LEVELS,),

    67           }),

    68         ('dbglevel',

    69          {'type': 'multiple_choice', 'metavar': '<dbg level>',

    70           'default': None,

    71           'choices': DBG_FLAGS,

    72           'help': ('Set the server debugging flags; you may choose several '

    73                    'values in %s; imply "debug" loglevel' % (DBG_FLAGS,)),

    74           }),

    75         ('profile',

    76          {'action': 'store_true',

    77           'default': False,

    78           'help': 'Enable profiling'}),

    79         ('profile-output',

    80          {'type': 'string',

    81           'default': None,

    82           'help': 'Profiling output file (default: "program.prof")'}),

    83         ('profile-dump-every',

    84          {'type': 'int',

    85           'default': None,

    86           'metavar': 'N',

    87           'help': 'Dump profile stats to ouput every N requests '

    88                   '(default: 100)'}),

    89     )

    90     if cwversion >= (3, 21, 0):

    91         options = options + (

    92             ('param',

    93              {'short': 'p',

    94               'type': 'named',

    95               'metavar': 'key1:value1,key2:value2',

    96               'default': {},

    97               'help': 'override <key> configuration file option with <value>.',

    98               }),

    99         )

   100 

   101     _reloader_environ_key = 'CW_RELOADER_SHOULD_RUN'

   102     _reloader_filelist_environ_key = 'CW_RELOADER_FILELIST'

   103 

   104     def debug(self, msg):

   105         print('DEBUG - %s' % msg)

   106 

   107     def info(self, msg):

   108         print('INFO - %s' % msg)

   109 

   110     def ordered_instances(self):

   111         instances = super(PyramidStartHandler, self).ordered_instances()

   112         if (self['debug-mode'] or self['debug'] or self['reload']) \

   113                 and len(instances) > 1:

   114             raise BadCommandUsage(

   115                 '--debug-mode, --debug and --reload can be used on a single '

   116                 'instance only')

   117         return instances

   118 

   119     def quote_first_command_arg(self, arg):

   120         """

   121         There's a bug in Windows when running an executable that's

   122         located inside a path with a space in it.  This method handles

   123         that case, or on non-Windows systems or an executable with no

   124         spaces, it just leaves well enough alone.

   125         """

   126         if (sys.platform != 'win32' or ' ' not in arg):

   127             # Problem does not apply:

   128             return arg

   129         try:

   130             import win32api

   131         except ImportError:

   132             raise ValueError(

   133                 "The executable %r contains a space, and in order to "

   134                 "handle this issue you must have the win32api module "

   135                 "installed" % arg)

   136         arg = win32api.GetShortPathName(arg)

   137         return arg

   138 

   139     def _remove_pid_file(self, written_pid, filename):

   140         current_pid = os.getpid()

   141         if written_pid != current_pid:

   142             # A forked process must be exiting, not the process that

   143             # wrote the PID file

   144             return

   145         if not os.path.exists(filename):

   146             return

   147         with open(filename) as f:

   148             content = f.read().strip()

   149         try:

   150             pid_in_file = int(content)

   151         except ValueError:

   152             pass

   153         else:

   154             if pid_in_file != current_pid:

   155                 msg = "PID file %s contains %s, not expected PID %s"

   156                 self.out(msg % (filename, pid_in_file, current_pid))

   157                 return

   158         self.info("Removing PID file %s" % filename)

   159         try:

   160             os.unlink(filename)

   161             return

   162         except OSError as e:

   163             # Record, but don't give traceback

   164             self.out("Cannot remove PID file: (%s)" % e)

   165         # well, at least lets not leave the invalid PID around...

   166         try:

   167             with open(filename, 'w') as f:

   168                 f.write('')

   169         except OSError as e:

   170             self.out('Stale PID left in file: %s (%s)' % (filename, e))

   171         else:

   172             self.out('Stale PID removed')

   173 

   174     def record_pid(self, pid_file):

   175         pid = os.getpid()

   176         self.debug('Writing PID %s to %s' % (pid, pid_file))

   177         with open(pid_file, 'w') as f:

   178             f.write(str(pid))

   179         atexit.register(

   180             self._remove_pid_file, pid, pid_file)

   181 

   182     def daemonize(self, pid_file):

   183         pid = live_pidfile(pid_file)

   184         if pid:

   185             raise ExecutionError(

   186                 "Daemon is already running (PID: %s from PID file %s)"

   187                 % (pid, pid_file))

   188 

   189         self.debug('Entering daemon mode')

   190         pid = os.fork()

   191         if pid:

   192             # The forked process also has a handle on resources, so we

   193             # *don't* want proper termination of the process, we just

   194             # want to exit quick (which os._exit() does)

   195             os._exit(0)

   196         # Make this the session leader

   197         os.setsid()

   198         # Fork again for good measure!

   199         pid = os.fork()

   200         if pid:

   201             os._exit(0)

   202 

   203         # @@: Should we set the umask and cwd now?

   204 

   205         import resource  # Resource usage information.

   206         maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]

   207         if (maxfd == resource.RLIM_INFINITY):

   208             maxfd = MAXFD

   209         # Iterate through and close all file descriptors.

   210         for fd in range(0, maxfd):

   211             try:

   212                 os.close(fd)

   213             except OSError:  # ERROR, fd wasn't open to begin with (ignored)

   214                 pass

   215 

   216         if (hasattr(os, "devnull")):

   217             REDIRECT_TO = os.devnull

   218         else:

   219             REDIRECT_TO = "/dev/null"

   220         os.open(REDIRECT_TO, os.O_RDWR)  # standard input (0)

   221         # Duplicate standard input to standard output and standard error.

   222         os.dup2(0, 1)  # standard output (1)

   223         os.dup2(0, 2)  # standard error (2)

   224 

   225     def restart_with_reloader(self):

   226         self.debug('Starting subprocess with file monitor')

   227 

   228         with tempfile.NamedTemporaryFile(delete=False) as f:

   229             filelist_path = f.name

   230 

   231         while True:

   232             args = [self.quote_first_command_arg(sys.executable)] + sys.argv

   233             new_environ = os.environ.copy()

   234             new_environ[self._reloader_environ_key] = 'true'

   235             new_environ[self._reloader_filelist_environ_key] = filelist_path

   236             proc = None

   237             try:

   238                 try:

   239                     proc = subprocess.Popen(args, env=new_environ)

   240                     exit_code = proc.wait()

   241                     proc = None

   242                     print("Process exited with", exit_code)

   243                 except KeyboardInterrupt:

   244                     self.info('^C caught in monitor process')

   245                     return 1

   246             finally:

   247                 if proc is not None:

   248                     proc.terminate()

   249                     self.info(

   250                         'Waiting for the server to stop. Hit CTRL-C to exit')

   251                     exit_code = proc.wait()

   252 

   253             if exit_code != 3:

   254                 with open(filelist_path) as f:

   255                     filelist = [line.strip() for line in f]

   256                 if filelist:

   257                     self.info("Reloading failed. Waiting for a file to change")

   258                     mon = Monitor(extra_files=filelist, nomodules=True)

   259                     while mon.check_reload():

   260                         time.sleep(1)

   261                 else:

   262                     return exit_code

   263 

   264             self.info('%s %s %s' % ('-' * 20, 'Restarting', '-' * 20))

   265 

   266     def set_needreload(self):

   267         self._needreload = True

   268 

   269     def install_reloader(self, poll_interval, extra_files, filelist_path):

   270         mon = Monitor(

   271             poll_interval=poll_interval, extra_files=extra_files,

   272             atexit=self.set_needreload, filelist_path=filelist_path)

   273         mon_thread = threading.Thread(target=mon.periodic_reload)

   274         mon_thread.daemon = True

   275         mon_thread.start()

   276 

   277     def configfiles(self, cwconfig):

   278         """Generate instance configuration filenames"""

   279         yield cwconfig.main_config_file()

   280         for f in (

   281                 'sources', 'logging.conf', 'pyramid.ini', 'pyramid-debug.ini'):

   282             f = os.path.join(cwconfig.apphome, f)

   283             if os.path.exists(f):

   284                 yield f

   285 

   286     def i18nfiles(self, cwconfig):

   287         """Generate instance i18n files"""

   288         i18ndir = os.path.join(cwconfig.apphome, 'i18n')

   289         if os.path.exists(i18ndir):

   290             for lang in cwconfig.available_languages():

   291                 f = os.path.join(i18ndir, lang, 'LC_MESSAGES', 'cubicweb.mo')

   292                 if os.path.exists(f):

   293                     yield f

   294 

   295     def pyramid_instance(self, appid):

   296         self._needreload = False

   297 

   298         debugmode = self['debug-mode'] or self['debug']

   299         autoreload = self['reload'] or self['debug']

   300         daemonize = not (self['no-daemon'] or debugmode or autoreload)

   301 

   302         if autoreload and not os.environ.get(self._reloader_environ_key):

   303             return self.restart_with_reloader()

   304 

   305         cwconfig = cwcfg.config_for(appid, debugmode=debugmode)

   306         if cwversion >= (3, 21, 0):

   307             cwconfig.cmdline_options = self.config.param

   308         if autoreload:

   309             _turn_sigterm_into_systemexit()

   310             self.debug('Running reloading file monitor')

   311             extra_files = [sys.argv[0]]

   312             extra_files.extend(self.configfiles(cwconfig))

   313             extra_files.extend(self.i18nfiles(cwconfig))

   314             self.install_reloader(

   315                 self['reload-interval'], extra_files,

   316                 filelist_path=os.environ.get(

   317                     self._reloader_filelist_environ_key))

   318 

   319         if daemonize:

   320             self.daemonize(cwconfig['pid-file'])

   321             self.record_pid(cwconfig['pid-file'])

   322 

   323         if self['dbglevel']:

   324             self['loglevel'] = 'debug'

   325             set_debug('|'.join('DBG_' + x.upper() for x in self['dbglevel']))

   326         init_cmdline_log_threshold(cwconfig, self['loglevel'])

   327 

   328         app = wsgi_application_from_cwconfig(

   329             cwconfig, profile=self['profile'],

   330             profile_output=self['profile-output'],

   331             profile_dump_every=self['profile-dump-every']

   332         )

   333 

   334         host = cwconfig['interface']

   335         port = cwconfig['port'] or 8080

   336         repo = app.application.registry['cubicweb.repository']

   337         try:

   338             repo.start_looping_tasks()

   339             waitress.serve(app, host=host, port=port)

   340         finally:

   341             repo.shutdown()

   342         if self._needreload:

   343             return 3

   344         return 0

   345 

   346 CWCTL.register(PyramidStartHandler)

   347 

   348 

   349 def live_pidfile(pidfile):  # pragma: no cover

   350     """(pidfile:str) -> int | None

   351     Returns an int found in the named file, if there is one,

   352     and if there is a running process with that process id.

   353     Return None if no such process exists.

   354     """

   355     pid = read_pidfile(pidfile)

   356     if pid:

   357         try:

   358             os.kill(int(pid), 0)

   359             return pid

   360         except OSError as e:

   361             if e.errno == errno.EPERM:

   362                 return pid

   363     return None

   364 

   365 

   366 def read_pidfile(filename):

   367     if os.path.exists(filename):

   368         try:

   369             with open(filename) as f:

   370                 content = f.read()

   371             return int(content.strip())

   372         except (ValueError, IOError):

   373             return None

   374     else:

   375         return None

   376 

   377 

   378 def _turn_sigterm_into_systemexit():

   379     """Attempts to turn a SIGTERM exception into a SystemExit exception."""

   380     try:

   381         import signal

   382     except ImportError:

   383         return

   384 

   385     def handle_term(signo, frame):

   386         raise SystemExit

   387     signal.signal(signal.SIGTERM, handle_term)

   388 

   389 

   390 class Monitor(object):

   391     """A file monitor and server stopper.

   392 

   393     It is a simplified version of pyramid pserve.Monitor, with little changes:

   394 

   395     -   The constructor takes extra_files, atexit, nomodules and filelist_path

   396     -   The process is stopped by auto-kill with signal SIGTERM

   397     """

   398 

   399     def __init__(self, poll_interval=1, extra_files=[], atexit=None,

   400                  nomodules=False, filelist_path=None):

   401         self.module_mtimes = {}

   402         self.keep_running = True

   403         self.poll_interval = poll_interval

   404         self.extra_files = extra_files

   405         self.atexit = atexit

   406         self.nomodules = nomodules

   407         self.filelist_path = filelist_path

   408 

   409     def _exit(self):

   410         if self.atexit:

   411             self.atexit()

   412         os.kill(os.getpid(), signal.SIGTERM)

   413 

   414     def periodic_reload(self):

   415         while True:

   416             if not self.check_reload():

   417                 self._exit()

   418                 break

   419             time.sleep(self.poll_interval)

   420 

   421     def check_reload(self):

   422         filenames = list(self.extra_files)

   423 

   424         if not self.nomodules:

   425             for module in list(sys.modules.values()):

   426                 try:

   427                     filename = module.__file__

   428                 except (AttributeError, ImportError):

   429                     continue

   430                 if filename is not None:

   431                     filenames.append(filename)

   432 

   433         for filename in filenames:

   434             try:

   435                 stat = os.stat(filename)

   436                 if stat:

   437                     mtime = stat.st_mtime

   438                 else:

   439                     mtime = 0

   440             except (OSError, IOError):

   441                 continue

   442             if filename.endswith('.pyc') and os.path.exists(filename[:-1]):

   443                 mtime = max(os.stat(filename[:-1]).st_mtime, mtime)

   444             if filename not in self.module_mtimes:

   445                 self.module_mtimes[filename] = mtime

   446             elif self.module_mtimes[filename] < mtime:

   447                 print('%s changed; reloading...' % filename)

   448                 return False

   449 

   450         if self.filelist_path:

   451             with open(self.filelist_path) as f:

   452                 filelist = set((line.strip() for line in f))

   453 

   454             filelist.update(filenames)

   455 

   456             with open(self.filelist_path, 'w') as f:

   457                 for filename in filelist:

   458                     f.write('%s\n' % filename)

   459 

   460         return True