Mercurial Mercurial > pub > hg > cubicweb / comparison
summary | shortlog | changelog | graph | tags | bookmarks | branches | files | changeset | file | latest | revisions | annotate | diff | comparison | raw | help
Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
etwist/server.py
brancholdstable
changeset 5422 0865e1e90674
parent 5421 8167de96c523
child 5423 e15abfdcce38
child 5424 8ecbcbff9777
equal deleted inserted replaced
4985:02b52bf9f5f8 5422:0865e1e90674 
       
     1 # copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
 
       
     2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
 
       
     3 #
 
       
     4 # This file is part of CubicWeb.
 
       
     5 #
 
       
     6 # CubicWeb is free software: you can redistribute it and/or modify it under the
 
       
     7 # terms of the GNU Lesser General Public License as published by the Free
 
       
     8 # Software Foundation, either version 2.1 of the License, or (at your option)
 
       
     9 # any later version.
 
       
    10 #
 
       
    11 # logilab-common is distributed in the hope that it will be useful, but WITHOUT
 
       
    12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 
       
    13 # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
 
       
    14 # details.
 
       
    15 #
 
       
    16 # You should have received a copy of the GNU Lesser General Public License along
 
       
    17 # with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
 
     1 """twisted server for CubicWeb web instances
 
    18 """twisted server for CubicWeb web instances
 
     2 
    19 
     3 :organization: Logilab
 
       
     4 :copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2.
 
       
     5 :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
 
       
     6 :license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses
 
       
     7 """
 
    20 """
 
     8 __docformat__ = "restructuredtext en"
 
    21 __docformat__ = "restructuredtext en"
 
     9 
    22 
    10 import sys
 
    23 import sys
 
    11 import os
 
    24 import os
 
    12 import select
 
    25 import select
 
    13 import errno
 
    26 import errno
 
    14 import hotshot
 
       
    15 from time import mktime
 
    27 from time import mktime
 
    16 from datetime import date, timedelta
 
    28 from datetime import date, timedelta
 
    17 from urlparse import urlsplit, urlunsplit
 
    29 from urlparse import urlsplit, urlunsplit
 
    18 
    30 
    19 from twisted.internet import reactor, task, threads
 
    31 from twisted.internet import reactor, task, threads
 
   111         # when we have an in-memory repository, clean unused sessions every XX
 
   123         # when we have an in-memory repository, clean unused sessions every XX
 
   112         # seconds and properly shutdown the server
 
   124         # seconds and properly shutdown the server
 
   113         if config.repo_method == 'inmemory':
 
   125         if config.repo_method == 'inmemory':
 
   114             reactor.addSystemEventTrigger('before', 'shutdown',
 
   126             reactor.addSystemEventTrigger('before', 'shutdown',
 
   115                                           self.shutdown_event)
 
   127                                           self.shutdown_event)
 
   116             # monkey patch start_looping_task to get proper reactor integration
 
       
   117             #self.appli.repo.__class__.start_looping_tasks = start_looping_tasks
 
       
   118             if config.pyro_enabled():
 
   128             if config.pyro_enabled():
 
   119                 # if pyro is enabled, we have to register to the pyro name
 
   129                 # if pyro is enabled, we have to register to the pyro name
 
   120                 # server, create a pyro daemon, and create a task to handle pyro
 
   130                 # server, create a pyro daemon, and create a task to handle pyro
 
   121                 # requests
 
   131                 # requests
 
   122                 self.pyro_daemon = self.appli.repo.pyro_register()
 
   132                 self.pyro_daemon = self.appli.repo.pyro_register()
 
   125             self.appli.repo.start_looping_tasks()
 
   135             self.appli.repo.start_looping_tasks()
 
   126         self.set_url_rewriter()
 
   136         self.set_url_rewriter()
 
   127         CW_EVENT_MANAGER.bind('after-registry-reload', self.set_url_rewriter)
 
   137         CW_EVENT_MANAGER.bind('after-registry-reload', self.set_url_rewriter)
 
   128 
   138 
   129     def start_service(self):
 
   139     def start_service(self):
 
   130         config = self.config
 
   140         start_task(self.appli.session_handler.clean_sessions_interval,
 
   131         interval = min(config['cleanup-session-time'] or 120,
 
   141                    self.appli.session_handler.clean_sessions)
 
   132                        config['cleanup-anonymous-session-time'] or 720) / 2.
 
       
   133         start_task(interval, self.appli.session_handler.clean_sessions)
 
       
   134 
   142 
   135     def set_url_rewriter(self):
 
   143     def set_url_rewriter(self):
 
   136         self.url_rewriter = self.appli.vreg['components'].select_or_none('urlrewriter')
 
   144         self.url_rewriter = self.appli.vreg['components'].select_or_none('urlrewriter')
 
   137 
   145 
   138     def shutdown_event(self):
 
   146     def shutdown_event(self):
 
   165                     else:
 
   173                     else:
 
   166                         # cube static data file
 
   174                         # cube static data file
 
   167                         datadir = self.config.locate_resource(segments[1])
 
   175                         datadir = self.config.locate_resource(segments[1])
 
   168                         if datadir is None:
 
   176                         if datadir is None:
 
   169                             return None, []
 
   177                             return None, []
 
   170                     self.info('static file %s from %s', segments[-1], datadir)
 
   178                     self.debug('static file %s from %s', segments[-1], datadir)
 
   171                     if segments[0] == 'data':
 
   179                     if segments[0] == 'data':
 
   172                         return static.File(str(datadir)), segments[1:]
 
   180                         return static.File(str(datadir)), segments[1:]
 
   173                     else:
 
   181                     else:
 
   174                         return LongTimeExpiringFile(datadir), segments[1:]
 
   182                         return LongTimeExpiringFile(datadir), segments[1:]
 
   175                 elif segments[0] == 'fckeditor':
 
   183                 elif segments[0] == 'fckeditor':
 
   180 
   188 
   181     def render(self, request):
 
   189     def render(self, request):
 
   182         """Render a page from the root resource"""
 
   190         """Render a page from the root resource"""
 
   183         # reload modified files in debug mode
 
   191         # reload modified files in debug mode
 
   184         if self.debugmode:
 
   192         if self.debugmode:
 
   185             self.appli.vreg.register_objects(self.config.vregistry_path())
 
   193             self.appli.vreg.reload_if_needed()
 
   186         if self.config['profile']: # default profiler don't trace threads
 
   194         if self.config['profile']: # default profiler don't trace threads
 
   187             return self.render_request(request)
 
   195             return self.render_request(request)
 
   188         else:
 
   196         else:
 
   189             return threads.deferToThread(self.render_request, request)
 
   197             return threads.deferToThread(self.render_request, request)
 
   190 
   198 
   284         return http.Response(code, req.headers_out, content)
 
   292         return http.Response(code, req.headers_out, content)
 
   285 
   293 
   286 from twisted.internet import defer
 
   294 from twisted.internet import defer
 
   287 from twisted.web2 import fileupload
 
   295 from twisted.web2 import fileupload
 
   288 
   296 
   289 # XXX set max file size to 100Mo: put max upload size in the configuration
 
   297 # XXX set max file size to 200MB: put max upload size in the configuration
 
   290 # line below for twisted >= 8.0, default param value for earlier version
 
   298 # line below for twisted >= 8.0, default param value for earlier version
 
   291 resource.PostableResource.maxSize = 100*1024*1024
 
   299 resource.PostableResource.maxSize = 200*1024*1024
 
   292 def parsePOSTData(request, maxMem=100*1024, maxFields=1024,
 
   300 def parsePOSTData(request, maxMem=100*1024, maxFields=1024,
 
   293                   maxSize=100*1024*1024):
 
   301                   maxSize=200*1024*1024):
 
   294     if request.stream.length == 0:
 
   302     if request.stream.length == 0:
 
   295         return defer.succeed(None)
 
   303         return defer.succeed(None)
 
   296 
   304 
   297     ctype = request.headers.getHeader('content-type')
 
   305     ctype = request.headers.getHeader('content-type')
 
   298 
   306 
   335 from logging import getLogger
 
   343 from logging import getLogger
 
   336 from cubicweb import set_log_methods
 
   344 from cubicweb import set_log_methods
 
   337 set_log_methods(CubicWebRootResource, getLogger('cubicweb.twisted'))
 
   345 set_log_methods(CubicWebRootResource, getLogger('cubicweb.twisted'))
 
   338 
   346 
   339 
   347 
   340 
   348 listiterator = type(iter([]))
 
   341 def _gc_debug():
 
   349 
       
   350 def _gc_debug(all=True):
 
   342     import gc
 
   351     import gc
 
   343     from pprint import pprint
 
   352     from pprint import pprint
 
   344     from cubicweb.appobject import AppObject
 
   353     from cubicweb.appobject import AppObject
 
   345     gc.collect()
 
   354     gc.collect()
 
   346     count = 0
 
   355     count = 0
 
   347     acount = 0
 
   356     acount = 0
 
       
   357     fcount = 0
 
       
   358     rcount = 0
 
       
   359     ccount = 0
 
       
   360     scount = 0
 
   348     ocount = {}
 
   361     ocount = {}
 
       
   362     from rql.stmts import Union
 
       
   363     from cubicweb.schema import CubicWebSchema
 
       
   364     from cubicweb.rset import ResultSet
 
       
   365     from cubicweb.dbapi import Connection, Cursor
 
       
   366     from cubicweb.req import RequestSessionBase
 
       
   367     from cubicweb.server.repository import Repository
 
       
   368     from cubicweb.server.sources.native import NativeSQLSource
 
       
   369     from cubicweb.server.session import Session
 
       
   370     from cubicweb.devtools.testlib import CubicWebTC
 
       
   371     from logilab.common.testlib import TestSuite
 
       
   372     from optparse import Values
 
       
   373     import types, weakref
 
   349     for obj in gc.get_objects():
 
   374     for obj in gc.get_objects():
 
   350         if isinstance(obj, CubicWebTwistedRequestAdapter):
 
   375         if isinstance(obj, RequestSessionBase):
 
   351             count += 1
 
   376             count += 1
 
       
   377             if isinstance(obj, Session):
 
       
   378                 print '   session', obj, referrers(obj, True)
 
   352         elif isinstance(obj, AppObject):
 
   379         elif isinstance(obj, AppObject):
 
   353             acount += 1
 
   380             acount += 1
 
   354         else:
 
   381         elif isinstance(obj, ResultSet):
 
       
   382             rcount += 1
 
       
   383             #print '   rset', obj, referrers(obj)
 
       
   384         elif isinstance(obj, Repository):
 
       
   385             print '   REPO', obj, referrers(obj, True)
 
       
   386         #elif isinstance(obj, NativeSQLSource):
 
       
   387         #    print '   SOURCe', obj, referrers(obj)
 
       
   388         elif isinstance(obj, CubicWebTC):
 
       
   389             print '   TC', obj, referrers(obj)
 
       
   390         elif isinstance(obj, TestSuite):
 
       
   391             print '   SUITE', obj, referrers(obj)
 
       
   392         #elif isinstance(obj, Values):
 
       
   393         #    print '   values', '%#x' % id(obj), referrers(obj, True)
 
       
   394         elif isinstance(obj, Connection):
 
       
   395             ccount += 1
 
       
   396             #print '   cnx', obj, referrers(obj)
 
       
   397         #elif isinstance(obj, Cursor):
 
       
   398         #    ccount += 1
 
       
   399         #    print '   cursor', obj, referrers(obj)
 
       
   400         elif isinstance(obj, file):
 
       
   401             fcount += 1
 
       
   402         #    print '   open file', file.name, file.fileno
 
       
   403         elif isinstance(obj, CubicWebSchema):
 
       
   404             scount += 1
 
       
   405             print '   schema', obj, referrers(obj)
 
       
   406         elif not isinstance(obj, (type, tuple, dict, list, set, frozenset,
 
       
   407                                   weakref.ref, weakref.WeakKeyDictionary,
 
       
   408                                   listiterator,
 
       
   409                                   property, classmethod,
 
       
   410                                   types.ModuleType, types.MemberDescriptorType,
 
       
   411                                   types.FunctionType, types.MethodType)):
 
   355             try:
 
   412             try:
 
   356                 ocount[obj.__class__] += 1
 
   413                 ocount[obj.__class__] += 1
 
   357             except KeyError:
 
   414             except KeyError:
 
   358                 ocount[obj.__class__] = 1
 
   415                 ocount[obj.__class__] = 1
 
   359             except AttributeError:
 
   416             except AttributeError:
 
   360                 pass
 
   417                 pass
 
   361     print 'IN MEM REQUESTS', count
 
   418     if count:
 
   362     print 'IN MEM APPOBJECTS', acount
 
   419         print ' NB REQUESTS/SESSIONS', count
 
   363     ocount = sorted(ocount.items(), key=lambda x: x[1], reverse=True)[:20]
 
   420     if acount:
 
   364     pprint(ocount)
 
   421         print ' NB APPOBJECTS', acount
 
   365     print 'UNREACHABLE', gc.garbage
 
   422     if ccount:
 
       
   423         print ' NB CONNECTIONS', ccount
 
       
   424     if rcount:
 
       
   425         print ' NB RSETS', rcount
 
       
   426     if scount:
 
       
   427         print ' NB SCHEMAS', scount
 
       
   428     if fcount:
 
       
   429         print ' NB FILES', fcount
 
       
   430     if all:
 
       
   431         ocount = sorted(ocount.items(), key=lambda x: x[1], reverse=True)[:20]
 
       
   432         pprint(ocount)
 
       
   433     if gc.garbage:
 
       
   434         print 'UNREACHABLE', gc.garbage
 
       
   435 
       
   436 def referrers(obj, showobj=False):
 
       
   437     try:
 
       
   438         return sorted(set((type(x), showobj and x or getattr(x, '__name__', '%#x' % id(x)))
 
       
   439                           for x in _referrers(obj)))
 
       
   440     except TypeError:
 
       
   441         s = set()
 
       
   442         unhashable = []
 
       
   443         for x in _referrers(obj):
 
       
   444             try:
 
       
   445                 s.add(x)
 
       
   446             except TypeError:
 
       
   447                 unhashable.append(x)
 
       
   448         return sorted(s) + unhashable
 
       
   449 
       
   450 def _referrers(obj, seen=None, level=0):
 
       
   451     import gc, types
 
       
   452     from cubicweb.schema import CubicWebRelationSchema, CubicWebEntitySchema
 
       
   453     interesting = []
 
       
   454     if seen is None:
 
       
   455         seen = set()
 
       
   456     for x in gc.get_referrers(obj):
 
       
   457         if id(x) in seen:
 
       
   458             continue
 
       
   459         seen.add(id(x))
 
       
   460         if isinstance(x, types.FrameType):
 
       
   461             continue
 
       
   462         if isinstance(x, (CubicWebRelationSchema, CubicWebEntitySchema)):
 
       
   463             continue
 
       
   464         if isinstance(x, (list, tuple, set, dict, listiterator)):
 
       
   465             if level >= 5:
 
       
   466                 pass
 
       
   467                 #interesting.append(x)
 
       
   468             else:
 
       
   469                 interesting += _referrers(x, seen, level+1)
 
       
   470         else:
 
       
   471             interesting.append(x)
 
       
   472     return interesting
 
   366 
   473 
   367 def run(config, debug):
 
   474 def run(config, debug):
 
   368     # create the site
 
   475     # create the site
 
   369     root_resource = CubicWebRootResource(config, debug)
 
   476     root_resource = CubicWebRootResource(config, debug)
 
   370     website = server.Site(root_resource)
 
   477     website = server.Site(root_resource)
 
   395             uid = getpwnam(config['uid']).pw_uid
 
   502             uid = getpwnam(config['uid']).pw_uid
 
   396         os.setuid(uid)
 
   503         os.setuid(uid)
 
   397     root_resource.start_service()
 
   504     root_resource.start_service()
 
   398     logger.info('instance started on %s', root_resource.base_url)
 
   505     logger.info('instance started on %s', root_resource.base_url)
 
   399     if config['profile']:
 
   506     if config['profile']:
 
   400         prof = hotshot.Profile(config['profile'])
 
   507         import cProfile
 
   401         prof.runcall(reactor.run)
 
   508         cProfile.runctx('reactor.run()', globals(), locals(), config['profile'])
 
   402     else:
 
   509     else:
 
   403         reactor.run()
 
   510         reactor.run()
cubicweb