20 from twisted.web2 import channel, http, server, iweb

    21 from twisted.web import http, server

    21 from twisted.web2 import static, resource, responsecode

    22 from twisted.web import static, resource

    23 from twisted.web.server import NOT_DONE_YET

    24 

    25 from logilab.common.decorators import monkeypatch

    70 class LongTimeExpiringFile(static.File):

    76 class ForbiddenDirectoryLister(resource.Resource):

    71     """overrides static.File and sets a far futre ``Expires`` date

    77     def render(self, request):

    78         return HTTPResponse(twisted_request=request,

    79                             code=http.FORBIDDEN,

    80                             stream='Access forbidden')

    81 

    82 class File(static.File):

    83     """Prevent from listing directories"""

    84     def directoryListing(self):

    85         return ForbiddenDirectoryLister()

    86 

    87 

    88 class LongTimeExpiringFile(File):

    89     """overrides static.File and sets a far future ``Expires`` date

    82     def renderHTTP(self, request):

   100     def render(self, request):

    91         d = maybeDeferred(super(LongTimeExpiringFile, self).renderHTTP, request)

   108         d = maybeDeferred(super(LongTimeExpiringFile, self).render, request)

    95 class CubicWebRootResource(resource.PostableResource):

   112 class CubicWebRootResource(resource.Resource):

    96     addSlash = False

    97 

   148     def locateChild(self, request, segments):

   164     def getChild(self, path, request):

   150         if segments:

   166         pre_path = request.prePathURL()

   151             if segments[0] == 'https':

   167         # XXX testing pre_path[0] not enough?

   152                 segments = segments[1:]

   168         if any(s in pre_path

   153             if len(segments) >= 2:

   169                for s in (self.versioned_datadir, 'data', 'static')):

   154                 if segments[0] in (self.versioned_datadir, 'data', 'static'):

   170             # Anything in data/, static/ is treated as static files

   155                     # Anything in data/, static/ is treated as static files

   171 

   156                     if segments[0] == 'static':

   172             if 'static' in pre_path:

   157                         # instance static directory

   173                 # instance static directory

   158                         datadir = self.config.static_directory

   174                 datadir = self.config.static_directory

   159                     elif segments[1] == 'fckeditor':

   175             elif 'fckeditor' in pre_path:

   160                         fckeditordir = self.config.ext_resources['FCKEDITOR_PATH']

   176                 fckeditordir = self.config.ext_resources['FCKEDITOR_PATH']

   161                         return static.File(fckeditordir), segments[2:]

   177                 return File(fckeditordir)

   162                     else:

   178             else:

   163                         # cube static data file

   179                 # cube static data file

   164                         datadir = self.config.locate_resource(segments[1])

   180                 datadir = self.config.locate_resource(path)

   165                         if datadir is None:

   181                 if datadir is None:

   166                             return None, []

   182                     return self

   167                     self.debug('static file %s from %s', segments[-1], datadir)

   183             self.info('static file %s from %s', path, datadir)

   168                     if segments[0] == 'data':

   184             if 'data' in pre_path:

   169                         return static.File(str(datadir)), segments[1:]

   185                 return File(os.path.join(datadir, path))

   170                     else:

   186             else:

   171                         return LongTimeExpiringFile(datadir), segments[1:]

   187                 return LongTimeExpiringFile(datadir)

   172                 elif segments[0] == 'fckeditor':

   188         elif path == 'fckeditor':

   173                     fckeditordir = self.config.ext_resources['FCKEDITOR_PATH']

   189             fckeditordir = self.config.ext_resources['FCKEDITOR_PATH']

   174                     return static.File(fckeditordir), segments[1:]

   190             return File(fckeditordir)

   176         return self, ()

   192         return self

   186             return threads.deferToThread(self.render_request, request)

   202             deferred = threads.deferToThread(self.render_request, request)

   203             return NOT_DONE_YET

   212             return self.request_auth(req)

   229             return self.request_auth(request=req)

   214             return self.redirect(req, ex.location)

   231             return self.redirect(request=req, location=ex.location)

   217             return self.request_auth(req)

   234             return self.request_auth(request=req)

   234             return http.Response(stream=ex.content, code=ex.status,

   251             return HTTPResponse(stream=ex.content, code=ex.status,

   235                                  headers=req.headers_out or None)

   252                                 twisted_request=req._twreq,

   253                                 headers=req.headers_out)

   238             return http.Response(stream=ex.dumps(),

   256             return HTTPResponse(twisted_request=req._twreq, code=http.INTERNAL_SERVER_ERROR,

   239                                  code=responsecode.INTERNAL_SERVER_ERROR)

   257                                 stream=ex.dumps(), headers=req.headers_out)

   242             return http.Response(stream=result, code=responsecode.NOT_FOUND,

   260             return HTTPResponse(twisted_request=req._twreq, code=http.NOT_FOUND,

   243                                  headers=req.headers_out or None)

   261                                 stream=result, headers=req.headers_out)

   262 

   245             return self.request_auth(req)

   264             return self.request_auth(request=req)

   248                 return self.redirect(req, ex.url)

   267                 return self.redirect(request=req, location=ex.url)

   251             return self.request_auth(req, loggedout=True)

   270             return self.request_auth(request=req, loggedout=True)

   253             return self.redirect(req, ex.location)

   272             return self.redirect(request=req, location=ex.location)

   257         return http.Response(stream=result, code=responsecode.OK,

   276 

   258                              headers=req.headers_out or None)

   277         return HTTPResponse(twisted_request=req._twreq, code=http.OK,

   259 

   278                             stream=result, headers=req.headers_out)

   260     def redirect(self, req, location):

   279 

   261         req.headers_out.setHeader('location', str(location))

   280     def redirect(self, request, location):

   262         self.debug('redirecting to %s', location)

   281         self.debug('redirecting to %s', str(location))

   282         request.headers_out.setHeader('location', str(location))

   264         return http.Response(code=303, headers=req.headers_out)

   284         return HTTPResponse(twisted_request=request._twreq, code=303,

   265 

   285                             headers=request.headers_out)

   266     def request_auth(self, req, loggedout=False):

   286 

   267         if self.https_url and req.base_url() != self.https_url:

   287     def request_auth(self, request, loggedout=False):

   268             req.headers_out.setHeader('location', self.https_url + 'login')

   288         if self.https_url and request.base_url() != self.https_url:

   269             return http.Response(code=303, headers=req.headers_out)

   289             return self.redirect(request, self.https_url + 'login')

   271             code = responsecode.UNAUTHORIZED

   291             code = http.UNAUTHORIZED

   272         else:

   292         else:

   273             code = responsecode.FORBIDDEN

   293             code = http.FORBIDDEN

   275             if req.https:

   295             if request.https:

   276                 req._base_url =  self.base_url

   296                 request._base_url =  self.base_url

   277                 req.https = False

   297                 request.https = False

   278             content = self.appli.loggedout_content(req)

   298             content = self.appli.loggedout_content(request)

   279         else:

   299         else:

   280             content = self.appli.need_login_content(req)

   300             content = self.appli.need_login_content(request)

   281         return http.Response(code, req.headers_out, content)

   301         return HTTPResponse(twisted_request=request._twreq,

   282 

   302                             stream=content, code=code,

   283 from twisted.internet import defer

   303                             headers=request.headers_out)

   284 from twisted.web2 import fileupload

   304 

   285 

   305 #TODO

   286 # XXX set max file size to 100Mo: put max upload size in the configuration

   306 # # XXX max upload size in the configuration

   287 # line below for twisted >= 8.0, default param value for earlier version

   307 

   288 resource.PostableResource.maxSize = 100*1024*1024

   308 @monkeypatch(http.Request)

   289 def parsePOSTData(request, maxMem=100*1024, maxFields=1024,

   309 def requestReceived(self, command, path, version):

   290                   maxSize=100*1024*1024):

   310     """Called by channel when all data has been received.

   291     if request.stream.length == 0:

   311 

   292         return defer.succeed(None)

   312     This method is not intended for users.

   293 

   313     """

   294     ctype = request.headers.getHeader('content-type')

   314     self.content.seek(0,0)

   295 

   315     self.args = {}

   296     if ctype is None:

   316     self.files = {}

   297         return defer.succeed(None)

   317     self.stack = []

   298 

   318     self.method, self.uri = command, path

   299     def updateArgs(data):

   319     self.clientproto = version

   300         args = data

   320     x = self.uri.split('?', 1)

   301         request.args.update(args)

   321     if len(x) == 1:

   302 

   322         self.path = self.uri

   303     def updateArgsAndFiles(data):

   304         args, files = data

   305         request.args.update(args)

   306         request.files.update(files)

   307 

   308     def error(f):

   309         f.trap(fileupload.MimeFormatError)

   310         raise http.HTTPError(responsecode.BAD_REQUEST)

   311 

   312     if ctype.mediaType == 'application' and ctype.mediaSubtype == 'x-www-form-urlencoded':

   313         d = fileupload.parse_urlencoded(request.stream, keep_blank_values=True)

   314         d.addCallbacks(updateArgs, error)

   315         return d

   316     elif ctype.mediaType == 'multipart' and ctype.mediaSubtype == 'form-data':

   317         boundary = ctype.params.get('boundary')

   318         if boundary is None:

   319             return defer.fail(http.HTTPError(

   320                 http.StatusResponse(responsecode.BAD_REQUEST,

   321                                     "Boundary not specified in Content-Type.")))

   322         d = fileupload.parseMultipartFormData(request.stream, boundary,

   323                                               maxMem, maxFields, maxSize)

   324         d.addCallbacks(updateArgsAndFiles, error)

   325         return d

   327         raise http.HTTPError(responsecode.BAD_REQUEST)

   324         self.path, argstring = x

   328 

   325         self.args = http.parse_qs(argstring, 1)

   329 server.parsePOSTData = parsePOSTData

   326     # cache the client and server information, we'll need this later to be

   327     # serialized and sent with the request so CGIs will work remotely

   328     self.client = self.channel.transport.getPeer()

   329     self.host = self.channel.transport.getHost()

   330     # Argument processing

   331     ctype = self.getHeader('content-type')

   332     if self.method == "POST" and ctype:

   333         key, pdict = parse_header(ctype)

   334         if key == 'application/x-www-form-urlencoded':

   335             self.args.update(http.parse_qs(self.content.read(), 1))

   336         elif key == 'multipart/form-data':

   337             self.content.seek(0,0)

   338             form = FieldStorage(self.content, self.received_headers,

   339                                 environ={'REQUEST_METHOD': 'POST'},

   340                                 keep_blank_values=1,

   341                                 strict_parsing=1)

   342             for key in form:

   343                 value = form[key]

   344                 if isinstance(value, list):

   345                     self.args[key] = [v.value for v in value]

   346                 elif value.filename:

   347                     if value.done != -1: # -1 is transfer has been interrupted

   348                         self.files[key] = (value.filename, value.file)

   349                     else:

   350                         self.files[key] = (None, None)

   351                 else:

   352                     self.args[key] = value.value

   353     self.process()

   334 set_log_methods(CubicWebRootResource, getLogger('cubicweb.twisted'))

   358 LOGGER = getLogger('cubicweb.twisted')

   335 

   359 set_log_methods(CubicWebRootResource, LOGGER)

   336 

   337 listiterator = type(iter([]))

   338 

   339 def _gc_debug(all=True):

   340     import gc

   341     from pprint import pprint

   342     from cubicweb.appobject import AppObject

   343     gc.collect()

   344     count = 0

   345     acount = 0

   346     fcount = 0

   347     rcount = 0

   348     ccount = 0

   349     scount = 0

   350     ocount = {}

   351     from rql.stmts import Union

   352     from cubicweb.schema import CubicWebSchema

   353     from cubicweb.rset import ResultSet

   354     from cubicweb.dbapi import Connection, Cursor

   355     from cubicweb.req import RequestSessionBase

   356     from cubicweb.server.repository import Repository

   357     from cubicweb.server.sources.native import NativeSQLSource

   358     from cubicweb.server.session import Session

   359     from cubicweb.devtools.testlib import CubicWebTC

   360     from logilab.common.testlib import TestSuite

   361     from optparse import Values

   362     import types, weakref

   363     for obj in gc.get_objects():

   364         if isinstance(obj, RequestSessionBase):

   365             count += 1

   366             if isinstance(obj, Session):

   367                 print '   session', obj, referrers(obj, True)

   368         elif isinstance(obj, AppObject):

   369             acount += 1

   370         elif isinstance(obj, ResultSet):

   371             rcount += 1

   372             #print '   rset', obj, referrers(obj)

   373         elif isinstance(obj, Repository):

   374             print '   REPO', obj, referrers(obj, True)

   375         #elif isinstance(obj, NativeSQLSource):

   376         #    print '   SOURCe', obj, referrers(obj)

   377         elif isinstance(obj, CubicWebTC):

   378             print '   TC', obj, referrers(obj)

   379         elif isinstance(obj, TestSuite):

   380             print '   SUITE', obj, referrers(obj)

   381         #elif isinstance(obj, Values):

   382         #    print '   values', '%#x' % id(obj), referrers(obj, True)

   383         elif isinstance(obj, Connection):

   384             ccount += 1

   385             #print '   cnx', obj, referrers(obj)

   386         #elif isinstance(obj, Cursor):

   387         #    ccount += 1

   388         #    print '   cursor', obj, referrers(obj)

   389         elif isinstance(obj, file):

   390             fcount += 1

   391         #    print '   open file', file.name, file.fileno

   392         elif isinstance(obj, CubicWebSchema):

   393             scount += 1

   394             print '   schema', obj, referrers(obj)

   395         elif not isinstance(obj, (type, tuple, dict, list, set, frozenset,

   396                                   weakref.ref, weakref.WeakKeyDictionary,

   397                                   listiterator,

   398                                   property, classmethod,

   399                                   types.ModuleType, types.MemberDescriptorType,

   400                                   types.FunctionType, types.MethodType)):

   401             try:

   402                 ocount[obj.__class__] += 1

   403             except KeyError:

   404                 ocount[obj.__class__] = 1

   405             except AttributeError:

   406                 pass

   407     if count:

   408         print ' NB REQUESTS/SESSIONS', count

   409     if acount:

   410         print ' NB APPOBJECTS', acount

   411     if ccount:

   412         print ' NB CONNECTIONS', ccount

   413     if rcount:

   414         print ' NB RSETS', rcount

   415     if scount:

   416         print ' NB SCHEMAS', scount

   417     if fcount:

   418         print ' NB FILES', fcount

   419     if all:

   420         ocount = sorted(ocount.items(), key=lambda x: x[1], reverse=True)[:20]

   421         pprint(ocount)

   422     if gc.garbage:

   423         print 'UNREACHABLE', gc.garbage

   424 

   425 def referrers(obj, showobj=False):

   426     try:

   427         return sorted(set((type(x), showobj and x or getattr(x, '__name__', '%#x' % id(x)))

   428                           for x in _referrers(obj)))

   429     except TypeError:

   430         s = set()

   431         unhashable = []

   432         for x in _referrers(obj):

   433             try:

   434                 s.add(x)

   435             except TypeError:

   436                 unhashable.append(x)

   437         return sorted(s) + unhashable

   438 

   439 def _referrers(obj, seen=None, level=0):

   440     import gc, types

   441     from cubicweb.schema import CubicWebRelationSchema, CubicWebEntitySchema

   442     interesting = []

   443     if seen is None:

   444         seen = set()

   445     for x in gc.get_referrers(obj):

   446         if id(x) in seen:

   447             continue

   448         seen.add(id(x))

   449         if isinstance(x, types.FrameType):

   450             continue

   451         if isinstance(x, (CubicWebRelationSchema, CubicWebEntitySchema)):

   452             continue

   453         if isinstance(x, (list, tuple, set, dict, listiterator)):

   454             if level >= 5:

   455                 pass

   456                 #interesting.append(x)

   457             else:

   458                 interesting += _referrers(x, seen, level+1)

   459         else:

   460             interesting.append(x)

   461     return interesting

   469     reactor.listenTCP(port, channel.HTTPFactory(website))

   367     reactor.listenTCP(port, website)