[twisted] move out url logic from twisted (closes #2200593)
Data, fck and static file are now served by cubicweb controller by default.
Some hackish url rewriting is necessary to have them selected. This is necessary
because cubicweb does not selection controller on multi level path.
- 'Expires' header seems to conflict with Cache-Control set by HTTPCacheManger
(don't know yet why HTTPCacheManager is involved here)
mod_wsgi is now usable in simple cases (e.g. still requires persisten sessions).
--- a/etwist/server.py Thu Mar 01 17:22:56 2012 +0100
+++ b/etwist/server.py Mon Feb 27 17:02:31 2012 +0100
@@ -69,166 +69,6 @@
return baseurl
-class ForbiddenDirectoryLister(resource.Resource):
- def render(self, request):
- return HTTPResponse(twisted_request=request,
- code=http.FORBIDDEN,
- stream='Access forbidden')
-
-
-class NoListingFile(static.File):
- def __init__(self, config, path=None):
- if path is None:
- path = config.static_directory
- static.File.__init__(self, path)
- self.config = config
-
- def set_expires(self, request):
- if not self.config.debugmode:
- # XXX: Don't provide additional resource information to error responses
- #
- # the HTTP RFC recommands not going further than 1 year ahead
- expires = date.today() + timedelta(days=6*30)
- request.setHeader('Expires', generateDateTime(mktime(expires.timetuple())))
-
- def directoryListing(self):
- return ForbiddenDirectoryLister()
-
-
-class DataLookupDirectory(NoListingFile):
- def __init__(self, config, path):
- self.md5_version = config.instance_md5_version()
- NoListingFile.__init__(self, config, path)
- self.here = path
- self._defineChildResources()
- if self.config.debugmode:
- self.data_modconcat_basepath = '/data/??'
- else:
- self.data_modconcat_basepath = '/data/%s/??' % self.md5_version
-
- def _defineChildResources(self):
- self.putChild(self.md5_version, self)
-
- def getChild(self, path, request):
- if not path:
- uri = request.uri
- if uri.startswith('/https/'):
- uri = uri[6:]
- if uri.startswith(self.data_modconcat_basepath):
- resource_relpath = uri[len(self.data_modconcat_basepath):]
- if resource_relpath:
- paths = resource_relpath.split(',')
- try:
- self.set_expires(request)
- return ConcatFiles(self.config, paths)
- except ConcatFileNotFoundError:
- return self.childNotFound
- return self.directoryListing()
- childpath = join(self.here, path)
- dirpath, rid = self.config.locate_resource(childpath)
- if dirpath is None:
- # resource not found
- return self.childNotFound
- filepath = os.path.join(dirpath, rid)
- if os.path.isdir(filepath):
- resource = DataLookupDirectory(self.config, childpath)
- # cache resource for this segment path to avoid recomputing
- # directory lookup
- self.putChild(path, resource)
- return resource
- else:
- self.set_expires(request)
- return NoListingFile(self.config, filepath)
-
-
-class FCKEditorResource(NoListingFile):
-
- def getChild(self, path, request):
- pre_path = request.path.split('/')[1:]
- if pre_path[0] == 'https':
- pre_path.pop(0)
- uiprops = self.config.https_uiprops
- else:
- uiprops = self.config.uiprops
- return static.File(osp.join(uiprops['FCKEDITOR_PATH'], path))
-
-
-class LongTimeExpiringFile(DataLookupDirectory):
- """overrides static.File and sets a far future ``Expires`` date
- on the resouce.
-
- versions handling is done by serving static files by different
- URLs for each version. For instance::
-
- http://localhost:8080/data-2.48.2/cubicweb.css
- http://localhost:8080/data-2.49.0/cubicweb.css
- etc.
-
- """
- def _defineChildResources(self):
- pass
-
-
-class ConcatFileNotFoundError(CubicWebException):
- pass
-
-
-class ConcatFiles(LongTimeExpiringFile):
- def __init__(self, config, paths):
- _, ext = osp.splitext(paths[0])
- self._resources = {}
- # create a unique / predictable filename. We don't consider cubes
- # version since uicache is cleared at server startup, and file's dates
- # are checked in debug mode
- fname = 'cache_concat_' + md5(';'.join(paths)).hexdigest() + ext
- filepath = osp.join(config.appdatahome, 'uicache', fname)
- LongTimeExpiringFile.__init__(self, config, filepath)
- self._concat_cached_filepath(filepath, paths)
-
- def _resource(self, path):
- try:
- return self._resources[path]
- except KeyError:
- self._resources[path] = self.config.locate_resource(path)
- return self._resources[path]
-
- def _concat_cached_filepath(self, filepath, paths):
- if not self._up_to_date(filepath, paths):
- with open(filepath, 'wb') as f:
- for path in paths:
- dirpath, rid = self._resource(path)
- if rid is None:
- # In production mode log an error, do not return a 404
- # XXX the erroneous content is cached anyway
- LOGGER.error('concatenated data url error: %r file '
- 'does not exist', path)
- if self.config.debugmode:
- raise ConcatFileNotFoundError(path)
- else:
- for line in open(osp.join(dirpath, rid)):
- f.write(line)
- f.write('\n')
-
- def _up_to_date(self, filepath, paths):
- """
- The concat-file is considered up-to-date if it exists.
- In debug mode, an additional check is performed to make sure that
- concat-file is more recent than all concatenated files
- """
- if not osp.isfile(filepath):
- return False
- if self.config.debugmode:
- concat_lastmod = os.stat(filepath).st_mtime
- for path in paths:
- dirpath, rid = self._resource(path)
- if rid is None:
- raise ConcatFileNotFoundError(path)
- path = osp.join(dirpath, rid)
- if os.stat(path).st_mtime > concat_lastmod:
- return False
- return True
-
-
class CubicWebRootResource(resource.Resource):
def __init__(self, config, vreg=None):
resource.Resource.__init__(self)
@@ -240,9 +80,6 @@
self.https_url = config['https-url']
global MAX_POST_LENGTH
MAX_POST_LENGTH = config['max-post-length']
- self.putChild('static', NoListingFile(config))
- self.putChild('fckeditor', FCKEditorResource(self.config, ''))
- self.putChild('data', DataLookupDirectory(self.config, ''))
def init_publisher(self):
config = self.config
--- a/etwist/test/unittest_server.py Thu Mar 01 17:22:56 2012 +0100
+++ b/etwist/test/unittest_server.py Mon Feb 27 17:02:31 2012 +0100
@@ -19,8 +19,7 @@
import os, os.path as osp, glob
from cubicweb.devtools.testlib import CubicWebTC
-from cubicweb.etwist.server import (host_prefixed_baseurl, ConcatFiles,
- ConcatFileNotFoundError)
+from cubicweb.etwist.server import host_prefixed_baseurl
class HostPrefixedBaseURLTC(CubicWebTC):
@@ -54,36 +53,6 @@
self._check('http://localhost:8080/hg/', 'code.cubicweb.org',
'http://localhost:8080/hg/')
-
if __name__ == '__main__':
from logilab.common.testlib import unittest_main
unittest_main()
-
-
-
-class ConcatFilesTC(CubicWebTC):
-
- def tearDown(self):
- super(ConcatFilesTC, self).tearDown()
- self._cleanup_concat_cache()
- self.config.debugmode = False
-
- def _cleanup_concat_cache(self):
- uicachedir = osp.join(self.config.apphome, 'uicache')
- for fname in glob.glob(osp.join(uicachedir, 'cache_concat_*')):
- os.unlink(osp.join(uicachedir, fname))
-
- def test_cache(self):
- concat = ConcatFiles(self.config, ('cubicweb.ajax.js', 'jquery.js'))
- self.assertTrue(osp.isfile(concat.path))
-
- def test_404(self):
- # when not in debug mode, should not crash
- ConcatFiles(self.config, ('cubicweb.ajax.js', 'dummy.js'))
- # in debug mode, raise error
- self.config.debugmode = True
- try:
- self.assertRaises(ConcatFileNotFoundError, ConcatFiles, self.config,
- ('cubicweb.ajax.js', 'dummy.js'))
- finally:
- self.config.debugmode = False
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/web/test/unittest_views_staticcontrollers.py Mon Feb 27 17:02:31 2012 +0100
@@ -0,0 +1,86 @@
+from __future__ import with_statement
+
+from cubicweb.devtools.testlib import CubicWebTC
+
+import os
+import os.path as osp
+import glob
+
+from cubicweb.utils import HTMLHead
+from cubicweb.web import StatusResponse
+from cubicweb.web.views.staticcontrollers import ConcatFilesHandler
+
+class ConcatFilesTC(CubicWebTC):
+
+ def tearDown(self):
+ super(ConcatFilesTC, self).tearDown()
+ self._cleanup_concat_cache()
+
+ def _cleanup_concat_cache(self):
+ uicachedir = osp.join(self.config.apphome, 'uicache')
+ for fname in glob.glob(osp.join(uicachedir, 'cache_concat_*')):
+ os.unlink(osp.join(uicachedir, fname))
+
+ def _publish_js_files(self, js_files):
+ req = self.request()
+ head = HTMLHead(req)
+ url = head.concat_urls([req.data_url(js_file) for js_file in js_files])[len(req.base_url()):]
+ req._url = url
+ return self.app_publish(req, url)
+
+ def expected_content(self, js_files):
+ content = u''
+ for js_file in js_files:
+ dirpath, rid = self.config.locate_resource(js_file)
+ if dirpath is not None: # ignore resources not found
+ with open(osp.join(dirpath, rid)) as f:
+ content += f.read() + '\n'
+ return content
+
+ def test_cache(self):
+ js_files = ('cubicweb.ajax.js', 'jquery.js')
+ try:
+ result = self._publish_js_files(js_files)
+ except StatusResponse, exc:
+ if exc.status == 404:
+ self.fail('unable to serve cubicweb.js+jquery.js')
+ # let the exception propagate for any other status (e.g 500)
+ raise
+ # check result content
+ self.assertEqual(result, self.expected_content(js_files))
+ # make sure we kept a cached version on filesystem
+ concat_hander = ConcatFilesHandler(self.config)
+ filepath = concat_hander.build_filepath(js_files)
+ self.assertTrue(osp.isfile(filepath))
+
+
+ def test_invalid_file_in_debug_mode(self):
+ js_files = ('cubicweb.ajax.js', 'dummy.js')
+ # in debug mode, an error is raised
+ self.config.debugmode = True
+ try:
+ result = self._publish_js_files(js_files)
+ self.fail('invalid concat js should return a 404 in debug mode')
+ except StatusResponse, exc:
+ if exc.status != 404:
+ self.fail('invalid concat js should return a 404 in debug mode')
+ finally:
+ self.config.debugmode = False
+
+ def test_invalid_file_in_production_mode(self):
+ js_files = ('cubicweb.ajax.js', 'dummy.js')
+ try:
+ result = self._publish_js_files(js_files)
+ except StatusResponse, exc:
+ if exc.status == 404:
+ self.fail('invalid concat js should NOT return a 404 in debug mode')
+ # let the exception propagate for any other status (e.g 500)
+ raise
+ # check result content
+ self.assertEqual(result, self.expected_content(js_files))
+
+
+if __name__ == '__main__':
+ from logilab.common.testlib import unittest_main
+ unittest_main()
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/web/views/staticcontrollers.py Mon Feb 27 17:02:31 2012 +0100
@@ -0,0 +1,230 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of CubicWeb.
+#
+# CubicWeb is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option)
+# any later version.
+#
+# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
+"""Set of static resources controllers for :
+
+- /data/...
+- /static/...
+- /fckeditor/...
+
+"""
+
+import os
+import os.path as osp
+import hashlib
+import mimetypes
+from time import mktime
+from datetime import datetime, timedelta
+from logging import getLogger
+
+from cubicweb.web import NotFound
+from cubicweb.web.http_headers import generateDateTime
+from cubicweb.web.controller import Controller
+from cubicweb.web.views.urlrewrite import URLRewriter
+
+
+
+class StaticFileController(Controller):
+ """an abtract class to serve static file
+
+ Make sure to add your subclass to the STATIC_CONTROLLERS list"""
+ __abstract__ = True
+ directory_listing_allowed = False
+
+ def max_age(self, path):
+ """max cache TTL"""
+ return 60*60*24*7
+
+ def static_file(self, path):
+ """Return full content of a static file.
+
+ XXX iterable content would be better
+ """
+ debugmode = self._cw.vreg.config.debugmode
+ if osp.isdir(path):
+ if self.directory_listing_allowed:
+ return u''
+ raise NotFound(path)
+ if not osp.isfile(path):
+ raise NotFound()
+ if not debugmode:
+ # XXX: Don't provide additional resource information to error responses
+ #
+ # the HTTP RFC recommands not going further than 1 year ahead
+ expires = datetime.now() + timedelta(days=6*30)
+ self._cw.set_header('Expires', generateDateTime(mktime(expires.timetuple())))
+ # XXX elif uri.startswith('/https/'): uri = uri[6:]
+ mimetype, encoding = mimetypes.guess_type(path)
+ self._cw.set_content_type(mimetype, osp.basename(path), encoding)
+ return file(path).read()
+
+ @property
+ def relpath(self):
+ """path of a requested file relative to the controller"""
+ path = self._cw.form.get('static_relative_path')
+ if path is None:
+ path = self._cw.relative_path(includeparams=True)
+ return path
+
+
+class ConcatFilesHandler(object):
+ """Emulating the behavior of modconcat
+
+ this serve multiple file as a single one.
+ """
+
+ def __init__(self, config):
+ self._resources = {}
+ self.config = config
+ self.logger = getLogger('cubicweb.web')
+
+ def _resource(self, path):
+ """get the resouce"""
+ try:
+ return self._resources[path]
+ except KeyError:
+ self._resources[path] = self.config.locate_resource(path)
+ return self._resources[path]
+
+ def _up_to_date(self, filepath, paths):
+ """
+ The concat-file is considered up-to-date if it exists.
+ In debug mode, an additional check is performed to make sure that
+ concat-file is more recent than all concatenated files
+ """
+ if not osp.isfile(filepath):
+ return False
+ if self.config.debugmode:
+ concat_lastmod = os.stat(filepath).st_mtime
+ for path in paths:
+ dirpath, rid = self._resource(path)
+ if rid is None:
+ raise NotFound(path)
+ path = osp.join(dirpath, rid)
+ if os.stat(path).st_mtime > concat_lastmod:
+ return False
+ return True
+
+ def build_filepath(self, paths):
+ """return the filepath that will be used to cache concatenation of `paths`
+ """
+ _, ext = osp.splitext(paths[0])
+ fname = 'cache_concat_' + hashlib.md5(';'.join(paths)).hexdigest() + ext
+ return osp.join(self.config.appdatahome, 'uicache', fname)
+
+ def concat_cached_filepath(self, paths):
+ filepath = self.build_filepath(paths)
+ if not self._up_to_date(filepath, paths):
+ with open(filepath, 'wb') as f:
+ for path in paths:
+ dirpath, rid = self._resource(path)
+ if rid is None:
+ # In production mode log an error, do not return a 404
+ # XXX the erroneous content is cached anyway
+ self.logger.error('concatenated data url error: %r file '
+ 'does not exist', path)
+ if self.config.debugmode:
+ raise NotFound(path)
+ else:
+ for line in open(osp.join(dirpath, rid)):
+ f.write(line)
+ f.write('\n')
+ return filepath
+
+
+class DataController(StaticFileController):
+ """Controller in charge of serving static file in /data/
+
+ Handle modeconcat like url.
+ """
+
+ __regid__ = 'data'
+
+ def __init__(self, *args, **kwargs):
+ super(DataController, self).__init__(*args, **kwargs)
+ config = self._cw.vreg.config
+ md5_version = config.instance_md5_version()
+ self.base_datapath = config.data_relpath()
+ self.data_modconcat_basepath = '%s??' % self.base_datapath
+ self.concat_files_registry = ConcatFilesHandler(config)
+
+ def publish(self, rset=None):
+ config = self._cw.vreg.config
+ # includeparams=True for modconcat-like urls
+ relpath = self.relpath
+ if relpath.startswith(self.data_modconcat_basepath):
+ paths = relpath[len(self.data_modconcat_basepath):].split(',')
+ filepath = self.concat_files_registry.concat_cached_filepath(paths)
+ return self.static_file(filepath)
+ else:
+ relpath = relpath[len(self.base_datapath):] # skip leading '/data/'
+ dirpath, rid = config.locate_resource(relpath)
+ if dirpath is None:
+ raise NotFound()
+ return self.static_file(osp.join(dirpath, rid))
+
+
+class FCKEditorController(StaticFileController):
+ """Controller in charge of serving FCKEditor related file
+
+ The motivational for a dedicated controller have been lost.
+ """
+
+ __regid__ = 'fckeditor'
+
+ def publish(self, rset=None):
+ config = self._cw.vreg.config
+ if self._cw.https:
+ uiprops = config.https_uiprops
+ else:
+ uiprops = config.uiprops
+ relpath = self.relpath
+ return self.static_file(osp.join(uiprops['FCKEDITOR_PATH'], relpath))
+
+
+class StaticDirectoryController(StaticFileController):
+ """Controller in charge of serving static file in /static/
+ """
+ __regid__ = 'static'
+
+ def publish(self, rset=None):
+ staticdir = self._cw.vreg.config.static_directory
+ relpath = self.relpath
+ return self.static_file(osp.join(staticdir, relpath))
+
+STATIC_CONTROLLERS = [DataController, FCKEditorController,
+ StaticDirectoryController]
+
+class StaticControlerRewriter(URLRewriter):
+ """a quick and dirty rewritter in charge of server static file.
+
+ This is a work around the flatness of url handling in cubicweb."""
+
+ __regid__ = 'static'
+
+ priority = 10
+
+ def rewrite(self, req, uri):
+ for ctrl in STATIC_CONTROLLERS:
+ if uri.startswith('/%s/' % ctrl.__regid__):
+ break
+ else:
+ self.debug("not a static file uri: %s", uri)
+ raise KeyError(uri)
+ relpath = self._cw.relative_path(includeparams=False)
+ self._cw.form['static_relative_path'] = self._cw.relative_path(includeparams=True)
+ return ctrl.__regid__, None