# HG changeset patch # User Adrien Di Mascio # Date 1304060910 -7200 # Node ID 1144a2d90314a3c5fcc125da19c6d0717e279132 # Parent acd7f0e9f2762f55c9501ca769d263d6a69fb39d# Parent 771f594c12a283df2a278567ce8cdfc5e8a32546 merge diff -r 771f594c12a2 -r 1144a2d90314 etwist/server.py --- a/etwist/server.py Wed Apr 27 16:38:01 2011 +0200 +++ b/etwist/server.py Fri Apr 29 09:08:30 2011 +0200 @@ -17,14 +17,19 @@ # with CubicWeb. If not, see . """twisted server for CubicWeb web instances""" +from __future__ import with_statement + __docformat__ = "restructuredtext en" import sys import os +import os.path as osp import select import errno import traceback import threading +import re +import hashlib from os.path import join from time import mktime from datetime import date, timedelta @@ -41,7 +46,8 @@ from logilab.common.decorators import monkeypatch -from cubicweb import AuthenticationError, ConfigurationError, CW_EVENT_MANAGER +from cubicweb import (AuthenticationError, ConfigurationError, + CW_EVENT_MANAGER, CubicWebException) from cubicweb.utils import json_dumps from cubicweb.web import Redirect, DirectResponse, StatusResponse, LogOut from cubicweb.web.application import CubicWebPublisher @@ -70,13 +76,70 @@ code=http.FORBIDDEN, stream='Access forbidden') -class File(static.File): - """Prevent from listing directories""" + +class NoListingFile(static.File): def directoryListing(self): return ForbiddenDirectoryLister() -class LongTimeExpiringFile(File): +class DataLookupDirectory(NoListingFile): + def __init__(self, config, path): + self.md5_version = config.instance_md5_version() + NoListingFile.__init__(self, path) + self.config = config + self.here = path + self._defineChildResources() + if self.config.debugmode: + self.data_modconcat_basepath = '/data/??' + else: + self.data_modconcat_basepath = '/data/%s/??' % self.md5_version + + def _defineChildResources(self): + self.putChild(self.md5_version, self) + + def getChild(self, path, request): + if not path: + if request.uri.startswith(self.data_modconcat_basepath): + resource_relpath = request.uri[len(self.data_modconcat_basepath):] + if resource_relpath: + paths = resource_relpath.split(',') + try: + return ConcatFiles(self.config, paths) + except ConcatFileNotFoundError: + return self.childNotFound + return self.directoryListing() + childpath = join(self.here, path) + dirpath, rid = self.config.locate_resource(childpath) + if dirpath is None: + # resource not found + return self.childNotFound + filepath = os.path.join(dirpath, rid) + if os.path.isdir(filepath): + resource = DataLookupDirectory(self.config, childpath) + # cache resource for this segment path to avoid recomputing + # directory lookup + self.putChild(path, resource) + return resource + else: + return NoListingFile(filepath) + + +class FCKEditorResource(NoListingFile): + def __init__(self, config, path): + NoListingFile.__init__(self, path) + self.config = config + + def getChild(self, path, request): + pre_path = request.path.split('/')[1:] + if pre_path[0] == 'https': + pre_path.pop(0) + uiprops = self.config.https_uiprops + else: + uiprops = self.config.uiprops + return static.File(osp.join(uiprops['FCKEDITOR_PATH'], path)) + + +class LongTimeExpiringFile(DataLookupDirectory): """overrides static.File and sets a far future ``Expires`` date on the resouce. @@ -88,28 +151,60 @@ etc. """ + def _defineChildResources(self): + pass + def render(self, request): # XXX: Don't provide additional resource information to error responses # # the HTTP RFC recommands not going further than 1 year ahead expires = date.today() + timedelta(days=6*30) request.setHeader('Expires', generateDateTime(mktime(expires.timetuple()))) - return File.render(self, request) + return DataLookupDirectory.render(self, request) + + +class ConcatFileNotFoundError(CubicWebException): + pass + + +class ConcatFiles(LongTimeExpiringFile): + def __init__(self, config, paths): + path = self._concat_cached_filepath(config, paths) + LongTimeExpiringFile.__init__(self, config, path) + + @staticmethod + def _concat_cached_filepath(config, paths): + _, ext = osp.splitext(paths[0]) + # create a unique / predictable filename + fname = hashlib.md5(';'.join(paths)).hexdigest() + ext + filepath = osp.join(config.appdatahome, 'uicache', fname) + if not osp.isfile(filepath): + concat_data = [] + for path in paths: + dirpath, rid = config.locate_resource(path) + if rid is None: + raise ConcatFileNotFoundError(path) + concat_data.append(open(osp.join(dirpath, rid)).read()) + with open(filepath, 'wb') as f: + f.write('\n'.join(concat_data)) + return filepath + class CubicWebRootResource(resource.Resource): def __init__(self, config, vreg=None): + resource.Resource.__init__(self) self.config = config # instantiate publisher here and not in init_publisher to get some # checks done before daemonization (eg versions consistency) self.appli = CubicWebPublisher(config, vreg=vreg) self.base_url = config['base-url'] self.https_url = config['https-url'] - self.children = {} - self.static_directories = set(('data%s' % config.instance_md5_version(), - 'data', 'static', 'fckeditor')) global MAX_POST_LENGTH MAX_POST_LENGTH = config['max-post-length'] + self.putChild('static', NoListingFile(config.static_directory)) + self.putChild('fckeditor', FCKEditorResource(self.config, '')) + self.putChild('data', DataLookupDirectory(self.config, '')) def init_publisher(self): config = self.config @@ -152,38 +247,6 @@ def getChild(self, path, request): """Indicate which resource to use to process down the URL's path""" - pre_path = request.path.split('/')[1:] - if pre_path[0] == 'https': - pre_path.pop(0) - uiprops = self.config.https_uiprops - else: - uiprops = self.config.uiprops - directory = pre_path[0] - # Anything in data/, static/, fckeditor/ and the generated versioned - # data directory is treated as static files - if directory in self.static_directories: - # take care fckeditor may appears as root directory or as a data - # subdirectory - if directory == 'static': - return File(self.config.static_directory) - if directory == 'fckeditor': - return File(uiprops['FCKEDITOR_PATH']) - if directory != 'data': - # versioned directory, use specific file with http cache - # headers so their are cached for a very long time - cls = LongTimeExpiringFile - else: - cls = File - if path == 'fckeditor': - return cls(uiprops['FCKEDITOR_PATH']) - if path == directory: # recurse - return self - datadir, path = self.config.locate_resource(path) - if datadir is None: - return self # recurse - self.debug('static file %s from %s', path, datadir) - return cls(join(datadir, path)) - # Otherwise we use this single resource return self def render(self, request): diff -r 771f594c12a2 -r 1144a2d90314 test/unittest_utils.py --- a/test/unittest_utils.py Wed Apr 27 16:38:01 2011 +0200 +++ b/test/unittest_utils.py Fri Apr 29 09:08:30 2011 +0200 @@ -22,8 +22,8 @@ import datetime from logilab.common.testlib import TestCase, unittest_main - -from cubicweb.utils import make_uid, UStringIO, SizeConstrainedList, RepeatList +from cubicweb.devtools.testlib import CubicWebTC +from cubicweb.utils import make_uid, UStringIO, SizeConstrainedList, RepeatList, HTMLHead from cubicweb.entity import Entity try: @@ -155,6 +155,102 @@ def test_encoding_unknown_stuff(self): self.assertEqual(self.encode(TestCase), 'null') +class HTMLHeadTC(CubicWebTC): + def test_concat_urls(self): + base_url = u'http://test.fr/data/' + head = HTMLHead(base_url) + urls = [base_url + u'bob1.js', + base_url + u'bob2.js', + base_url + u'bob3.js'] + result = head.concat_urls(urls) + expected = u'http://test.fr/data/??bob1.js,bob2.js,bob3.js' + self.assertEqual(result, expected) + + def test_group_urls(self): + base_url = u'http://test.fr/data/' + head = HTMLHead(base_url) + urls_spec = [(base_url + u'bob0.js', None), + (base_url + u'bob1.js', None), + (u'http://ext.com/bob2.js', None), + (u'http://ext.com/bob3.js', None), + (base_url + u'bob4.css', 'all'), + (base_url + u'bob5.css', 'all'), + (base_url + u'bob6.css', 'print'), + (base_url + u'bob7.css', 'print'), + (base_url + u'bob8.css', ('all', u'[if IE 8]')), + (base_url + u'bob9.css', ('print', u'[if IE 8]')) + ] + result = head.group_urls(urls_spec) + expected = [(base_url + u'??bob0.js,bob1.js', None), + (u'http://ext.com/bob2.js', None), + (u'http://ext.com/bob3.js', None), + (base_url + u'??bob4.css,bob5.css', 'all'), + (base_url + u'??bob6.css,bob7.css', 'print'), + (base_url + u'bob8.css', ('all', u'[if IE 8]')), + (base_url + u'bob9.css', ('print', u'[if IE 8]')) + ] + self.assertEqual(list(result), expected) + + def test_getvalue_with_concat(self): + base_url = u'http://test.fr/data/' + head = HTMLHead(base_url) + head.add_js(base_url + u'bob0.js') + head.add_js(base_url + u'bob1.js') + head.add_js(u'http://ext.com/bob2.js') + head.add_js(u'http://ext.com/bob3.js') + head.add_css(base_url + u'bob4.css') + head.add_css(base_url + u'bob5.css') + head.add_css(base_url + u'bob6.css', 'print') + head.add_css(base_url + u'bob7.css', 'print') + head.add_ie_css(base_url + u'bob8.css') + head.add_ie_css(base_url + u'bob9.css', 'print', u'[if lt IE 7]') + result = head.getvalue() + expected = u""" + + + + + + + +""" + self.assertEqual(result, expected) + + def test_getvalue_without_concat(self): + base_url = u'http://test.fr/data/' + head = HTMLHead() + head.add_js(base_url + u'bob0.js') + head.add_js(base_url + u'bob1.js') + head.add_js(u'http://ext.com/bob2.js') + head.add_js(u'http://ext.com/bob3.js') + head.add_css(base_url + u'bob4.css') + head.add_css(base_url + u'bob5.css') + head.add_css(base_url + u'bob6.css', 'print') + head.add_css(base_url + u'bob7.css', 'print') + head.add_ie_css(base_url + u'bob8.css') + head.add_ie_css(base_url + u'bob9.css', 'print', u'[if lt IE 7]') + result = head.getvalue() + expected = u""" + + + + + + + + + + +""" + self.assertEqual(result, expected) if __name__ == '__main__': unittest_main() diff -r 771f594c12a2 -r 1144a2d90314 utils.py --- a/utils.py Wed Apr 27 16:38:01 2011 +0200 +++ b/utils.py Fri Apr 29 09:08:30 2011 +0200 @@ -240,7 +240,7 @@ xhtml_safe_script_opening = u'' - def __init__(self): + def __init__(self, datadir_url=None): super(HTMLHead, self).__init__() self.jsvars = [] self.jsfiles = [] @@ -248,6 +248,7 @@ self.ie_cssfiles = [] self.post_inlined_scripts = [] self.pagedata_unload = False + self.datadir_url = datadir_url def add_raw(self, rawheader): @@ -284,7 +285,7 @@ if jsfile not in self.jsfiles: self.jsfiles.append(jsfile) - def add_css(self, cssfile, media): + def add_css(self, cssfile, media='all'): """adds `cssfile` to the list of javascripts used in the webpage This function checks if the file has already been added @@ -304,6 +305,45 @@ self.post_inlined_scripts.append(self.js_unload_code) self.pagedata_unload = True + def concat_urls(self, urls): + """concatenates urls into one url usable by Apache mod_concat + + This method returns the url without modifying it if there is only + one element in the list + :param urls: list of local urls/filenames to concatenate + """ + if len(urls) == 1: + return urls[0] + len_prefix = len(self.datadir_url) + concated = u','.join(url[len_prefix:] for url in urls) + return (u'%s??%s' % (self.datadir_url, concated)) + + def group_urls(self, urls_spec): + """parses urls_spec in order to generate concatenated urls + for js and css includes + + This method checks if the file is local and if it shares options + with direct neighbors + :param urls_spec: entire list of urls/filenames to inspect + """ + concatable = [] + prev_islocal = False + prev_key = None + for url, key in urls_spec: + islocal = url.startswith(self.datadir_url) + if concatable and (islocal != prev_islocal or key != prev_key): + yield (self.concat_urls(concatable), prev_key) + del concatable[:] + if not islocal: + yield (url, key) + else: + concatable.append(url) + prev_islocal = islocal + prev_key = key + if concatable: + yield (self.concat_urls(concatable), prev_key) + + def getvalue(self, skiphead=False): """reimplement getvalue to provide a consistent (and somewhat browser optimzed cf. http://stevesouders.com/cuzillion) order in external @@ -321,18 +361,20 @@ w(vardecl + u'\n') w(self.xhtml_safe_script_closing) # 2/ css files - for cssfile, media in self.cssfiles: + for cssfile, media in (self.group_urls(self.cssfiles) if self.datadir_url else self.cssfiles): w(u'\n' % (media, xml_escape(cssfile))) # 3/ ie css if necessary if self.ie_cssfiles: - for cssfile, media, iespec in self.ie_cssfiles: + ie_cssfiles = ((x, (y, z)) for x, y, z in self.ie_cssfiles) + for cssfile, (media, iespec) in (self.group_urls(ie_cssfiles) if self.datadir_url else ie_cssfiles): w(u' \n') # 4/ js files - for jsfile in self.jsfiles: + jsfiles = ((x, None) for x in self.jsfiles) + for jsfile, media in self.group_urls(jsfiles) if self.datadir_url else jsfiles: w(u'\n' % xml_escape(jsfile)) # 5/ post inlined scripts (i.e. scripts depending on other JS files) diff -r 771f594c12a2 -r 1144a2d90314 web/request.py --- a/web/request.py Wed Apr 27 16:38:01 2011 +0200 +++ b/web/request.py Fri Apr 29 09:08:30 2011 +0200 @@ -92,7 +92,7 @@ self.uiprops = vreg.config.uiprops self.datadir_url = vreg.config.datadir_url # raw html headers that can be added from any view - self.html_headers = HTMLHead() + self.html_headers = HTMLHead(self.datadir_url) # form parameters self.setup_params(form) # dictionnary that may be used to store request data that has to be @@ -256,7 +256,7 @@ """used by AutomaticWebTest to clear html headers between tests on the same resultset """ - self.html_headers = HTMLHead() + self.html_headers = HTMLHead(self.datadir_url) return self # web state helpers ####################################################### @@ -415,7 +415,8 @@ @cached # so it's writed only once def fckeditor_config(self): - self.add_js('fckeditor/fckeditor.js') + fckeditor_url = self.build_url('fckeditor/fckeditor.js') + self.add_js(fckeditor_url, localfile=False) self.html_headers.define_var('fcklang', self.lang) self.html_headers.define_var('fckconfigpath', self.data_url('cubicweb.fckcwconfig.js')) @@ -888,10 +889,20 @@ def _parse_accept_header(raw_header, value_parser=None, value_sort_key=None): """returns an ordered list accepted types - returned value is a list of 2-tuple (value, score), ordered - by score. Exact type of `value` will depend on what `value_parser` - will reutrn. if `value_parser` is None, then the raw value, as found - in the http header, is used. + :param value_parser: a function to parse a raw accept chunk. If None + is provided, the function defaults to identity. If a function is provided, + it must accept 2 parameters ``value`` and ``other_params``. ``value`` is + the value found before the first ';', `other_params` is a dictionary + built from all other chunks after this first ';' + + :param value_sort_key: a key function to sort values found in the accept + header. This function will be passed a 3-tuple + (raw_value, parsed_value, score). If None is provided, the default + sort_key is 1./score + + :return: a list of 3-tuple (raw_value, parsed_value, score), + ordered by score. ``parsed_value`` will be the return value of + ``value_parser(raw_value)`` """ if value_sort_key is None: value_sort_key = lambda infos: 1./infos[-1] @@ -926,7 +937,7 @@ 'text/html;level=1', `mimetypeinfo` will be ('text', '*', {'level': '1'}) """ try: - media_type, media_subtype = value.strip().split('/') + media_type, media_subtype = value.strip().split('/', 1) except ValueError: # safety belt : '/' should always be present media_type = value.strip() media_subtype = '*' diff -r 771f594c12a2 -r 1144a2d90314 web/webconfig.py --- a/web/webconfig.py Wed Apr 27 16:38:01 2011 +0200 +++ b/web/webconfig.py Fri Apr 29 09:08:30 2011 +0200 @@ -300,19 +300,14 @@ if not (self.repairing or self.creating): self.global_set_option('base-url', baseurl) httpsurl = self['https-url'] + datadir_path = 'data/' if self.debugmode else 'data/%s/' % self.instance_md5_version() if httpsurl: if httpsurl[-1] != '/': httpsurl += '/' if not self.repairing: self.global_set_option('https-url', httpsurl) - if self.debugmode: - self.https_datadir_url = httpsurl + 'data/' - else: - self.https_datadir_url = httpsurl + 'data%s/' % self.instance_md5_version() - if self.debugmode: - self.datadir_url = baseurl + 'data/' - else: - self.datadir_url = baseurl + 'data%s/' % self.instance_md5_version() + self.https_datadir_url = httpsurl + datadir_path + self.datadir_url = baseurl + datadir_path def _build_ui_properties(self): # self.datadir_url[:-1] to remove trailing / diff -r 771f594c12a2 -r 1144a2d90314 web/webctl.py --- a/web/webctl.py Wed Apr 27 16:38:01 2011 +0200 +++ b/web/webctl.py Fri Apr 29 09:08:30 2011 +0200 @@ -21,9 +21,22 @@ __docformat__ = "restructuredtext en" +import os, os.path as osp +from shutil import copy + from logilab.common.shellutils import ASK -from cubicweb.toolsutils import CommandHandler, underline_title +from cubicweb import ExecutionError +from cubicweb.cwctl import CWCTL +from cubicweb.cwconfig import CubicWebConfiguration as cwcfg +from cubicweb.toolsutils import Command, CommandHandler, underline_title + + +try: + from os import symlink as linkdir +except ImportError: + from shutil import copytree as linkdir + class WebCreateHandler(CommandHandler): cmdname = 'create' @@ -43,3 +56,57 @@ def postcreate(self, *args, **kwargs): """hooks called once instance's initialization has been completed""" + + +class GenStaticDataDir(Command): + """Create a directory merging all data directory content from cubes and CW. + """ + name = 'gen-static-datadir' + arguments = ' [dirpath]' + min_args = 1 + max_args = 2 + + options = () + + def run(self, args): + appid = args.pop(0) + config = cwcfg.config_for(appid) + if args: + dest = args[0] + else: + dest = osp.join(config.appdatahome, 'data') + if osp.exists(dest): + raise ExecutionError('Directory %s already exists. ' + 'Remove it first.' % dest) + config.quick_start = True # notify this is not a regular start + # list all resources (no matter their order) + resources = set() + for datadir in self._datadirs(config): + for dirpath, dirnames, filenames in os.walk(datadir): + rel_dirpath = dirpath[len(datadir)+1:] + resources.update(osp.join(rel_dirpath, f) for f in filenames) + # locate resources and copy them to destination + for resource in resources: + dirname = osp.dirname(resource) + dest_resource = osp.join(dest, dirname) + if not osp.isdir(dest_resource): + os.makedirs(dest_resource) + resource_dir, resource_path = config.locate_resource(resource) + copy(osp.join(resource_dir, resource_path), dest_resource) + # handle md5 version subdirectory + linkdir(dest, osp.join(dest, config.instance_md5_version())) + print ('You can use apache rewrite rule below :\n' + 'RewriteRule ^/data/(.*) %s/$1 [L]' % dest) + + def _datadirs(self, config): + repo = config.repository() + if config._cubes is None: + # web only config + config.init_cubes(repo.get_cubes()) + for cube in repo.get_cubes(): + cube_datadir = osp.join(cwcfg.cube_dir(cube), 'data') + if osp.isdir(cube_datadir): + yield cube_datadir + yield osp.join(config.shared_dir(), 'data') + +CWCTL.register(GenStaticDataDir)