65 netloc = host |
71 netloc = host |
66 baseurl = urlunsplit((scheme, netloc, url, query, fragment)) |
72 baseurl = urlunsplit((scheme, netloc, url, query, fragment)) |
67 return baseurl |
73 return baseurl |
68 |
74 |
69 |
75 |
70 class LongTimeExpiringFile(static.File): |
76 class ForbiddenDirectoryLister(resource.Resource): |
71 """overrides static.File and sets a far futre ``Expires`` date |
77 def render(self, request): |
|
78 return HTTPResponse(twisted_request=request, |
|
79 code=http.FORBIDDEN, |
|
80 stream='Access forbidden') |
|
81 |
|
82 class File(static.File): |
|
83 """Prevent from listing directories""" |
|
84 def directoryListing(self): |
|
85 return ForbiddenDirectoryLister() |
|
86 |
|
87 |
|
88 class LongTimeExpiringFile(File): |
|
89 """overrides static.File and sets a far future ``Expires`` date |
72 on the resouce. |
90 on the resouce. |
73 |
91 |
74 versions handling is done by serving static files by different |
92 versions handling is done by serving static files by different |
75 URLs for each version. For instance:: |
93 URLs for each version. For instance:: |
76 |
94 |
77 http://localhost:8080/data-2.48.2/cubicweb.css |
95 http://localhost:8080/data-2.48.2/cubicweb.css |
78 http://localhost:8080/data-2.49.0/cubicweb.css |
96 http://localhost:8080/data-2.49.0/cubicweb.css |
79 etc. |
97 etc. |
80 |
98 |
81 """ |
99 """ |
82 def renderHTTP(self, request): |
100 def render(self, request): |
83 def setExpireHeader(response): |
101 def setExpireHeader(response): |
84 response = iweb.IResponse(response) |
|
85 # Don't provide additional resource information to error responses |
102 # Don't provide additional resource information to error responses |
86 if response.code < 400: |
103 if response.code < 400: |
87 # the HTTP RFC recommands not going further than 1 year ahead |
104 # the HTTP RFC recommands not going further than 1 year ahead |
88 expires = date.today() + timedelta(days=6*30) |
105 expires = date.today() + timedelta(days=6*30) |
89 response.headers.setHeader('Expires', mktime(expires.timetuple())) |
106 response.headers.setHeader('Expires', mktime(expires.timetuple())) |
90 return response |
107 return response |
91 d = maybeDeferred(super(LongTimeExpiringFile, self).renderHTTP, request) |
108 d = maybeDeferred(super(LongTimeExpiringFile, self).render, request) |
92 return d.addCallback(setExpireHeader) |
109 return d.addCallback(setExpireHeader) |
93 |
110 |
94 |
111 |
95 class CubicWebRootResource(resource.PostableResource): |
112 class CubicWebRootResource(resource.Resource): |
96 addSlash = False |
|
97 |
|
98 def __init__(self, config, debug=None): |
113 def __init__(self, config, debug=None): |
99 self.debugmode = debug |
114 self.debugmode = debug |
100 self.config = config |
115 self.config = config |
101 # instantiate publisher here and not in init_publisher to get some |
116 # instantiate publisher here and not in init_publisher to get some |
102 # checks done before daemonization (eg versions consistency) |
117 # checks done before daemonization (eg versions consistency) |
103 self.appli = CubicWebPublisher(config, debug=self.debugmode) |
118 self.appli = CubicWebPublisher(config, debug=self.debugmode) |
104 self.base_url = config['base-url'] |
119 self.base_url = config['base-url'] |
105 self.https_url = config['https-url'] |
120 self.https_url = config['https-url'] |
106 self.versioned_datadir = 'data%s' % config.instance_md5_version() |
121 self.versioned_datadir = 'data%s' % config.instance_md5_version() |
|
122 self.children = {} |
107 |
123 |
108 def init_publisher(self): |
124 def init_publisher(self): |
109 config = self.config |
125 config = self.config |
110 # when we have an in-memory repository, clean unused sessions every XX |
126 # when we have an in-memory repository, clean unused sessions every XX |
111 # seconds and properly shutdown the server |
127 # seconds and properly shutdown the server |
143 try: |
159 try: |
144 self.pyro_daemon.handleRequests(self.pyro_listen_timeout) |
160 self.pyro_daemon.handleRequests(self.pyro_listen_timeout) |
145 except select.error: |
161 except select.error: |
146 return |
162 return |
147 |
163 |
148 def locateChild(self, request, segments): |
164 def getChild(self, path, request): |
149 """Indicate which resource to use to process down the URL's path""" |
165 """Indicate which resource to use to process down the URL's path""" |
150 if segments: |
166 pre_path = request.prePathURL() |
151 if segments[0] == 'https': |
167 # XXX testing pre_path[0] not enough? |
152 segments = segments[1:] |
168 if any(s in pre_path |
153 if len(segments) >= 2: |
169 for s in (self.versioned_datadir, 'data', 'static')): |
154 if segments[0] in (self.versioned_datadir, 'data', 'static'): |
170 # Anything in data/, static/ is treated as static files |
155 # Anything in data/, static/ is treated as static files |
171 |
156 if segments[0] == 'static': |
172 if 'static' in pre_path: |
157 # instance static directory |
173 # instance static directory |
158 datadir = self.config.static_directory |
174 datadir = self.config.static_directory |
159 elif segments[1] == 'fckeditor': |
175 elif 'fckeditor' in pre_path: |
160 fckeditordir = self.config.ext_resources['FCKEDITOR_PATH'] |
176 fckeditordir = self.config.ext_resources['FCKEDITOR_PATH'] |
161 return static.File(fckeditordir), segments[2:] |
177 return File(fckeditordir) |
162 else: |
178 else: |
163 # cube static data file |
179 # cube static data file |
164 datadir = self.config.locate_resource(segments[1]) |
180 datadir = self.config.locate_resource(path) |
165 if datadir is None: |
181 if datadir is None: |
166 return None, [] |
182 return self |
167 self.debug('static file %s from %s', segments[-1], datadir) |
183 self.info('static file %s from %s', path, datadir) |
168 if segments[0] == 'data': |
184 if 'data' in pre_path: |
169 return static.File(str(datadir)), segments[1:] |
185 return File(os.path.join(datadir, path)) |
170 else: |
186 else: |
171 return LongTimeExpiringFile(datadir), segments[1:] |
187 return LongTimeExpiringFile(datadir) |
172 elif segments[0] == 'fckeditor': |
188 elif path == 'fckeditor': |
173 fckeditordir = self.config.ext_resources['FCKEDITOR_PATH'] |
189 fckeditordir = self.config.ext_resources['FCKEDITOR_PATH'] |
174 return static.File(fckeditordir), segments[1:] |
190 return File(fckeditordir) |
175 # Otherwise we use this single resource |
191 # Otherwise we use this single resource |
176 return self, () |
192 return self |
177 |
193 |
178 def render(self, request): |
194 def render(self, request): |
179 """Render a page from the root resource""" |
195 """Render a page from the root resource""" |
180 # reload modified files in debug mode |
196 # reload modified files in debug mode |
181 if self.debugmode: |
197 if self.debugmode: |
182 self.appli.vreg.register_objects(self.config.vregistry_path()) |
198 self.appli.vreg.register_objects(self.config.vregistry_path()) |
183 if self.config['profile']: # default profiler don't trace threads |
199 if self.config['profile']: # default profiler don't trace threads |
184 return self.render_request(request) |
200 return self.render_request(request) |
185 else: |
201 else: |
186 return threads.deferToThread(self.render_request, request) |
202 deferred = threads.deferToThread(self.render_request, request) |
|
203 return NOT_DONE_YET |
187 |
204 |
188 def render_request(self, request): |
205 def render_request(self, request): |
189 origpath = request.path |
206 origpath = request.path |
190 host = request.host |
207 host = request.host |
191 # dual http/https access handling: expect a rewrite rule to prepend |
208 # dual http/https access handling: expect a rewrite rule to prepend |
229 try: |
246 try: |
230 result = self.appli.publish(path, req) |
247 result = self.appli.publish(path, req) |
231 except DirectResponse, ex: |
248 except DirectResponse, ex: |
232 return ex.response |
249 return ex.response |
233 except StatusResponse, ex: |
250 except StatusResponse, ex: |
234 return http.Response(stream=ex.content, code=ex.status, |
251 return HTTPResponse(stream=ex.content, code=ex.status, |
235 headers=req.headers_out or None) |
252 twisted_request=req._twreq, |
|
253 headers=req.headers_out) |
236 except RemoteCallFailed, ex: |
254 except RemoteCallFailed, ex: |
237 req.set_header('content-type', 'application/json') |
255 req.set_header('content-type', 'application/json') |
238 return http.Response(stream=ex.dumps(), |
256 return HTTPResponse(twisted_request=req._twreq, code=http.INTERNAL_SERVER_ERROR, |
239 code=responsecode.INTERNAL_SERVER_ERROR) |
257 stream=ex.dumps(), headers=req.headers_out) |
240 except NotFound: |
258 except NotFound: |
241 result = self.appli.notfound_content(req) |
259 result = self.appli.notfound_content(req) |
242 return http.Response(stream=result, code=responsecode.NOT_FOUND, |
260 return HTTPResponse(twisted_request=req._twreq, code=http.NOT_FOUND, |
243 headers=req.headers_out or None) |
261 stream=result, headers=req.headers_out) |
|
262 |
244 except ExplicitLogin: # must be before AuthenticationError |
263 except ExplicitLogin: # must be before AuthenticationError |
245 return self.request_auth(req) |
264 return self.request_auth(request=req) |
246 except AuthenticationError, ex: |
265 except AuthenticationError, ex: |
247 if self.config['auth-mode'] == 'cookie' and getattr(ex, 'url', None): |
266 if self.config['auth-mode'] == 'cookie' and getattr(ex, 'url', None): |
248 return self.redirect(req, ex.url) |
267 return self.redirect(request=req, location=ex.url) |
249 # in http we have to request auth to flush current http auth |
268 # in http we have to request auth to flush current http auth |
250 # information |
269 # information |
251 return self.request_auth(req, loggedout=True) |
270 return self.request_auth(request=req, loggedout=True) |
252 except Redirect, ex: |
271 except Redirect, ex: |
253 return self.redirect(req, ex.location) |
272 return self.redirect(request=req, location=ex.location) |
254 # request may be referenced by "onetime callback", so clear its entity |
273 # request may be referenced by "onetime callback", so clear its entity |
255 # cache to avoid memory usage |
274 # cache to avoid memory usage |
256 req.drop_entity_cache() |
275 req.drop_entity_cache() |
257 return http.Response(stream=result, code=responsecode.OK, |
276 |
258 headers=req.headers_out or None) |
277 return HTTPResponse(twisted_request=req._twreq, code=http.OK, |
259 |
278 stream=result, headers=req.headers_out) |
260 def redirect(self, req, location): |
279 |
261 req.headers_out.setHeader('location', str(location)) |
280 def redirect(self, request, location): |
262 self.debug('redirecting to %s', location) |
281 self.debug('redirecting to %s', str(location)) |
|
282 request.headers_out.setHeader('location', str(location)) |
263 # 303 See other |
283 # 303 See other |
264 return http.Response(code=303, headers=req.headers_out) |
284 return HTTPResponse(twisted_request=request._twreq, code=303, |
265 |
285 headers=request.headers_out) |
266 def request_auth(self, req, loggedout=False): |
286 |
267 if self.https_url and req.base_url() != self.https_url: |
287 def request_auth(self, request, loggedout=False): |
268 req.headers_out.setHeader('location', self.https_url + 'login') |
288 if self.https_url and request.base_url() != self.https_url: |
269 return http.Response(code=303, headers=req.headers_out) |
289 return self.redirect(request, self.https_url + 'login') |
270 if self.config['auth-mode'] == 'http': |
290 if self.config['auth-mode'] == 'http': |
271 code = responsecode.UNAUTHORIZED |
291 code = http.UNAUTHORIZED |
272 else: |
292 else: |
273 code = responsecode.FORBIDDEN |
293 code = http.FORBIDDEN |
274 if loggedout: |
294 if loggedout: |
275 if req.https: |
295 if request.https: |
276 req._base_url = self.base_url |
296 request._base_url = self.base_url |
277 req.https = False |
297 request.https = False |
278 content = self.appli.loggedout_content(req) |
298 content = self.appli.loggedout_content(request) |
279 else: |
299 else: |
280 content = self.appli.need_login_content(req) |
300 content = self.appli.need_login_content(request) |
281 return http.Response(code, req.headers_out, content) |
301 return HTTPResponse(twisted_request=request._twreq, |
282 |
302 stream=content, code=code, |
283 from twisted.internet import defer |
303 headers=request.headers_out) |
284 from twisted.web2 import fileupload |
304 |
285 |
305 #TODO |
286 # XXX set max file size to 100Mo: put max upload size in the configuration |
306 # # XXX max upload size in the configuration |
287 # line below for twisted >= 8.0, default param value for earlier version |
307 |
288 resource.PostableResource.maxSize = 100*1024*1024 |
308 @monkeypatch(http.Request) |
289 def parsePOSTData(request, maxMem=100*1024, maxFields=1024, |
309 def requestReceived(self, command, path, version): |
290 maxSize=100*1024*1024): |
310 """Called by channel when all data has been received. |
291 if request.stream.length == 0: |
311 |
292 return defer.succeed(None) |
312 This method is not intended for users. |
293 |
313 """ |
294 ctype = request.headers.getHeader('content-type') |
314 self.content.seek(0,0) |
295 |
315 self.args = {} |
296 if ctype is None: |
316 self.files = {} |
297 return defer.succeed(None) |
317 self.stack = [] |
298 |
318 self.method, self.uri = command, path |
299 def updateArgs(data): |
319 self.clientproto = version |
300 args = data |
320 x = self.uri.split('?', 1) |
301 request.args.update(args) |
321 if len(x) == 1: |
302 |
322 self.path = self.uri |
303 def updateArgsAndFiles(data): |
|
304 args, files = data |
|
305 request.args.update(args) |
|
306 request.files.update(files) |
|
307 |
|
308 def error(f): |
|
309 f.trap(fileupload.MimeFormatError) |
|
310 raise http.HTTPError(responsecode.BAD_REQUEST) |
|
311 |
|
312 if ctype.mediaType == 'application' and ctype.mediaSubtype == 'x-www-form-urlencoded': |
|
313 d = fileupload.parse_urlencoded(request.stream, keep_blank_values=True) |
|
314 d.addCallbacks(updateArgs, error) |
|
315 return d |
|
316 elif ctype.mediaType == 'multipart' and ctype.mediaSubtype == 'form-data': |
|
317 boundary = ctype.params.get('boundary') |
|
318 if boundary is None: |
|
319 return defer.fail(http.HTTPError( |
|
320 http.StatusResponse(responsecode.BAD_REQUEST, |
|
321 "Boundary not specified in Content-Type."))) |
|
322 d = fileupload.parseMultipartFormData(request.stream, boundary, |
|
323 maxMem, maxFields, maxSize) |
|
324 d.addCallbacks(updateArgsAndFiles, error) |
|
325 return d |
|
326 else: |
323 else: |
327 raise http.HTTPError(responsecode.BAD_REQUEST) |
324 self.path, argstring = x |
328 |
325 self.args = http.parse_qs(argstring, 1) |
329 server.parsePOSTData = parsePOSTData |
326 # cache the client and server information, we'll need this later to be |
|
327 # serialized and sent with the request so CGIs will work remotely |
|
328 self.client = self.channel.transport.getPeer() |
|
329 self.host = self.channel.transport.getHost() |
|
330 # Argument processing |
|
331 ctype = self.getHeader('content-type') |
|
332 if self.method == "POST" and ctype: |
|
333 key, pdict = parse_header(ctype) |
|
334 if key == 'application/x-www-form-urlencoded': |
|
335 self.args.update(http.parse_qs(self.content.read(), 1)) |
|
336 elif key == 'multipart/form-data': |
|
337 self.content.seek(0,0) |
|
338 form = FieldStorage(self.content, self.received_headers, |
|
339 environ={'REQUEST_METHOD': 'POST'}, |
|
340 keep_blank_values=1, |
|
341 strict_parsing=1) |
|
342 for key in form: |
|
343 value = form[key] |
|
344 if isinstance(value, list): |
|
345 self.args[key] = [v.value for v in value] |
|
346 elif value.filename: |
|
347 if value.done != -1: # -1 is transfer has been interrupted |
|
348 self.files[key] = (value.filename, value.file) |
|
349 else: |
|
350 self.files[key] = (None, None) |
|
351 else: |
|
352 self.args[key] = value.value |
|
353 self.process() |
330 |
354 |
331 |
355 |
332 from logging import getLogger |
356 from logging import getLogger |
333 from cubicweb import set_log_methods |
357 from cubicweb import set_log_methods |
334 set_log_methods(CubicWebRootResource, getLogger('cubicweb.twisted')) |
358 LOGGER = getLogger('cubicweb.twisted') |
335 |
359 set_log_methods(CubicWebRootResource, LOGGER) |
336 |
|
337 listiterator = type(iter([])) |
|
338 |
|
339 def _gc_debug(all=True): |
|
340 import gc |
|
341 from pprint import pprint |
|
342 from cubicweb.appobject import AppObject |
|
343 gc.collect() |
|
344 count = 0 |
|
345 acount = 0 |
|
346 fcount = 0 |
|
347 rcount = 0 |
|
348 ccount = 0 |
|
349 scount = 0 |
|
350 ocount = {} |
|
351 from rql.stmts import Union |
|
352 from cubicweb.schema import CubicWebSchema |
|
353 from cubicweb.rset import ResultSet |
|
354 from cubicweb.dbapi import Connection, Cursor |
|
355 from cubicweb.req import RequestSessionBase |
|
356 from cubicweb.server.repository import Repository |
|
357 from cubicweb.server.sources.native import NativeSQLSource |
|
358 from cubicweb.server.session import Session |
|
359 from cubicweb.devtools.testlib import CubicWebTC |
|
360 from logilab.common.testlib import TestSuite |
|
361 from optparse import Values |
|
362 import types, weakref |
|
363 for obj in gc.get_objects(): |
|
364 if isinstance(obj, RequestSessionBase): |
|
365 count += 1 |
|
366 if isinstance(obj, Session): |
|
367 print ' session', obj, referrers(obj, True) |
|
368 elif isinstance(obj, AppObject): |
|
369 acount += 1 |
|
370 elif isinstance(obj, ResultSet): |
|
371 rcount += 1 |
|
372 #print ' rset', obj, referrers(obj) |
|
373 elif isinstance(obj, Repository): |
|
374 print ' REPO', obj, referrers(obj, True) |
|
375 #elif isinstance(obj, NativeSQLSource): |
|
376 # print ' SOURCe', obj, referrers(obj) |
|
377 elif isinstance(obj, CubicWebTC): |
|
378 print ' TC', obj, referrers(obj) |
|
379 elif isinstance(obj, TestSuite): |
|
380 print ' SUITE', obj, referrers(obj) |
|
381 #elif isinstance(obj, Values): |
|
382 # print ' values', '%#x' % id(obj), referrers(obj, True) |
|
383 elif isinstance(obj, Connection): |
|
384 ccount += 1 |
|
385 #print ' cnx', obj, referrers(obj) |
|
386 #elif isinstance(obj, Cursor): |
|
387 # ccount += 1 |
|
388 # print ' cursor', obj, referrers(obj) |
|
389 elif isinstance(obj, file): |
|
390 fcount += 1 |
|
391 # print ' open file', file.name, file.fileno |
|
392 elif isinstance(obj, CubicWebSchema): |
|
393 scount += 1 |
|
394 print ' schema', obj, referrers(obj) |
|
395 elif not isinstance(obj, (type, tuple, dict, list, set, frozenset, |
|
396 weakref.ref, weakref.WeakKeyDictionary, |
|
397 listiterator, |
|
398 property, classmethod, |
|
399 types.ModuleType, types.MemberDescriptorType, |
|
400 types.FunctionType, types.MethodType)): |
|
401 try: |
|
402 ocount[obj.__class__] += 1 |
|
403 except KeyError: |
|
404 ocount[obj.__class__] = 1 |
|
405 except AttributeError: |
|
406 pass |
|
407 if count: |
|
408 print ' NB REQUESTS/SESSIONS', count |
|
409 if acount: |
|
410 print ' NB APPOBJECTS', acount |
|
411 if ccount: |
|
412 print ' NB CONNECTIONS', ccount |
|
413 if rcount: |
|
414 print ' NB RSETS', rcount |
|
415 if scount: |
|
416 print ' NB SCHEMAS', scount |
|
417 if fcount: |
|
418 print ' NB FILES', fcount |
|
419 if all: |
|
420 ocount = sorted(ocount.items(), key=lambda x: x[1], reverse=True)[:20] |
|
421 pprint(ocount) |
|
422 if gc.garbage: |
|
423 print 'UNREACHABLE', gc.garbage |
|
424 |
|
425 def referrers(obj, showobj=False): |
|
426 try: |
|
427 return sorted(set((type(x), showobj and x or getattr(x, '__name__', '%#x' % id(x))) |
|
428 for x in _referrers(obj))) |
|
429 except TypeError: |
|
430 s = set() |
|
431 unhashable = [] |
|
432 for x in _referrers(obj): |
|
433 try: |
|
434 s.add(x) |
|
435 except TypeError: |
|
436 unhashable.append(x) |
|
437 return sorted(s) + unhashable |
|
438 |
|
439 def _referrers(obj, seen=None, level=0): |
|
440 import gc, types |
|
441 from cubicweb.schema import CubicWebRelationSchema, CubicWebEntitySchema |
|
442 interesting = [] |
|
443 if seen is None: |
|
444 seen = set() |
|
445 for x in gc.get_referrers(obj): |
|
446 if id(x) in seen: |
|
447 continue |
|
448 seen.add(id(x)) |
|
449 if isinstance(x, types.FrameType): |
|
450 continue |
|
451 if isinstance(x, (CubicWebRelationSchema, CubicWebEntitySchema)): |
|
452 continue |
|
453 if isinstance(x, (list, tuple, set, dict, listiterator)): |
|
454 if level >= 5: |
|
455 pass |
|
456 #interesting.append(x) |
|
457 else: |
|
458 interesting += _referrers(x, seen, level+1) |
|
459 else: |
|
460 interesting.append(x) |
|
461 return interesting |
|
462 |
360 |
463 def run(config, debug): |
361 def run(config, debug): |
464 # create the site |
362 # create the site |
465 root_resource = CubicWebRootResource(config, debug) |
363 root_resource = CubicWebRootResource(config, debug) |
466 website = server.Site(root_resource) |
364 website = server.Site(root_resource) |
467 # serve it via standard HTTP on port set in the configuration |
365 # serve it via standard HTTP on port set in the configuration |
468 port = config['port'] or 8080 |
366 port = config['port'] or 8080 |
469 reactor.listenTCP(port, channel.HTTPFactory(website)) |
367 reactor.listenTCP(port, website) |
470 logger = getLogger('cubicweb.twisted') |
368 logger = getLogger('cubicweb.twisted') |
471 if not debug: |
369 if not debug: |
472 if sys.platform == 'win32': |
370 if sys.platform == 'win32': |
473 raise ConfigurationError("Under windows, you must use the service management " |
371 raise ConfigurationError("Under windows, you must use the service management " |
474 "commands (e.g : 'net start my_instance)'") |
372 "commands (e.g : 'net start my_instance)'") |