|
1 """twisted server for CubicWeb web applications |
|
2 |
|
3 :organization: Logilab |
|
4 :copyright: 2001-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
|
5 :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr |
|
6 """ |
|
7 __docformat__ = "restructuredtext en" |
|
8 |
|
9 import sys |
|
10 import select |
|
11 |
|
12 from mx.DateTime import today, RelativeDate |
|
13 |
|
14 from twisted.application import service, strports |
|
15 from twisted.internet import reactor, task, threads |
|
16 from twisted.internet.defer import maybeDeferred |
|
17 from twisted.web2 import channel, http, server, iweb |
|
18 from twisted.web2 import static, resource, responsecode |
|
19 |
|
20 from cubicweb import ObjectNotFound |
|
21 from cubicweb.web import (AuthenticationError, NotFound, Redirect, |
|
22 RemoteCallFailed, DirectResponse, StatusResponse, |
|
23 ExplicitLogin) |
|
24 from cubicweb.web.application import CubicWebPublisher |
|
25 |
|
26 from cubicweb.etwist.request import CubicWebTwistedRequestAdapter |
|
27 |
|
28 |
|
29 def start_task(interval, func): |
|
30 lc = task.LoopingCall(func) |
|
31 lc.start(interval) |
|
32 |
|
33 def start_looping_tasks(repo): |
|
34 for interval, func in repo._looping_tasks: |
|
35 repo.info('starting twisted task %s with interval %.2fs', |
|
36 func.__name__, interval) |
|
37 def catch_error_func(repo=repo, func=func): |
|
38 try: |
|
39 func() |
|
40 except: |
|
41 repo.exception('error in looping task') |
|
42 start_task(interval, catch_error_func) |
|
43 # ensure no tasks will be further added |
|
44 repo._looping_tasks = () |
|
45 |
|
46 |
|
47 class LongTimeExpiringFile(static.File): |
|
48 """overrides static.File and sets a far futre ``Expires`` date |
|
49 on the resouce. |
|
50 |
|
51 versions handling is done by serving static files by different |
|
52 URLs for each version. For instance:: |
|
53 |
|
54 http://localhost:8080/data-2.48.2/cubicweb.css |
|
55 http://localhost:8080/data-2.49.0/cubicweb.css |
|
56 etc. |
|
57 |
|
58 """ |
|
59 def renderHTTP(self, request): |
|
60 def setExpireHeader(response): |
|
61 response = iweb.IResponse(response) |
|
62 # Don't provide additional resource information to error responses |
|
63 if response.code < 400: |
|
64 # the HTTP RFC recommands not going further than 1 year ahead |
|
65 expires = today() + RelativeDate(months=6) |
|
66 response.headers.setHeader('Expires', int(expires.ticks())) |
|
67 return response |
|
68 d = maybeDeferred(super(LongTimeExpiringFile, self).renderHTTP, request) |
|
69 return d.addCallback(setExpireHeader) |
|
70 |
|
71 |
|
72 class CubicWebRootResource(resource.PostableResource): |
|
73 addSlash = False |
|
74 |
|
75 def __init__(self, config, debug=None): |
|
76 self.appli = CubicWebPublisher(config, debug=debug) |
|
77 self.debugmode = debug |
|
78 self.config = config |
|
79 self.base_url = config['base-url'] or config.default_base_url() |
|
80 self.versioned_datadir = 'data%s' % config.instance_md5_version() |
|
81 assert self.base_url[-1] == '/' |
|
82 self.https_url = config['https-url'] |
|
83 assert not self.https_url or self.https_url[-1] == '/' |
|
84 # when we have an in-memory repository, clean unused sessions every XX |
|
85 # seconds and properly shutdown the server |
|
86 if config.repo_method == 'inmemory': |
|
87 reactor.addSystemEventTrigger('before', 'shutdown', |
|
88 self.shutdown_event) |
|
89 # monkey path start_looping_task to get proper reactor integration |
|
90 self.appli.repo.__class__.start_looping_tasks = start_looping_tasks |
|
91 if config.pyro_enabled(): |
|
92 # if pyro is enabled, we have to register to the pyro name |
|
93 # server, create a pyro daemon, and create a task to handle pyro |
|
94 # requests |
|
95 self.pyro_daemon = self.appli.repo.pyro_register() |
|
96 self.pyro_listen_timeout = 0.02 |
|
97 start_task(1, self.pyro_loop_event) |
|
98 self.appli.repo.start_looping_tasks() |
|
99 try: |
|
100 self.url_rewriter = self.appli.vreg.select_component('urlrewriter') |
|
101 except ObjectNotFound: |
|
102 self.url_rewriter = None |
|
103 interval = min(config['cleanup-session-time'] or 120, |
|
104 config['cleanup-anonymous-session-time'] or 720) / 2. |
|
105 start_task(interval, self.appli.session_handler.clean_sessions) |
|
106 |
|
107 def shutdown_event(self): |
|
108 """callback fired when the server is shutting down to properly |
|
109 clean opened sessions |
|
110 """ |
|
111 self.appli.repo.shutdown() |
|
112 |
|
113 def pyro_loop_event(self): |
|
114 """listen for pyro events""" |
|
115 try: |
|
116 self.pyro_daemon.handleRequests(self.pyro_listen_timeout) |
|
117 except select.error: |
|
118 return |
|
119 |
|
120 def locateChild(self, request, segments): |
|
121 """Indicate which resource to use to process down the URL's path""" |
|
122 if segments: |
|
123 if segments[0] == 'https': |
|
124 segments = segments[1:] |
|
125 if len(segments) >= 2: |
|
126 if segments[0] in (self.versioned_datadir, 'data'): |
|
127 # Anything in data/ is treated as static files |
|
128 datadir = self.config.locate_resource(segments[1]) |
|
129 if datadir is None: |
|
130 return None, [] |
|
131 self.info('static file %s from %s', segments[-1], datadir) |
|
132 if segments[0] == 'data': |
|
133 return static.File(str(datadir)), segments[1:] |
|
134 else: |
|
135 return LongTimeExpiringFile(datadir), segments[1:] |
|
136 elif segments[0] == 'fckeditor': |
|
137 fckeditordir = self.config.ext_resources['FCKEDITOR_PATH'] |
|
138 return static.File(fckeditordir), segments[1:] |
|
139 # Otherwise we use this single resource |
|
140 return self, () |
|
141 |
|
142 def render(self, request): |
|
143 """Render a page from the root resource""" |
|
144 # reload modified files (only in development or debug mode) |
|
145 if self.config.mode == 'dev' or self.debugmode: |
|
146 self.appli.vreg.register_objects(self.config.vregistry_path()) |
|
147 if self.config['profile']: # default profiler don't trace threads |
|
148 return self.render_request(request) |
|
149 else: |
|
150 return threads.deferToThread(self.render_request, request) |
|
151 |
|
152 def render_request(self, request): |
|
153 origpath = request.path |
|
154 host = request.host |
|
155 # dual http/https access handling: expect a rewrite rule to prepend |
|
156 # 'https' to the path to detect https access |
|
157 if origpath.split('/', 2)[1] == 'https': |
|
158 origpath = origpath[6:] |
|
159 request.uri = request.uri[6:] |
|
160 https = True |
|
161 baseurl = self.https_url or self.base_url |
|
162 else: |
|
163 https = False |
|
164 baseurl = self.base_url |
|
165 req = CubicWebTwistedRequestAdapter(request, self.appli.vreg, https, baseurl) |
|
166 if req.authmode == 'http': |
|
167 # activate realm-based auth |
|
168 realm = self.config['realm'] |
|
169 req.set_header('WWW-Authenticate', [('Basic', {'realm' : realm })], raw=False) |
|
170 try: |
|
171 self.appli.connect(req) |
|
172 except AuthenticationError: |
|
173 return self.request_auth(req) |
|
174 except Redirect, ex: |
|
175 return self.redirect(req, ex.location) |
|
176 if https and req.cnx.anonymous_connection: |
|
177 # don't allow anonymous on https connection |
|
178 return self.request_auth(req) |
|
179 if self.url_rewriter is not None: |
|
180 # XXX should occurs before authentication? |
|
181 try: |
|
182 path = self.url_rewriter.rewrite(host, origpath) |
|
183 except Redirect, ex: |
|
184 return self.redirect(req, ex.location) |
|
185 request.uri.replace(origpath, path, 1) |
|
186 else: |
|
187 path = origpath |
|
188 if not path or path == "/": |
|
189 path = 'view' |
|
190 try: |
|
191 result = self.appli.publish(path, req) |
|
192 except DirectResponse, ex: |
|
193 return ex.response |
|
194 except StatusResponse, ex: |
|
195 return http.Response(stream=ex.content, code=ex.status, |
|
196 headers=req.headers_out or None) |
|
197 except RemoteCallFailed, ex: |
|
198 req.set_header('content-type', 'application/json') |
|
199 return http.Response(stream=ex.dumps(), |
|
200 code=responsecode.INTERNAL_SERVER_ERROR) |
|
201 except NotFound: |
|
202 result = self.appli.notfound_content(req) |
|
203 return http.Response(stream=result, code=responsecode.NOT_FOUND, |
|
204 headers=req.headers_out or None) |
|
205 except ExplicitLogin: # must be before AuthenticationError |
|
206 return self.request_auth(req) |
|
207 except AuthenticationError: |
|
208 if self.config['auth-mode'] == 'cookie': |
|
209 # in cookie mode redirecting to the index view is enough : |
|
210 # either anonymous connection is allowed and the page will |
|
211 # be displayed or we'll be redirected to the login form |
|
212 msg = req._('you have been logged out') |
|
213 if req.https: |
|
214 req._base_url = self.base_url |
|
215 req.https = False |
|
216 url = req.build_url('view', vid='index', __message=msg) |
|
217 return self.redirect(req, url) |
|
218 else: |
|
219 # in http we have to request auth to flush current http auth |
|
220 # information |
|
221 return self.request_auth(req, loggedout=True) |
|
222 except Redirect, ex: |
|
223 return self.redirect(req, ex.location) |
|
224 if not result: |
|
225 # no result, something went wrong... |
|
226 self.error('no data (%s)', req) |
|
227 # 500 Internal server error |
|
228 return self.redirect(req, req.build_url('error')) |
|
229 # request may be referenced by "onetime callback", so clear its entity |
|
230 # cache to avoid memory usage |
|
231 req.drop_entity_cache() |
|
232 return http.Response(stream=result, code=responsecode.OK, |
|
233 headers=req.headers_out or None) |
|
234 |
|
235 def redirect(self, req, location): |
|
236 req.headers_out.setHeader('location', str(location)) |
|
237 self.debug('redirecting to %s', location) |
|
238 # 303 See other |
|
239 return http.Response(code=303, headers=req.headers_out) |
|
240 |
|
241 def request_auth(self, req, loggedout=False): |
|
242 if self.https_url and req.base_url() != self.https_url: |
|
243 req.headers_out.setHeader('location', self.https_url + 'login') |
|
244 return http.Response(code=303, headers=req.headers_out) |
|
245 if self.config['auth-mode'] == 'http': |
|
246 code = responsecode.UNAUTHORIZED |
|
247 else: |
|
248 code = responsecode.FORBIDDEN |
|
249 if loggedout: |
|
250 if req.https: |
|
251 req._base_url = self.base_url |
|
252 req.https = False |
|
253 content = self.appli.loggedout_content(req) |
|
254 else: |
|
255 content = self.appli.need_login_content(req) |
|
256 return http.Response(code, req.headers_out, content) |
|
257 |
|
258 |
|
259 # This part gets run when you run this file via: "twistd -noy demo.py" |
|
260 def main(appid, cfgname): |
|
261 """Starts an cubicweb twisted server for an application |
|
262 |
|
263 appid: application's identifier |
|
264 cfgname: name of the configuration to use (twisted or all-in-one) |
|
265 """ |
|
266 from cubicweb.cwconfig import CubicWebConfiguration |
|
267 from cubicweb.etwist import twconfig # trigger configuration registration |
|
268 config = CubicWebConfiguration.config_for(appid, cfgname) |
|
269 # XXX why calling init_available_cubes here ? |
|
270 config.init_available_cubes() |
|
271 # create the site and application objects |
|
272 if '-n' in sys.argv: # debug mode |
|
273 cubicweb = CubicWebRootResource(config, debug=True) |
|
274 else: |
|
275 cubicweb = CubicWebRootResource(config) |
|
276 #toplevel = vhost.VHostURIRewrite(base_url, cubicweb) |
|
277 toplevel = cubicweb |
|
278 website = server.Site(toplevel) |
|
279 application = service.Application("cubicweb") |
|
280 # serve it via standard HTTP on port set in the configuration |
|
281 s = strports.service('tcp:%04d' % (config['port'] or 8080), |
|
282 channel.HTTPFactory(website)) |
|
283 s.setServiceParent(application) |
|
284 return application |
|
285 |
|
286 |
|
287 from twisted.python import failure |
|
288 from twisted.internet import defer |
|
289 from twisted.web2 import fileupload |
|
290 |
|
291 # XXX set max file size to 100Mo: put max upload size in the configuration |
|
292 # line below for twisted >= 8.0, default param value for earlier version |
|
293 resource.PostableResource.maxSize = 100*1024*1024 |
|
294 def parsePOSTData(request, maxMem=100*1024, maxFields=1024, |
|
295 maxSize=100*1024*1024): |
|
296 if request.stream.length == 0: |
|
297 return defer.succeed(None) |
|
298 |
|
299 ctype = request.headers.getHeader('content-type') |
|
300 |
|
301 if ctype is None: |
|
302 return defer.succeed(None) |
|
303 |
|
304 def updateArgs(data): |
|
305 args = data |
|
306 request.args.update(args) |
|
307 |
|
308 def updateArgsAndFiles(data): |
|
309 args, files = data |
|
310 request.args.update(args) |
|
311 request.files.update(files) |
|
312 |
|
313 def error(f): |
|
314 f.trap(fileupload.MimeFormatError) |
|
315 raise http.HTTPError(responsecode.BAD_REQUEST) |
|
316 |
|
317 if ctype.mediaType == 'application' and ctype.mediaSubtype == 'x-www-form-urlencoded': |
|
318 d = fileupload.parse_urlencoded(request.stream, keep_blank_values=True) |
|
319 d.addCallbacks(updateArgs, error) |
|
320 return d |
|
321 elif ctype.mediaType == 'multipart' and ctype.mediaSubtype == 'form-data': |
|
322 boundary = ctype.params.get('boundary') |
|
323 if boundary is None: |
|
324 return defer.fail(http.HTTPError( |
|
325 http.StatusResponse(responsecode.BAD_REQUEST, |
|
326 "Boundary not specified in Content-Type."))) |
|
327 d = fileupload.parseMultipartFormData(request.stream, boundary, |
|
328 maxMem, maxFields, maxSize) |
|
329 d.addCallbacks(updateArgsAndFiles, error) |
|
330 return d |
|
331 else: |
|
332 raise http.HTTPError(responsecode.BAD_REQUEST) |
|
333 |
|
334 server.parsePOSTData = parsePOSTData |
|
335 |
|
336 |
|
337 from logging import getLogger |
|
338 from cubicweb import set_log_methods |
|
339 set_log_methods(CubicWebRootResource, getLogger('cubicweb.twisted')) |
|
340 |
|
341 |
|
342 |
|
343 def _gc_debug(): |
|
344 import gc |
|
345 from pprint import pprint |
|
346 from cubicweb.vregistry import VObject |
|
347 gc.collect() |
|
348 count = 0 |
|
349 acount = 0 |
|
350 ocount = {} |
|
351 for obj in gc.get_objects(): |
|
352 if isinstance(obj, CubicWebTwistedRequestAdapter): |
|
353 count += 1 |
|
354 elif isinstance(obj, VObject): |
|
355 acount += 1 |
|
356 else: |
|
357 try: |
|
358 ocount[obj.__class__]+= 1 |
|
359 except KeyError: |
|
360 ocount[obj.__class__] = 1 |
|
361 except AttributeError: |
|
362 pass |
|
363 print 'IN MEM REQUESTS', count |
|
364 print 'IN MEM APPOBJECTS', acount |
|
365 ocount = sorted(ocount.items(), key=lambda x: x[1], reverse=True)[:20] |
|
366 pprint(ocount) |
|
367 print 'UNREACHABLE', gc.garbage |