"""twisted server for CubicWeb web applications:organization: Logilab:copyright: 2001-2009 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2.:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses"""__docformat__="restructuredtext en"importsysimportselectfromtimeimportmktimefromdatetimeimportdate,timedeltafromurlparseimporturlsplit,urlunsplitfromtwisted.applicationimportservice,strportsfromtwisted.internetimportreactor,task,threadsfromtwisted.internet.deferimportmaybeDeferredfromtwisted.web2importchannel,http,server,iwebfromtwisted.web2importstatic,resource,responsecodefromcubicwebimportObjectNotFoundfromcubicweb.webimport(AuthenticationError,NotFound,Redirect,RemoteCallFailed,DirectResponse,StatusResponse,ExplicitLogin)fromcubicweb.web.applicationimportCubicWebPublisherfromcubicweb.etwist.requestimportCubicWebTwistedRequestAdapterdefstart_task(interval,func):lc=task.LoopingCall(func)lc.start(interval)defstart_looping_tasks(repo):forinterval,funcinrepo._looping_tasks:repo.info('starting twisted task %s with interval %.2fs',func.__name__,interval)defcatch_error_func(repo=repo,func=func):try:func()except:repo.exception('error in looping task')start_task(interval,catch_error_func)# ensure no tasks will be further addedrepo._looping_tasks=()defhost_prefixed_baseurl(baseurl,host):scheme,netloc,url,query,fragment=urlsplit(baseurl)netloc_domain='.'+'.'.join(netloc.split('.')[-2:])ifhost.endswith(netloc_domain):netloc=hostbaseurl=urlunsplit((scheme,netloc,url,query,fragment))returnbaseurlclassLongTimeExpiringFile(static.File):"""overrides static.File and sets a far futre ``Expires`` date on the resouce. versions handling is done by serving static files by different URLs for each version. For instance:: http://localhost:8080/data-2.48.2/cubicweb.css http://localhost:8080/data-2.49.0/cubicweb.css etc. """defrenderHTTP(self,request):defsetExpireHeader(response):response=iweb.IResponse(response)# Don't provide additional resource information to error responsesifresponse.code<400:# the HTTP RFC recommands not going further than 1 year aheadexpires=date.today()+timedelta(days=6*30)response.headers.setHeader('Expires',mktime(expires.timetuple()))returnresponsed=maybeDeferred(super(LongTimeExpiringFile,self).renderHTTP,request)returnd.addCallback(setExpireHeader)classCubicWebRootResource(resource.PostableResource):addSlash=Falsedef__init__(self,config,debug=None):self.appli=CubicWebPublisher(config,debug=debug)self.debugmode=debugself.config=configself.base_url=config['base-url']orconfig.default_base_url()self.versioned_datadir='data%s'%config.instance_md5_version()assertself.base_url[-1]=='/'self.https_url=config['https-url']assertnotself.https_urlorself.https_url[-1]=='/'# when we have an in-memory repository, clean unused sessions every XX# seconds and properly shutdown the serverifconfig.repo_method=='inmemory':reactor.addSystemEventTrigger('before','shutdown',self.shutdown_event)# monkey patch start_looping_task to get proper reactor integrationself.appli.repo.__class__.start_looping_tasks=start_looping_tasksifconfig.pyro_enabled():# if pyro is enabled, we have to register to the pyro name# server, create a pyro daemon, and create a task to handle pyro# requestsself.pyro_daemon=self.appli.repo.pyro_register()self.pyro_listen_timeout=0.02start_task(1,self.pyro_loop_event)self.appli.repo.start_looping_tasks()try:self.url_rewriter=self.appli.vreg.select_component('urlrewriter')exceptObjectNotFound:self.url_rewriter=Noneinterval=min(config['cleanup-session-time']or120,config['cleanup-anonymous-session-time']or720)/2.start_task(interval,self.appli.session_handler.clean_sessions)defshutdown_event(self):"""callback fired when the server is shutting down to properly clean opened sessions """self.appli.repo.shutdown()defpyro_loop_event(self):"""listen for pyro events"""try:self.pyro_daemon.handleRequests(self.pyro_listen_timeout)exceptselect.error:returndeflocateChild(self,request,segments):"""Indicate which resource to use to process down the URL's path"""ifsegments:ifsegments[0]=='https':segments=segments[1:]iflen(segments)>=2:ifsegments[0]in(self.versioned_datadir,'data','static'):# Anything in data/, static/ is treated as static filesifsegments[0]=='static':# instance static directorydatadir=self.config.static_directoryelse:# cube static data filedatadir=self.config.locate_resource(segments[1])ifdatadirisNone:returnNone,[]self.info('static file %s from %s',segments[-1],datadir)ifsegments[0]=='data':returnstatic.File(str(datadir)),segments[1:]else:returnLongTimeExpiringFile(datadir),segments[1:]elifsegments[0]=='fckeditor':fckeditordir=self.config.ext_resources['FCKEDITOR_PATH']returnstatic.File(fckeditordir),segments[1:]# Otherwise we use this single resourcereturnself,()defrender(self,request):"""Render a page from the root resource"""# reload modified files (only in development or debug mode)ifself.config.mode=='dev'orself.debugmode:self.appli.vreg.register_objects(self.config.vregistry_path())ifself.config['profile']:# default profiler don't trace threadsreturnself.render_request(request)else:returnthreads.deferToThread(self.render_request,request)defrender_request(self,request):origpath=request.pathhost=request.host# dual http/https access handling: expect a rewrite rule to prepend# 'https' to the path to detect https accessiforigpath.split('/',2)[1]=='https':origpath=origpath[6:]request.uri=request.uri[6:]https=Truebaseurl=self.https_urlorself.base_urlelse:https=Falsebaseurl=self.base_urlifself.config['use-request-subdomain']:baseurl=host_prefixed_baseurl(baseurl,host)self.warning('used baseurl is %s for this request',baseurl)req=CubicWebTwistedRequestAdapter(request,self.appli.vreg,https,baseurl)ifreq.authmode=='http':# activate realm-based authrealm=self.config['realm']req.set_header('WWW-Authenticate',[('Basic',{'realm':realm})],raw=False)try:self.appli.connect(req)exceptAuthenticationError:returnself.request_auth(req)exceptRedirect,ex:returnself.redirect(req,ex.location)ifhttpsandreq.cnx.anonymous_connection:# don't allow anonymous on https connectionreturnself.request_auth(req)ifself.url_rewriterisnotNone:# XXX should occur before authentication?try:path=self.url_rewriter.rewrite(host,origpath,req)exceptRedirect,ex:returnself.redirect(req,ex.location)request.uri.replace(origpath,path,1)else:path=origpathifnotpathorpath=="/":path='view'try:result=self.appli.publish(path,req)exceptDirectResponse,ex:returnex.responseexceptStatusResponse,ex:returnhttp.Response(stream=ex.content,code=ex.status,headers=req.headers_outorNone)exceptRemoteCallFailed,ex:req.set_header('content-type','application/json')returnhttp.Response(stream=ex.dumps(),code=responsecode.INTERNAL_SERVER_ERROR)exceptNotFound:result=self.appli.notfound_content(req)returnhttp.Response(stream=result,code=responsecode.NOT_FOUND,headers=req.headers_outorNone)exceptExplicitLogin:# must be before AuthenticationErrorreturnself.request_auth(req)exceptAuthenticationError:ifself.config['auth-mode']=='cookie':# in cookie mode redirecting to the index view is enough :# either anonymous connection is allowed and the page will# be displayed or we'll be redirected to the login formmsg=req._('you have been logged out')ifreq.https:req._base_url=self.base_urlreq.https=Falseurl=req.build_url('view',vid='index',__message=msg)returnself.redirect(req,url)else:# in http we have to request auth to flush current http auth# informationreturnself.request_auth(req,loggedout=True)exceptRedirect,ex:returnself.redirect(req,ex.location)# request may be referenced by "onetime callback", so clear its entity# cache to avoid memory usagereq.drop_entity_cache()returnhttp.Response(stream=result,code=responsecode.OK,headers=req.headers_outorNone)defredirect(self,req,location):req.headers_out.setHeader('location',str(location))self.debug('redirecting to %s',location)# 303 See otherreturnhttp.Response(code=303,headers=req.headers_out)defrequest_auth(self,req,loggedout=False):ifself.https_urlandreq.base_url()!=self.https_url:req.headers_out.setHeader('location',self.https_url+'login')returnhttp.Response(code=303,headers=req.headers_out)ifself.config['auth-mode']=='http':code=responsecode.UNAUTHORIZEDelse:code=responsecode.FORBIDDENifloggedout:ifreq.https:req._base_url=self.base_urlreq.https=Falsecontent=self.appli.loggedout_content(req)else:content=self.appli.need_login_content(req)returnhttp.Response(code,req.headers_out,content)# This part gets run when you run this file via: "twistd -noy demo.py"defmain(appid,cfgname):"""Starts an cubicweb twisted server for an application appid: application's identifier cfgname: name of the configuration to use (twisted or all-in-one) """fromcubicweb.cwconfigimportCubicWebConfigurationfromcubicweb.etwistimporttwconfig# trigger configuration registrationconfig=CubicWebConfiguration.config_for(appid,cfgname)# XXX why calling init_available_cubes here ?config.init_available_cubes()# create the site and application objectsif'-n'insys.argv:# debug modecubicweb=CubicWebRootResource(config,debug=True)else:cubicweb=CubicWebRootResource(config)#toplevel = vhost.VHostURIRewrite(base_url, cubicweb)toplevel=cubicwebwebsite=server.Site(toplevel)application=service.Application("cubicweb")# serve it via standard HTTP on port set in the configurations=strports.service('tcp:%04d'%(config['port']or8080),channel.HTTPFactory(website))s.setServiceParent(application)returnapplicationfromtwisted.pythonimportfailurefromtwisted.internetimportdeferfromtwisted.web2importfileupload# XXX set max file size to 100Mo: put max upload size in the configuration# line below for twisted >= 8.0, default param value for earlier versionresource.PostableResource.maxSize=100*1024*1024defparsePOSTData(request,maxMem=100*1024,maxFields=1024,maxSize=100*1024*1024):ifrequest.stream.length==0:returndefer.succeed(None)ctype=request.headers.getHeader('content-type')ifctypeisNone:returndefer.succeed(None)defupdateArgs(data):args=datarequest.args.update(args)defupdateArgsAndFiles(data):args,files=datarequest.args.update(args)request.files.update(files)deferror(f):f.trap(fileupload.MimeFormatError)raisehttp.HTTPError(responsecode.BAD_REQUEST)ifctype.mediaType=='application'andctype.mediaSubtype=='x-www-form-urlencoded':d=fileupload.parse_urlencoded(request.stream,keep_blank_values=True)d.addCallbacks(updateArgs,error)returndelifctype.mediaType=='multipart'andctype.mediaSubtype=='form-data':boundary=ctype.params.get('boundary')ifboundaryisNone:returndefer.fail(http.HTTPError(http.StatusResponse(responsecode.BAD_REQUEST,"Boundary not specified in Content-Type.")))d=fileupload.parseMultipartFormData(request.stream,boundary,maxMem,maxFields,maxSize)d.addCallbacks(updateArgsAndFiles,error)returndelse:raisehttp.HTTPError(responsecode.BAD_REQUEST)server.parsePOSTData=parsePOSTDatafromloggingimportgetLoggerfromcubicwebimportset_log_methodsset_log_methods(CubicWebRootResource,getLogger('cubicweb.twisted'))def_gc_debug():importgcfrompprintimportpprintfromcubicweb.vregistryimportVObjectgc.collect()count=0acount=0ocount={}forobjingc.get_objects():ifisinstance(obj,CubicWebTwistedRequestAdapter):count+=1elifisinstance(obj,VObject):acount+=1else:try:ocount[obj.__class__]+=1exceptKeyError:ocount[obj.__class__]=1exceptAttributeError:passprint'IN MEM REQUESTS',countprint'IN MEM APPOBJECTS',acountocount=sorted(ocount.items(),key=lambdax:x[1],reverse=True)[:20]pprint(ocount)print'UNREACHABLE',gc.garbage