[shell] make process_script available throuhg c-c shell / migration script context
"""twisted server for CubicWeb web instances:organization: Logilab:copyright: 2001-2009 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2.:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses"""__docformat__="restructuredtext en"importsysimportosimportselectimporthotshotfromtimeimportmktimefromdatetimeimportdate,timedeltafromurlparseimporturlsplit,urlunsplitfromtwisted.applicationimportstrportsfromtwisted.internetimportreactor,task,threadsfromtwisted.internet.deferimportmaybeDeferredfromtwisted.web2importchannel,http,server,iwebfromtwisted.web2importstatic,resource,responsecodefromcubicwebimportObjectNotFound,CW_EVENT_MANAGERfromcubicweb.webimport(AuthenticationError,NotFound,Redirect,RemoteCallFailed,DirectResponse,StatusResponse,ExplicitLogin)fromcubicweb.web.applicationimportCubicWebPublisherfromcubicweb.etwist.requestimportCubicWebTwistedRequestAdapterdefdaemonize():# XXX unix specific# XXX factorize w/ code in cw.server.server and cw.server.serverctl# (start-repository command)# See http://www.erlenstar.demon.co.uk/unix/faq_toc.html#TOC16ifos.fork():# launch child and...return1os.setsid()ifos.fork():# launch child again.return1# move to the root to avoit mount pbos.chdir('/')# set paranoid umaskos.umask(077)null=os.open('/dev/null',os.O_RDWR)foriinrange(3):try:os.dup2(null,i)exceptOSError,e:ife.errno!=errno.EBADF:raiseos.close(null)returnNonedefstart_task(interval,func):lc=task.LoopingCall(func)# wait until interval has expired to actually start the task, else we have# to wait all task to be finished for the server to be actually startedlc.start(interval,now=False)defhost_prefixed_baseurl(baseurl,host):scheme,netloc,url,query,fragment=urlsplit(baseurl)netloc_domain='.'+'.'.join(netloc.split('.')[-2:])ifhost.endswith(netloc_domain):netloc=hostbaseurl=urlunsplit((scheme,netloc,url,query,fragment))returnbaseurlclassLongTimeExpiringFile(static.File):"""overrides static.File and sets a far futre ``Expires`` date on the resouce. versions handling is done by serving static files by different URLs for each version. For instance:: http://localhost:8080/data-2.48.2/cubicweb.css http://localhost:8080/data-2.49.0/cubicweb.css etc. """defrenderHTTP(self,request):defsetExpireHeader(response):response=iweb.IResponse(response)# Don't provide additional resource information to error responsesifresponse.code<400:# the HTTP RFC recommands not going further than 1 year aheadexpires=date.today()+timedelta(days=6*30)response.headers.setHeader('Expires',mktime(expires.timetuple()))returnresponsed=maybeDeferred(super(LongTimeExpiringFile,self).renderHTTP,request)returnd.addCallback(setExpireHeader)classCubicWebRootResource(resource.PostableResource):addSlash=Falsedef__init__(self,config,debug=None):self.debugmode=debugself.config=configself.base_url=config['base-url']orconfig.default_base_url()assertself.base_url[-1]=='/'self.https_url=config['https-url']assertnotself.https_urlorself.https_url[-1]=='/'definit_publisher(self):config=self.configself.appli=CubicWebPublisher(config,debug=self.debugmode)self.versioned_datadir='data%s'%config.instance_md5_version()# when we have an in-memory repository, clean unused sessions every XX# seconds and properly shutdown the serverifconfig.repo_method=='inmemory':reactor.addSystemEventTrigger('before','shutdown',self.shutdown_event)# monkey patch start_looping_task to get proper reactor integration#self.appli.repo.__class__.start_looping_tasks = start_looping_tasksifconfig.pyro_enabled():# if pyro is enabled, we have to register to the pyro name# server, create a pyro daemon, and create a task to handle pyro# requestsself.pyro_daemon=self.appli.repo.pyro_register()self.pyro_listen_timeout=0.02self.appli.repo.looping_task(1,self.pyro_loop_event)self.appli.repo.start_looping_tasks()self.set_url_rewriter()CW_EVENT_MANAGER.bind('after-registry-reload',self.set_url_rewriter)defstart_service(self):config=self.configinterval=min(config['cleanup-session-time']or120,config['cleanup-anonymous-session-time']or720)/2.start_task(interval,self.appli.session_handler.clean_sessions)defset_url_rewriter(self):self.url_rewriter=self.appli.vreg['components'].select_object('urlrewriter')defshutdown_event(self):"""callback fired when the server is shutting down to properly clean opened sessions """self.appli.repo.shutdown()defpyro_loop_event(self):"""listen for pyro events"""try:self.pyro_daemon.handleRequests(self.pyro_listen_timeout)exceptselect.error:returndeflocateChild(self,request,segments):"""Indicate which resource to use to process down the URL's path"""ifsegments:ifsegments[0]=='https':segments=segments[1:]iflen(segments)>=2:ifsegments[0]in(self.versioned_datadir,'data','static'):# Anything in data/, static/ is treated as static filesifsegments[0]=='static':# instance static directorydatadir=self.config.static_directoryelifsegments[1]=='fckeditor':fckeditordir=self.config.ext_resources['FCKEDITOR_PATH']returnstatic.File(fckeditordir),segments[2:]else:# cube static data filedatadir=self.config.locate_resource(segments[1])ifdatadirisNone:returnNone,[]self.info('static file %s from %s',segments[-1],datadir)ifsegments[0]=='data':returnstatic.File(str(datadir)),segments[1:]else:returnLongTimeExpiringFile(datadir),segments[1:]elifsegments[0]=='fckeditor':fckeditordir=self.config.ext_resources['FCKEDITOR_PATH']returnstatic.File(fckeditordir),segments[1:]# Otherwise we use this single resourcereturnself,()defrender(self,request):"""Render a page from the root resource"""# reload modified files in debug modeifself.debugmode:self.appli.vreg.register_objects(self.config.vregistry_path())ifself.config['profile']:# default profiler don't trace threadsreturnself.render_request(request)else:returnthreads.deferToThread(self.render_request,request)defrender_request(self,request):origpath=request.pathhost=request.host# dual http/https access handling: expect a rewrite rule to prepend# 'https' to the path to detect https accessiforigpath.split('/',2)[1]=='https':origpath=origpath[6:]request.uri=request.uri[6:]https=Truebaseurl=self.https_urlorself.base_urlelse:https=Falsebaseurl=self.base_urlifself.config['use-request-subdomain']:baseurl=host_prefixed_baseurl(baseurl,host)self.warning('used baseurl is %s for this request',baseurl)req=CubicWebTwistedRequestAdapter(request,self.appli.vreg,https,baseurl)ifreq.authmode=='http':# activate realm-based authrealm=self.config['realm']req.set_header('WWW-Authenticate',[('Basic',{'realm':realm})],raw=False)try:self.appli.connect(req)exceptAuthenticationError:returnself.request_auth(req)exceptRedirect,ex:returnself.redirect(req,ex.location)ifhttpsandreq.cnx.anonymous_connection:# don't allow anonymous on https connectionreturnself.request_auth(req)ifself.url_rewriterisnotNone:# XXX should occur before authentication?try:path=self.url_rewriter.rewrite(host,origpath,req)exceptRedirect,ex:returnself.redirect(req,ex.location)request.uri.replace(origpath,path,1)else:path=origpathifnotpathorpath=="/":path='view'try:result=self.appli.publish(path,req)exceptDirectResponse,ex:returnex.responseexceptStatusResponse,ex:returnhttp.Response(stream=ex.content,code=ex.status,headers=req.headers_outorNone)exceptRemoteCallFailed,ex:req.set_header('content-type','application/json')returnhttp.Response(stream=ex.dumps(),code=responsecode.INTERNAL_SERVER_ERROR)exceptNotFound:result=self.appli.notfound_content(req)returnhttp.Response(stream=result,code=responsecode.NOT_FOUND,headers=req.headers_outorNone)exceptExplicitLogin:# must be before AuthenticationErrorreturnself.request_auth(req)exceptAuthenticationError:ifself.config['auth-mode']=='cookie':# in cookie mode redirecting to the index view is enough :# either anonymous connection is allowed and the page will# be displayed or we'll be redirected to the login formmsg=req._('you have been logged out')ifreq.https:req._base_url=self.base_urlreq.https=Falseurl=req.build_url('view',vid='index',__message=msg)returnself.redirect(req,url)else:# in http we have to request auth to flush current http auth# informationreturnself.request_auth(req,loggedout=True)exceptRedirect,ex:returnself.redirect(req,ex.location)# request may be referenced by "onetime callback", so clear its entity# cache to avoid memory usagereq.drop_entity_cache()returnhttp.Response(stream=result,code=responsecode.OK,headers=req.headers_outorNone)defredirect(self,req,location):req.headers_out.setHeader('location',str(location))self.debug('redirecting to %s',location)# 303 See otherreturnhttp.Response(code=303,headers=req.headers_out)defrequest_auth(self,req,loggedout=False):ifself.https_urlandreq.base_url()!=self.https_url:req.headers_out.setHeader('location',self.https_url+'login')returnhttp.Response(code=303,headers=req.headers_out)ifself.config['auth-mode']=='http':code=responsecode.UNAUTHORIZEDelse:code=responsecode.FORBIDDENifloggedout:ifreq.https:req._base_url=self.base_urlreq.https=Falsecontent=self.appli.loggedout_content(req)else:content=self.appli.need_login_content(req)returnhttp.Response(code,req.headers_out,content)fromtwisted.pythonimportfailurefromtwisted.internetimportdeferfromtwisted.web2importfileupload# XXX set max file size to 100Mo: put max upload size in the configuration# line below for twisted >= 8.0, default param value for earlier versionresource.PostableResource.maxSize=100*1024*1024defparsePOSTData(request,maxMem=100*1024,maxFields=1024,maxSize=100*1024*1024):ifrequest.stream.length==0:returndefer.succeed(None)ctype=request.headers.getHeader('content-type')ifctypeisNone:returndefer.succeed(None)defupdateArgs(data):args=datarequest.args.update(args)defupdateArgsAndFiles(data):args,files=datarequest.args.update(args)request.files.update(files)deferror(f):f.trap(fileupload.MimeFormatError)raisehttp.HTTPError(responsecode.BAD_REQUEST)ifctype.mediaType=='application'andctype.mediaSubtype=='x-www-form-urlencoded':d=fileupload.parse_urlencoded(request.stream,keep_blank_values=True)d.addCallbacks(updateArgs,error)returndelifctype.mediaType=='multipart'andctype.mediaSubtype=='form-data':boundary=ctype.params.get('boundary')ifboundaryisNone:returndefer.fail(http.HTTPError(http.StatusResponse(responsecode.BAD_REQUEST,"Boundary not specified in Content-Type.")))d=fileupload.parseMultipartFormData(request.stream,boundary,maxMem,maxFields,maxSize)d.addCallbacks(updateArgsAndFiles,error)returndelse:raisehttp.HTTPError(responsecode.BAD_REQUEST)server.parsePOSTData=parsePOSTDatafromloggingimportgetLoggerfromcubicwebimportset_log_methodsset_log_methods(CubicWebRootResource,getLogger('cubicweb.twisted'))def_gc_debug():importgcfrompprintimportpprintfromcubicweb.appobjectimportAppObjectgc.collect()count=0acount=0ocount={}forobjingc.get_objects():ifisinstance(obj,CubicWebTwistedRequestAdapter):count+=1elifisinstance(obj,AppObject):acount+=1else:try:ocount[obj.__class__]+=1exceptKeyError:ocount[obj.__class__]=1exceptAttributeError:passprint'IN MEM REQUESTS',countprint'IN MEM APPOBJECTS',acountocount=sorted(ocount.items(),key=lambdax:x[1],reverse=True)[:20]pprint(ocount)print'UNREACHABLE',gc.garbagedefrun(config,debug):# create the siteroot_resource=CubicWebRootResource(config,debug)website=server.Site(root_resource)# serve it via standard HTTP on port set in the configurationport=config['port']or8080reactor.listenTCP(port,channel.HTTPFactory(website))logger=getLogger('cubicweb.twisted')logger.info('instance started on %s',root_resource.base_url)ifnotdebug:print'instance starting in the background'ifdaemonize():return# child processifconfig['pid-file']:# ensure the directory where the pid-file should be set exists (for# instance /var/run/cubicweb may be deleted on computer restart) piddir=os.path.dirname(config['pid-file'])ifnotos.path.exists(piddir):os.makedirs(piddir)file(config['pid-file'],'w').write(str(os.getpid()))root_resource.init_publisher()# before changing uidifconfig['uid']isnotNone:try:uid=int(config['uid'])exceptValueError:frompwdimportgetpwnamuid=getpwnam(config['uid']).pw_uidos.setuid(uid)root_resource.start_service()ifconfig['profile']:prof=hotshot.Profile(config['profile'])prof.runcall(reactor.run)else:reactor.run()