[selectors] new is_in_state select to avoid common error when writing state based selector
"""twisted server for CubicWeb web instances:organization: Logilab:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2.:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses"""__docformat__="restructuredtext en"importsysimportosimportselectimporterrnofromtimeimportmktimefromdatetimeimportdate,timedeltafromurlparseimporturlsplit,urlunsplitfromtwisted.internetimportreactor,task,threadsfromtwisted.internet.deferimportmaybeDeferredfromtwisted.web2importchannel,http,server,iwebfromtwisted.web2importstatic,resource,responsecodefromcubicwebimportConfigurationError,CW_EVENT_MANAGERfromcubicweb.webimport(AuthenticationError,NotFound,Redirect,RemoteCallFailed,DirectResponse,StatusResponse,ExplicitLogin)fromcubicweb.web.applicationimportCubicWebPublisherfromcubicweb.etwist.requestimportCubicWebTwistedRequestAdapterdefdaemonize():# XXX unix specific# XXX factorize w/ code in cw.server.server and cw.server.serverctl# (start-repository command)# See http://www.erlenstar.demon.co.uk/unix/faq_toc.html#TOC16ifos.fork():# launch child and...return1os.setsid()ifos.fork():# launch child again.return1# move to the root to avoit mount pbos.chdir('/')# set paranoid umaskos.umask(077)null=os.open('/dev/null',os.O_RDWR)foriinrange(3):try:os.dup2(null,i)exceptOSError,e:ife.errno!=errno.EBADF:raiseos.close(null)returnNonedefstart_task(interval,func):lc=task.LoopingCall(func)# wait until interval has expired to actually start the task, else we have# to wait all task to be finished for the server to be actually startedlc.start(interval,now=False)defhost_prefixed_baseurl(baseurl,host):scheme,netloc,url,query,fragment=urlsplit(baseurl)netloc_domain='.'+'.'.join(netloc.split('.')[-2:])ifhost.endswith(netloc_domain):netloc=hostbaseurl=urlunsplit((scheme,netloc,url,query,fragment))returnbaseurlclassLongTimeExpiringFile(static.File):"""overrides static.File and sets a far futre ``Expires`` date on the resouce. versions handling is done by serving static files by different URLs for each version. For instance:: http://localhost:8080/data-2.48.2/cubicweb.css http://localhost:8080/data-2.49.0/cubicweb.css etc. """defrenderHTTP(self,request):defsetExpireHeader(response):response=iweb.IResponse(response)# Don't provide additional resource information to error responsesifresponse.code<400:# the HTTP RFC recommands not going further than 1 year aheadexpires=date.today()+timedelta(days=6*30)response.headers.setHeader('Expires',mktime(expires.timetuple()))returnresponsed=maybeDeferred(super(LongTimeExpiringFile,self).renderHTTP,request)returnd.addCallback(setExpireHeader)classCubicWebRootResource(resource.PostableResource):addSlash=Falsedef__init__(self,config,debug=None):self.debugmode=debugself.config=config# instantiate publisher here and not in init_publisher to get some# checks done before daemonization (eg versions consistency)self.appli=CubicWebPublisher(config,debug=self.debugmode)self.base_url=config['base-url']self.https_url=config['https-url']self.versioned_datadir='data%s'%config.instance_md5_version()definit_publisher(self):config=self.config# when we have an in-memory repository, clean unused sessions every XX# seconds and properly shutdown the serverifconfig.repo_method=='inmemory':reactor.addSystemEventTrigger('before','shutdown',self.shutdown_event)ifconfig.pyro_enabled():# if pyro is enabled, we have to register to the pyro name# server, create a pyro daemon, and create a task to handle pyro# requestsself.pyro_daemon=self.appli.repo.pyro_register()self.pyro_listen_timeout=0.02self.appli.repo.looping_task(1,self.pyro_loop_event)self.appli.repo.start_looping_tasks()self.set_url_rewriter()CW_EVENT_MANAGER.bind('after-registry-reload',self.set_url_rewriter)defstart_service(self):config=self.configinterval=min(config['cleanup-session-time']or120,config['cleanup-anonymous-session-time']or720)/2.start_task(interval,self.appli.session_handler.clean_sessions)defset_url_rewriter(self):self.url_rewriter=self.appli.vreg['components'].select_or_none('urlrewriter')defshutdown_event(self):"""callback fired when the server is shutting down to properly clean opened sessions """self.appli.repo.shutdown()defpyro_loop_event(self):"""listen for pyro events"""try:self.pyro_daemon.handleRequests(self.pyro_listen_timeout)exceptselect.error:returndeflocateChild(self,request,segments):"""Indicate which resource to use to process down the URL's path"""ifsegments:ifsegments[0]=='https':segments=segments[1:]iflen(segments)>=2:ifsegments[0]in(self.versioned_datadir,'data','static'):# Anything in data/, static/ is treated as static filesifsegments[0]=='static':# instance static directorydatadir=self.config.static_directoryelifsegments[1]=='fckeditor':fckeditordir=self.config.ext_resources['FCKEDITOR_PATH']returnstatic.File(fckeditordir),segments[2:]else:# cube static data filedatadir=self.config.locate_resource(segments[1])ifdatadirisNone:returnNone,[]self.info('static file %s from %s',segments[-1],datadir)ifsegments[0]=='data':returnstatic.File(str(datadir)),segments[1:]else:returnLongTimeExpiringFile(datadir),segments[1:]elifsegments[0]=='fckeditor':fckeditordir=self.config.ext_resources['FCKEDITOR_PATH']returnstatic.File(fckeditordir),segments[1:]# Otherwise we use this single resourcereturnself,()defrender(self,request):"""Render a page from the root resource"""# reload modified files in debug modeifself.debugmode:self.appli.vreg.register_objects(self.config.vregistry_path())ifself.config['profile']:# default profiler don't trace threadsreturnself.render_request(request)else:returnthreads.deferToThread(self.render_request,request)defrender_request(self,request):origpath=request.pathhost=request.host# dual http/https access handling: expect a rewrite rule to prepend# 'https' to the path to detect https accessiforigpath.split('/',2)[1]=='https':origpath=origpath[6:]request.uri=request.uri[6:]https=Truebaseurl=self.https_urlorself.base_urlelse:https=Falsebaseurl=self.base_urlifself.config['use-request-subdomain']:baseurl=host_prefixed_baseurl(baseurl,host)self.warning('used baseurl is %s for this request',baseurl)req=CubicWebTwistedRequestAdapter(request,self.appli.vreg,https,baseurl)ifreq.authmode=='http':# activate realm-based authrealm=self.config['realm']req.set_header('WWW-Authenticate',[('Basic',{'realm':realm})],raw=False)try:self.appli.connect(req)exceptAuthenticationError:returnself.request_auth(req)exceptRedirect,ex:returnself.redirect(req,ex.location)ifhttpsandreq.cnx.anonymous_connection:# don't allow anonymous on https connectionreturnself.request_auth(req)ifself.url_rewriterisnotNone:# XXX should occur before authentication?try:path=self.url_rewriter.rewrite(host,origpath,req)exceptRedirect,ex:returnself.redirect(req,ex.location)request.uri.replace(origpath,path,1)else:path=origpathifnotpathorpath=="/":path='view'try:result=self.appli.publish(path,req)exceptDirectResponse,ex:returnex.responseexceptStatusResponse,ex:returnhttp.Response(stream=ex.content,code=ex.status,headers=req.headers_outorNone)exceptRemoteCallFailed,ex:req.set_header('content-type','application/json')returnhttp.Response(stream=ex.dumps(),code=responsecode.INTERNAL_SERVER_ERROR)exceptNotFound:result=self.appli.notfound_content(req)returnhttp.Response(stream=result,code=responsecode.NOT_FOUND,headers=req.headers_outorNone)exceptExplicitLogin:# must be before AuthenticationErrorreturnself.request_auth(req)exceptAuthenticationError,ex:ifself.config['auth-mode']=='cookie'andgetattr(ex,'url',None):returnself.redirect(req,ex.url)# in http we have to request auth to flush current http auth# informationreturnself.request_auth(req,loggedout=True)exceptRedirect,ex:returnself.redirect(req,ex.location)# request may be referenced by "onetime callback", so clear its entity# cache to avoid memory usagereq.drop_entity_cache()returnhttp.Response(stream=result,code=responsecode.OK,headers=req.headers_outorNone)defredirect(self,req,location):req.headers_out.setHeader('location',str(location))self.debug('redirecting to %s',location)# 303 See otherreturnhttp.Response(code=303,headers=req.headers_out)defrequest_auth(self,req,loggedout=False):ifself.https_urlandreq.base_url()!=self.https_url:req.headers_out.setHeader('location',self.https_url+'login')returnhttp.Response(code=303,headers=req.headers_out)ifself.config['auth-mode']=='http':code=responsecode.UNAUTHORIZEDelse:code=responsecode.FORBIDDENifloggedout:ifreq.https:req._base_url=self.base_urlreq.https=Falsecontent=self.appli.loggedout_content(req)else:content=self.appli.need_login_content(req)returnhttp.Response(code,req.headers_out,content)fromtwisted.internetimportdeferfromtwisted.web2importfileupload# XXX set max file size to 100Mo: put max upload size in the configuration# line below for twisted >= 8.0, default param value for earlier versionresource.PostableResource.maxSize=100*1024*1024defparsePOSTData(request,maxMem=100*1024,maxFields=1024,maxSize=100*1024*1024):ifrequest.stream.length==0:returndefer.succeed(None)ctype=request.headers.getHeader('content-type')ifctypeisNone:returndefer.succeed(None)defupdateArgs(data):args=datarequest.args.update(args)defupdateArgsAndFiles(data):args,files=datarequest.args.update(args)request.files.update(files)deferror(f):f.trap(fileupload.MimeFormatError)raisehttp.HTTPError(responsecode.BAD_REQUEST)ifctype.mediaType=='application'andctype.mediaSubtype=='x-www-form-urlencoded':d=fileupload.parse_urlencoded(request.stream,keep_blank_values=True)d.addCallbacks(updateArgs,error)returndelifctype.mediaType=='multipart'andctype.mediaSubtype=='form-data':boundary=ctype.params.get('boundary')ifboundaryisNone:returndefer.fail(http.HTTPError(http.StatusResponse(responsecode.BAD_REQUEST,"Boundary not specified in Content-Type.")))d=fileupload.parseMultipartFormData(request.stream,boundary,maxMem,maxFields,maxSize)d.addCallbacks(updateArgsAndFiles,error)returndelse:raisehttp.HTTPError(responsecode.BAD_REQUEST)server.parsePOSTData=parsePOSTDatafromloggingimportgetLoggerfromcubicwebimportset_log_methodsset_log_methods(CubicWebRootResource,getLogger('cubicweb.twisted'))listiterator=type(iter([]))def_gc_debug(all=True):importgcfrompprintimportpprintfromcubicweb.appobjectimportAppObjectgc.collect()count=0acount=0fcount=0rcount=0ccount=0scount=0ocount={}fromrql.stmtsimportUnionfromcubicweb.schemaimportCubicWebSchemafromcubicweb.rsetimportResultSetfromcubicweb.dbapiimportConnection,Cursorfromcubicweb.reqimportRequestSessionBasefromcubicweb.server.repositoryimportRepositoryfromcubicweb.server.sources.nativeimportNativeSQLSourcefromcubicweb.server.sessionimportSessionfromcubicweb.devtools.testlibimportCubicWebTCfromlogilab.common.testlibimportTestSuitefromoptparseimportValuesimporttypes,weakrefforobjingc.get_objects():ifisinstance(obj,RequestSessionBase):count+=1ifisinstance(obj,Session):print' session',obj,referrers(obj,True)elifisinstance(obj,AppObject):acount+=1elifisinstance(obj,ResultSet):rcount+=1#print ' rset', obj, referrers(obj)elifisinstance(obj,Repository):print' REPO',obj,referrers(obj,True)#elif isinstance(obj, NativeSQLSource):# print ' SOURCe', obj, referrers(obj)elifisinstance(obj,CubicWebTC):print' TC',obj,referrers(obj)elifisinstance(obj,TestSuite):print' SUITE',obj,referrers(obj)#elif isinstance(obj, Values):# print ' values', '%#x' % id(obj), referrers(obj, True)elifisinstance(obj,Connection):ccount+=1#print ' cnx', obj, referrers(obj)#elif isinstance(obj, Cursor):# ccount += 1# print ' cursor', obj, referrers(obj)elifisinstance(obj,file):fcount+=1# print ' open file', file.name, file.filenoelifisinstance(obj,CubicWebSchema):scount+=1print' schema',obj,referrers(obj)elifnotisinstance(obj,(type,tuple,dict,list,set,frozenset,weakref.ref,weakref.WeakKeyDictionary,listiterator,property,classmethod,types.ModuleType,types.MemberDescriptorType,types.FunctionType,types.MethodType)):try:ocount[obj.__class__]+=1exceptKeyError:ocount[obj.__class__]=1exceptAttributeError:passifcount:print' NB REQUESTS/SESSIONS',countifacount:print' NB APPOBJECTS',acountifccount:print' NB CONNECTIONS',ccountifrcount:print' NB RSETS',rcountifscount:print' NB SCHEMAS',scountiffcount:print' NB FILES',fcountifall:ocount=sorted(ocount.items(),key=lambdax:x[1],reverse=True)[:20]pprint(ocount)ifgc.garbage:print'UNREACHABLE',gc.garbagedefreferrers(obj,showobj=False):try:returnsorted(set((type(x),showobjandxorgetattr(x,'__name__','%#x'%id(x)))forxin_referrers(obj)))exceptTypeError:s=set()unhashable=[]forxin_referrers(obj):try:s.add(x)exceptTypeError:unhashable.append(x)returnsorted(s)+unhashabledef_referrers(obj,seen=None,level=0):importgc,typesfromcubicweb.schemaimportCubicWebRelationSchema,CubicWebEntitySchemainteresting=[]ifseenisNone:seen=set()forxingc.get_referrers(obj):ifid(x)inseen:continueseen.add(id(x))ifisinstance(x,types.FrameType):continueifisinstance(x,(CubicWebRelationSchema,CubicWebEntitySchema)):continueifisinstance(x,(list,tuple,set,dict,listiterator)):iflevel>=5:pass#interesting.append(x)else:interesting+=_referrers(x,seen,level+1)else:interesting.append(x)returninterestingdefrun(config,debug):# create the siteroot_resource=CubicWebRootResource(config,debug)website=server.Site(root_resource)# serve it via standard HTTP on port set in the configurationport=config['port']or8080reactor.listenTCP(port,channel.HTTPFactory(website))logger=getLogger('cubicweb.twisted')ifnotdebug:ifsys.platform=='win32':raiseConfigurationError("Under windows, you must use the service management ""commands (e.g : 'net start my_instance)'")print'instance starting in the background'ifdaemonize():return# child processifconfig['pid-file']:# ensure the directory where the pid-file should be set exists (for# instance /var/run/cubicweb may be deleted on computer restart)piddir=os.path.dirname(config['pid-file'])ifnotos.path.exists(piddir):os.makedirs(piddir)file(config['pid-file'],'w').write(str(os.getpid()))root_resource.init_publisher()# before changing uidifconfig['uid']isnotNone:try:uid=int(config['uid'])exceptValueError:frompwdimportgetpwnamuid=getpwnam(config['uid']).pw_uidos.setuid(uid)root_resource.start_service()logger.info('instance started on %s',root_resource.base_url)ifconfig['profile']:importcProfilecProfile.runctx('reactor.run()',globals(),locals(),config['profile'])else:reactor.run()