don't fail when trying to update relation cache for relation deletion, this may occurs regularly
"""provide a minimal RQL support for google appengine dbmodel:organization: Logilab:copyright: 2008-2009 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2.:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses"""__docformat__="restructuredtext en"fromdatetimeimportdatetimefromrqlimportRQLHelper,nodesfromcubicwebimportBinaryfromcubicweb.rsetimportResultSetfromcubicweb.serverimportSQL_CONNECT_HOOKSfromgoogle.appengine.api.datastoreimportKey,Get,Query,Entityfromgoogle.appengine.api.datastore_typesimportText,Blobfromgoogle.appengine.api.datastore_errorsimportEntityNotFoundError,BadKeyErrordefetype_from_key(key):returnKey(key).kind()defposs_var_types(myvar,ovar,kind,solutions):returnfrozenset(etypes[myvar]foretypesinsolutionsifetypes[ovar]==kind)defexpand_result(results,result,myvar,values,dsget=None):values=map(dsget,values)ifvalues:result[myvar]=values.pop(0)forvalueinvalues:newresult=result.copy()newresult[myvar]=valueresults.append(newresult)else:results.remove(result)def_resolve(restrictions,solutions,fixed):varname=restrictions[0].searched_varobjs=[]foretypeinfrozenset(etypes[varname]foretypesinsolutions):gqlargs={}query=Query(etype)forrestrictioninrestrictions:restriction.fill_query(fixed,query)pobjs=query.Run()ifvarnameinfixed:value=fixed[varname]objs+=(xforxinpobjsifx==value)else:objs+=pobjsifvarnameinfixedandnotobjs:raiseEidMismatch(varname,value)returnobjsdef_resolve_not(restrictions,solutions,fixed):restr=restrictions[0]constrvarname=restr.constraint_variflen(restrictions)>1ornotconstrvarnameinfixed:raiseNotImplementedError()varname=restr.searched_varobjs=[]foretypeinfrozenset(etypes[varname]foretypesinsolutions):gqlargs={}foroperatorin('<','>'):query=Query(etype)restr.fill_query(fixed,query,operator)objs+=query.Run()returnobjsdef_print_results(rlist):return'[%s]'%', '.join(_print_result(r)forrinrlist)def_print_result(rdict):string=[]fork,vinrdict.iteritems():ifisinstance(v,Entity):string.append('%s: %s'%(k,v.key()))#_print_result(v)))elifisinstance(v,list):string.append('%s: [%s]'%(k,', '.join(str(i)foriinv)))else:string.append('%s: %s'%(k,v))return'{%s}'%', '.join(string)classEidMismatch(Exception):def__init__(self,varname,value):self.varname=varnameself.value=valueclassRestriction(object):supported_operators=('=',)def__init__(self,rel):operator=rel.children[1].operatorifnotoperatorinself.supported_operators:raiseNotImplementedError('unsupported operator')self.rel=relself.operator=operatorself.rtype=rel.r_typeself.var=rel.children[0]def__repr__(self):return'<%s for %s>'%(self.__class__.__name__,self.rel)@propertydefrhs(self):returnself.rel.children[1].children[0]classMultipleRestriction(object):def__init__(self,restrictions):self.restrictions=restrictionsdefresolve(self,solutions,fixed):return_resolve(self.restrictions,solutions,fixed)classVariableSelection(Restriction):def__init__(self,rel,dsget,prefix='s'):Restriction.__init__(self,rel)self._dsget=dsgetself._not=self.rel.neged(strict=True)self._prefix=prefix+'_'def__repr__(self):return'<%s%s for %s>'%(self._prefix[0],self.__class__.__name__,self.rel)@propertydefsearched_var(self):ifself._prefix=='s_':returnself.var.namereturnself.rhs.name@propertydefconstraint_var(self):ifself._prefix=='s_':returnself.rhs.namereturnself.var.namedef_possible_values(self,myvar,ovar,entity,solutions,dsprefix):ifself.rtype=='identity':return(entity.key(),)value=entity.get(dsprefix+self.rtype)ifvalueisNone:return[]ifnotisinstance(value,list):value=[value]vartypes=poss_var_types(myvar,ovar,entity.kind(),solutions)return(vforvinvalueifv.kind()invartypes)defcomplete_and_filter(self,solutions,results):myvar=self.rhs.nameovar=self.var.namertype=self.rtypeifself.schema.rschema(rtype).is_final():# should be detected by rql.stcheck: "Any C WHERE NOT X attr C" doesn't make sense#if self._not:# raise NotImplementedError()forresultinresults:result[myvar]=result[ovar].get('s_'+rtype)elifself.var.nameinresults[0]:ifself.rhs.nameinresults[0]:self.filter(solutions,results)else:ifself._not:raiseNotImplementedError()forresultinresults[:]:values=self._possible_values(myvar,ovar,result[ovar],solutions,'s_')expand_result(results,result,myvar,values,self._dsget)else:assertself.rhs.nameinresults[0]self.object_complete_and_filter(solutions,results)deffilter(self,solutions,results):myvar=self.rhs.nameovar=self.var.namenewsols={}forresultinresults[:]:entity=result[ovar]key=entity.key()ifnotkeyinnewsols:values=self._possible_values(myvar,ovar,entity,solutions,'s_')newsols[key]=frozenset(vforvinvalues)ifself._not:ifresult[myvar].key()innewsols[key]:results.remove(result)elifnotresult[myvar].key()innewsols[key]:results.remove(result)defobject_complete_and_filter(self,solutions,results):ifself._not:raiseNotImplementedError()myvar=self.var.nameovar=self.rhs.nameforresultinresults[:]:values=self._possible_values(myvar,ovar,result[ovar],solutions,'o_')expand_result(results,result,myvar,values,self._dsget)classEidRestriction(Restriction):def__init__(self,rel,dsget):Restriction.__init__(self,rel)self._dsget=dsgetdefresolve(self,kwargs):value=self.rel.children[1].children[0].eval(kwargs)returnself._dsget(value)classRelationRestriction(VariableSelection):def_get_value(self,fixed):returnfixed[self.constraint_var].key()deffill_query(self,fixed,query,operator=None):restr='%s%s%s'%(self._prefix,self.rtype,operatororself.operator)query[restr]=self._get_value(fixed)defresolve(self,solutions,fixed):ifself.rtype=='identity':ifself._not:raiseNotImplementedError()return[fixed[self.constraint_var]]ifself._not:return_resolve_not([self],solutions,fixed)return_resolve([self],solutions,fixed)classNotRelationRestriction(RelationRestriction):def_get_value(self,fixed):returnNonedefresolve(self,solutions,fixed):ifself.rtype=='identity':raiseNotImplementedError()return_resolve([self],solutions,fixed)classAttributeRestriction(RelationRestriction):supported_operators=('=','>','>=','<','<=','ILIKE')def__init__(self,rel,kwargs):RelationRestriction.__init__(self,rel,None)value=self.rhs.eval(kwargs)self.value=valueifself.operator=='ILIKE':ifvalue.startswith('%'):raiseNotImplementedError('LIKE is only supported for prefix search')ifnotvalue.endswith('%'):raiseNotImplementedError('LIKE is only supported for prefix search')self.operator='>'self.value=value[:-1]defcomplete_and_filter(self,solutions,results):# check lhs var first in case this is a restrictionassertself._notmyvar,rtype,value=self.var.name,self.rtype,self.valueforresultinresults[:]:ifresult[myvar].get('s_'+rtype)==value:results.remove(result)def_get_value(self,fixed):returnself.valueclassDateAttributeRestriction(AttributeRestriction):"""just a thin layer on top af `AttributeRestriction` that tries to convert date strings such as in : Any X WHERE X creation_date >= '2008-03-04' """def__init__(self,rel,kwargs):super(DateAttributeRestriction,self).__init__(rel,kwargs)ifisinstance(self.value,basestring):# try:self.value=datetime.strptime(self.value,'%Y-%m-%d')# except Exception, exc:# from logging import error# error('unable to parse date %s with format %%Y-%%m-%%d (exc=%s)', value, exc)classAttributeInRestriction(AttributeRestriction):def__init__(self,rel,kwargs):RelationRestriction.__init__(self,rel,None)values=[]forcinself.rel.children[1].iget_nodes(nodes.Constant):values.append(c.eval(kwargs))self.value=values@propertydefoperator(self):return'in'classTypeRestriction(AttributeRestriction):def__init__(self,var):self.var=vardef__repr__(self):return'<%s for %s>'%(self.__class__.__name__,self.var)defresolve(self,solutions,fixed):objs=[]foretypeinfrozenset(etypes[self.var.name]foretypesinsolutions):objs+=Query(etype).Run()returnobjsdefappend_result(res,descr,i,j,value,etype):ifvalueisnotNone:ifisinstance(value,Text):value=unicode(value)elifisinstance(value,Blob):value=Binary(str(value))ifj==0:res.append([value])descr.append([etype])else:res[i].append(value)descr[i].append(etype)classValueResolver(object):def__init__(self,functions,args,term):self.functions=functionsself.args=argsself.term=termself._solution=self.term.stmt.solutions[0]defcompute(self,result):"""return (entity type, value) to which self.term is evaluated according to the given result dictionnary and to query arguments (self.args) """returnself.term.accept(self,result)defvisit_function(self,node,result):args=tuple(n.accept(self,result)[1]forninnode.children)value=self.functions[node.name](*args)returnnode.get_type(self._solution,self.args),valuedefvisit_variableref(self,node,result):value=result[node.name]try:etype=value.kind()value=str(value.key())exceptAttributeError:etype=self._solution[node.name]returnetype,valuedefvisit_constant(self,node,result):returnnode.get_type(kwargs=self.args),node.eval(self.args)classRQLInterpreter(object):"""algorithm: 1. visit the restriction clauses and collect restriction for each subject of a relation. Different restriction types are: * EidRestriction * AttributeRestriction * RelationRestriction * VariableSelection (not really a restriction) -> dictionary {<variable>: [restriction...], ...} 2. resolve eid restrictions 3. for each select in union: for each solution in select'solutions: 1. resolve variables which have attribute restriction 2. resolve relation restriction 3. resolve selection and add to global results """def__init__(self,schema):self.schema=schemaRestriction.schema=schema# yalta!self.rqlhelper=RQLHelper(schema,{'eid':etype_from_key})self._stored_proc={'LOWER':lambdax:x.lower(),'UPPER':lambdax:x.upper()}forcbinSQL_CONNECT_HOOKS.get('sqlite',[]):cb(self)# emulate sqlite connection interface so we can reuse stored proceduresdefcreate_function(self,name,nbargs,func):self._stored_proc[name]=funcdefcreate_aggregate(self,name,nbargs,func):self._stored_proc[name]=funcdefexecute(self,operation,parameters=None,eid_key=None,build_descr=True):rqlst=self.rqlhelper.parse(operation,annotate=True)try:self.rqlhelper.compute_solutions(rqlst,kwargs=parameters)exceptBadKeyError:results,description=[],[]else:results,description=self.interpret(rqlst,parameters)returnResultSet(results,operation,parameters,description,rqlst=rqlst)definterpret(self,node,kwargs,dsget=None):ifdsgetisNone:self._dsget=Getelse:self._dsget=dsgettry:returnnode.accept(self,kwargs)exceptNotImplementedError:self.critical('support for query not implemented: %s',node)raisedefvisit_union(self,node,kwargs):results,description=[],[]extra={'kwargs':kwargs}forchildinnode.children:pres,pdescr=self.visit_select(child,extra)results+=presdescription+=pdescrreturnresults,descriptiondefvisit_select(self,node,extra):constraints={}ifnode.whereisnotNone:node.where.accept(self,constraints,extra)fixed,toresolve,postresolve,postfilters={},{},{},[]# extract NOT filtersforvname,restrictionsinconstraints.items():forrestrinrestrictions[:]:ifisinstance(restr,AttributeRestriction)andrestr._not:postfilters.append(restr)restrictions.remove(restr)ifnotrestrictions:delconstraints[vname]# add TypeRestriction for variable which have no restrictions at allforvarname,varinnode.defined_vars.iteritems():ifnotvarnameinconstraints:constraints[varname]=[TypeRestriction(var)]#print node, constraints# compute eid restrictionskwargs=extra['kwargs']forvarname,restrictionsinconstraints.iteritems():forrestrinrestrictions[:]:ifisinstance(restr,EidRestriction):assertnotvarnameinfixedtry:value=restr.resolve(kwargs)fixed[varname]=valueexceptEntityNotFoundError:return[],[]restrictions.remove(restr)#print 'fixed', fixed.keys()# combine remaining restrictionsforvarname,restrictionsinconstraints.iteritems():forrestrinrestrictions:ifisinstance(restr,AttributeRestriction):toresolve.setdefault(varname,[]).append(restr)elifisinstance(restr,NotRelationRestriction)or(isinstance(restr,RelationRestriction)andnotrestr.searched_varinfixedandrestr.constraint_varinfixed):toresolve.setdefault(varname,[]).append(restr)else:postresolve.setdefault(varname,[]).append(restr)try:iflen(toresolve[varname])>1:toresolve[varname]=MultipleRestriction(toresolve[varname])else:toresolve[varname]=toresolve[varname][0]exceptKeyError:pass#print 'toresolve %s' % toresolve#print 'postresolve %s' % postresolve# resolve additional restrictionsiffixed:partres=[fixed.copy()]else:partres=[]forvarname,restrintoresolve.iteritems():varpartres=partres[:]try:values=tuple(restr.resolve(node.solutions,fixed))exceptEidMismatch,ex:varname=ex.varnamevalue=ex.valuepartres=[resforresinpartresifres[varname]!=value]ifpartres:continue# some join failed, no possible resultsreturn[],[]ifnotvalues:# some join failed, no possible resultsreturn[],[]ifnotvarpartres:# init resultsforvalueinvalues:partres.append({varname:value})elifnotvarnameinpartres[0]:# cartesian productforresinpartres:res[varname]=values[0]forresinpartres[:]:forvalueinvalues[1:]:res=res.copy()res[varname]=valuepartres.append(res)else:# unionforresinvarpartres:forvalueinvalues:res=res.copy()res[varname]=valuepartres.append(res)#print 'partres', len(partres)#print partres# Note: don't check for empty partres since constant selection may still# produce result at this point# sort to get RelationRestriction before AttributeSelectionrestrictions=sorted((restrforrestrictionsinpostresolve.itervalues()forrestrinrestrictions),key=lambdax:notisinstance(x,RelationRestriction))# compute stuff not doable in the previous step using datastore queriesforrestrinrestrictions+postfilters:restr.complete_and_filter(node.solutions,partres)ifnotpartres:# some join failed, no possible resultsreturn[],[]ifextra.pop('has_exists',False):# remove potential duplicates introduced by existstoremovevars=[v.nameforvinnode.defined_vars.itervalues()ifnotv.scopeisnode]iftoremovevars:newpartres=[]forresultinpartres:forvarintoremovevars:delresult[var]ifnotresultinnewpartres:newpartres.append(result)ifnotnewpartres:# some join failed, no possible resultsreturn[],[]partres=newpartresifnode.orderby:forsortterminreversed(node.orderby):resolver=ValueResolver(self._stored_proc,kwargs,sortterm.term)partres.sort(reverse=notsortterm.asc,key=lambdax:resolver.compute(x)[1])ifpartres:ifnode.offset:partres=partres[node.offset:]ifnode.limit:partres=partres[:node.limit]ifnotpartres:return[],[]#print 'completed partres', _print_results(partres)# compute resultsres,descr=[],[]forj,terminenumerate(node.selection):resolver=ValueResolver(self._stored_proc,kwargs,term)ifnotpartres:etype,value=resolver.compute({})# only constant selectedifnotres:res.append([])descr.append([])res[0].append(value)descr[0].append(etype)else:fori,solinenumerate(partres):etype,value=resolver.compute(sol)append_result(res,descr,i,j,value,etype)#print '--------->', resreturnres,descrdefvisit_and(self,node,constraints,extra):forchildinnode.children:child.accept(self,constraints,extra)defvisit_exists(self,node,constraints,extra):extra['has_exists']=Trueself.visit_and(node,constraints,extra)defvisit_not(self,node,constraints,extra):forchildinnode.children:child.accept(self,constraints,extra)try:extra.pop(node)exceptKeyError:raiseNotImplementedError()defvisit_relation(self,node,constraints,extra):ifnode.is_types_restriction():returnrschema=self.schema.rschema(node.r_type)neged=node.neged(strict=True)ifneged:# ok, we *may* process this Not node (not implemented error will be# raised later if we can't)extra[node.parent]=Trueifrschema.is_final():self._visit_final_relation(rschema,node,constraints,extra)elifneged:self._visit_non_final_neged_relation(rschema,node,constraints)else:self._visit_non_final_relation(rschema,node,constraints)def_visit_non_final_relation(self,rschema,node,constraints,not_=False):lhs,rhs=node.get_variable_parts()forv1,v2,prefixin((lhs,rhs,'s'),(rhs,lhs,'o')):#if not_:nbrels=len(v2.variable.stinfo['relations'])#else:# nbrels = len(v2.variable.stinfo['relations']) - len(v2.variable.stinfo['uidrels'])ifnbrels>1:constraints.setdefault(v1.name,[]).append(RelationRestriction(node,self._dsget,prefix))# just init an empty list for v2 variable to avoid a# TypeRestriction being added for itconstraints.setdefault(v2.name,[])breakelse:constraints.setdefault(rhs.name,[]).append(VariableSelection(node,self._dsget,'s'))def_visit_non_final_neged_relation(self,rschema,node,constraints):lhs,rhs=node.get_variable_parts()forv1,v2,prefixin((lhs,rhs,'s'),(rhs,lhs,'o')):stinfo=v2.variable.stinfoifnotstinfo['selected']andlen(stinfo['relations'])==1:constraints.setdefault(v1.name,[]).append(NotRelationRestriction(node,self._dsget,prefix))constraints.setdefault(v2.name,[])breakelse:self._visit_non_final_relation(rschema,node,constraints,True)def_visit_final_relation(self,rschema,node,constraints,extra):varname=node.children[0].nameifrschema.type=='eid':constraints.setdefault(varname,[]).append(EidRestriction(node,self._dsget))else:rhs=node.children[1].children[0]ifisinstance(rhs,nodes.VariableRef):constraints.setdefault(rhs.name,[]).append(VariableSelection(node,self._dsget))elifisinstance(rhs,nodes.Constant):ifrschema.objects()[0]in('Datetime','Date'):# XXXconstraints.setdefault(varname,[]).append(DateAttributeRestriction(node,extra['kwargs']))else:constraints.setdefault(varname,[]).append(AttributeRestriction(node,extra['kwargs']))elifisinstance(rhs,nodes.Function)andrhs.name=='IN':constraints.setdefault(varname,[]).append(AttributeInRestriction(node,extra['kwargs']))else:raiseNotImplementedError()def_not_implemented(self,*args,**kwargs):raiseNotImplementedError()visit_or=_not_implemented# shouldn't occursvisit_set=_not_implementedvisit_insert=_not_implementedvisit_delete=_not_implementedfromloggingimportgetLoggerfromcubicwebimportset_log_methodsset_log_methods(RQLInterpreter,getLogger('cubicweb.goa.rqlinterpreter'))set_log_methods(Restriction,getLogger('cubicweb.goa.rqlinterpreter'))