[entity] small optimization: once an entity has been completed, don't redo it (for nothing)
"""additional cubicweb-ctl commands and command handlers for cubicweb and cubicweb'scubes development:organization: Logilab:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2.:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses"""__docformat__="restructuredtext en"importsysfromdatetimeimportdatetimefromosimportmkdir,chdir,getcwdfromos.pathimportjoin,exists,abspath,basename,normpath,split,isdirfromcopyimportdeepcopyfromwarningsimportwarnfromlogilab.commonimportSTD_BLACKLISTfromlogilab.common.modutilsimportget_module_filesfromlogilab.common.textutilsimportsplitstripfromlogilab.common.shellutilsimportASKfromlogilab.common.clcommandsimportregister_commandsfromcubicweb.__pkginfo__importversionascubicwebversionfromcubicwebimport(CW_SOFTWARE_ROOTasBASEDIR,BadCommandUsage,underline_title)fromcubicweb.schemaimportCONSTRAINTSfromcubicweb.toolsutilsimportCommand,copy_skeletonfromcubicweb.web.webconfigimportWebConfigurationfromcubicweb.server.serverconfigimportServerConfigurationclassDevCubeConfiguration(ServerConfiguration,WebConfiguration):"""dummy config to get full library schema and entities"""creating=Truecubicweb_appobject_path=ServerConfiguration.cubicweb_appobject_path|WebConfiguration.cubicweb_appobject_pathcube_appobject_path=ServerConfiguration.cube_appobject_path|WebConfiguration.cube_appobject_pathdef__init__(self,cube):super(DevCubeConfiguration,self).__init__(cube)ifcubeisNone:self._cubes=()else:self._cubes=self.reorder_cubes(self.expand_cubes(self.my_cubes(cube)))defmy_cubes(self,cube):return(cube,)+self.cube_dependencies(cube)+self.cube_recommends(cube)@propertydefapphome(self):returnNonedefmain_config_file(self):returnNonedefinit_log(self,debug=None):passdefload_configuration(self):passclassDevDepConfiguration(DevCubeConfiguration):"""configuration to use to generate cubicweb po files or to use as "library" configuration to filter out message ids from cubicweb and dependencies of a cube """defmy_cubes(self,cube):returnself.cube_dependencies(cube)+self.cube_recommends(cube)defdefault_log_file(self):returnNonedefcleanup_sys_modules(config):# cleanup sys.modules, required when we're updating multiple cubesforname,modinsys.modules.items():ifmodisNone:# duh ? logilab.common.os for instancedelsys.modules[name]continueifnothasattr(mod,'__file__'):continueforpathinconfig.vregistry_path():ifmod.__file__.startswith(path):delsys.modules[name]break# fresh rtagsfromcubicwebimportrtagsfromcubicweb.webimportuicfgrtags.RTAGS[:]=[]reload(uicfg)defgenerate_schema_pot(w,cubedir=None):"""generate a pot file with schema specific i18n messages notice that relation definitions description and static vocabulary should be marked using '_' and extracted using xgettext """fromcubicweb.cwvregimportCubicWebVRegistrycube=cubedirandsplit(cubedir)[-1]libconfig=DevDepConfiguration(cube)libconfig.cleanup_interface_sobjects=Falsecleanup_sys_modules(libconfig)ifcubedir:config=DevCubeConfiguration(cube)config.cleanup_interface_sobjects=Falseelse:config=libconfiglibconfig=Noneschema=config.load_schema(remove_unused_rtypes=False)vreg=CubicWebVRegistry(config)# set_schema triggers objects registrationsvreg.set_schema(schema)w(DEFAULT_POT_HEAD)_generate_schema_pot(w,vreg,schema,libconfig=libconfig,cube=cube)def_generate_schema_pot(w,vreg,schema,libconfig=None,cube=None):fromcubicweb.common.i18nimportadd_msgfromcubicweb.webimportuicfgfromcubicweb.schemaimportMETA_RTYPES,SYSTEM_RTYPESno_context_rtypes=META_RTYPES|SYSTEM_RTYPESw('# schema pot file, generated on %s\n'%datetime.now().strftime('%Y-%m-%d %H:%M:%S'))w('# \n')w('# singular and plural forms for each entity type\n')w('\n')vregdone=set()iflibconfigisnotNone:fromcubicweb.cwvregimportCubicWebVRegistry,clear_rtag_objectslibschema=libconfig.load_schema(remove_unused_rtypes=False)rinlined=deepcopy(uicfg.autoform_is_inlined)appearsin_addmenu=deepcopy(uicfg.actionbox_appearsin_addmenu)clear_rtag_objects()cleanup_sys_modules(libconfig)libvreg=CubicWebVRegistry(libconfig)libvreg.set_schema(libschema)# trigger objects registrationlibrinlined=uicfg.autoform_is_inlinedlibappearsin_addmenu=uicfg.actionbox_appearsin_addmenu# prefill vregdone setlist(_iter_vreg_objids(libvreg,vregdone))else:libschema={}rinlined=uicfg.autoform_is_inlinedappearsin_addmenu=uicfg.actionbox_appearsin_addmenuforcstrtypeinCONSTRAINTS:add_msg(w,cstrtype)done=set()foreschemainsorted(schema.entities()):etype=eschema.typeifetypenotinlibschema:add_msg(w,etype)add_msg(w,'%s_plural'%etype)ifnoteschema.final:add_msg(w,'This %s'%etype)add_msg(w,'New %s'%etype)ifeschema.descriptionandnoteschema.descriptionindone:done.add(eschema.description)add_msg(w,eschema.description)ifeschema.final:continueforrschema,targetschemas,roleineschema.relation_definitions(True):fortschemaintargetschemas:ifrinlined.etype_get(eschema,rschema,role,tschema)and \(libconfigisNoneornotlibrinlined.etype_get(eschema,rschema,role,tschema)):add_msg(w,'add a %s'%tschema,'inlined:%s.%s.%s'%(etype,rschema,role))add_msg(w,'remove this %s'%tschema,'inlined:%s.%s.%s'%(etype,rschema,role))add_msg(w,'This %s'%tschema,'inlined:%s.%s.%s'%(etype,rschema,role))ifappearsin_addmenu.etype_get(eschema,rschema,role,tschema)and \(libconfigisNoneornotlibappearsin_addmenu.etype_get(eschema,rschema,role,tschema)):ifrole=='subject':label='add %s%s%s%s'%(eschema,rschema,tschema,role)label2="creating %s (%s%%(linkto)s %s%s)"%(tschema,eschema,rschema,tschema)else:label='add %s%s%s%s'%(tschema,rschema,eschema,role)label2="creating %s (%s%s%s%%(linkto)s)"%(tschema,tschema,rschema,eschema)add_msg(w,label)add_msg(w,label2)# XXX also generate "creating ...' messages for actions in the# addrelated submenuw('# subject and object forms for each relation type\n')w('# (no object form for final or symetric relation types)\n')w('\n')forrschemainsorted(schema.relations()):rtype=rschema.typeifrtypenotinlibschema:# bw compat, necessary until all translation of relation are done properly...add_msg(w,rtype)ifrschema.descriptionandrschema.descriptionnotindone:done.add(rschema.description)add_msg(w,rschema.description)done.add(rtype)librschema=Noneelse:librschema=libschema.rschema(rtype)# add context information only for non-metadata rtypesifrschemanotinno_context_rtypes:libsubjects=librschemaandlibrschema.subjects()or()forsubjschemainrschema.subjects():ifnotsubjschemainlibsubjects:add_msg(w,rtype,subjschema.type)ifnot(schema.rschema(rtype).finalorrschema.symetric):ifrschemanotinno_context_rtypes:libobjects=librschemaandlibrschema.objects()or()forobjschemainrschema.objects():ifnotobjschemainlibobjects:add_msg(w,'%s_object'%rtype,objschema.type)ifrtypenotinlibschema:# bw compat, necessary until all translation of relation are done properly...add_msg(w,'%s_object'%rtype)forobjidin_iter_vreg_objids(vreg,vregdone):add_msg(w,'%s_description'%objid)add_msg(w,objid)def_iter_vreg_objids(vreg,done,prefix=None):forreg,objdictinvreg.items():forobjectsinobjdict.values():forobjinobjects:objid='%s_%s'%(reg,obj.id)ifobjidindone:breakifobj.property_defs:yieldobjiddone.add(objid)breakdefdefined_in_library(etype,rtype,tetype,role):"""return true if the given relation definition exists in cubicweb's library """iflibschemaisNone:returnFalseifrole=='subject':subjtype,objtype=etype,tetypeelse:subjtype,objtype=tetype,etypetry:returnlibschema.rschema(rtype).has_rdef(subjtype,objtype)exceptKeyError:returnFalseLANGS=('en','fr','es')I18NDIR=join(BASEDIR,'i18n')DEFAULT_POT_HEAD=r'''msgid ""msgstr """Project-Id-Version: cubicweb %s\n""PO-Revision-Date: 2008-03-28 18:14+0100\n""Last-Translator: Logilab Team <contact@logilab.fr>\n""Language-Team: fr <contact@logilab.fr>\n""MIME-Version: 1.0\n""Content-Type: text/plain; charset=UTF-8\n""Content-Transfer-Encoding: 8bit\n""Generated-By: cubicweb-devtools\n""Plural-Forms: nplurals=2; plural=(n > 1);\n"'''%cubicwebversionclassUpdateCubicWebCatalogCommand(Command):"""Update i18n catalogs for cubicweb library. It will regenerate cubicweb/i18n/xx.po files. You'll have then to edit those files to add translations of newly added messages. """name='i18ncubicweb'defrun(self,args):"""run the command with its specific arguments"""ifargs:raiseBadCommandUsage('Too much arguments')importshutilimporttempfileimportyamsfromlogilab.common.fileutilsimportensure_fs_modefromlogilab.common.shellutilsimportglobfind,find,rmfromcubicweb.common.i18nimportextract_from_tal,executetempdir=tempfile.mkdtemp()potfiles=[join(I18NDIR,'static-messages.pot')]print'-> extract schema messages.'schemapot=join(tempdir,'schema.pot')potfiles.append(schemapot)# explicit close necessary else the file may not be yet flushed when# we'll using it belowschemapotstream=file(schemapot,'w')generate_schema_pot(schemapotstream.write,cubedir=None)schemapotstream.close()print'-> extract TAL messages.'tali18nfile=join(tempdir,'tali18n.py')extract_from_tal(find(join(BASEDIR,'web'),('.py','.pt')),tali18nfile)print'-> generate .pot files.'forid,files,langin[('pycubicweb',get_module_files(BASEDIR)+list(globfind(join(BASEDIR,'misc','migration'),'*.py')),None),('schemadescr',globfind(join(BASEDIR,'schemas'),'*.py'),None),('yams',get_module_files(yams.__path__[0]),None),('tal',[tali18nfile],None),('js',globfind(join(BASEDIR,'web'),'cub*.js'),'java'),]:cmd='xgettext --no-location --omit-header -k_ -o %s%s'iflangisnotNone:cmd+=' -L %s'%langpotfile=join(tempdir,'%s.pot'%id)execute(cmd%(potfile,' '.join('"%s"'%fforfinfiles)))ifexists(potfile):potfiles.append(potfile)else:print'-> WARNING: %s file was not generated'%potfileprint'-> merging %i .pot files'%len(potfiles)cubicwebpot=join(tempdir,'cubicweb.pot')execute('msgcat -o %s%s'%(cubicwebpot,' '.join('"%s"'%fforfinpotfiles)))print'-> merging main pot file with existing translations.'chdir(I18NDIR)toedit=[]forlanginLANGS:target='%s.po'%langexecute('msgmerge -N --sort-output -o "%snew" "%s" "%s"'%(target,target,cubicwebpot))ensure_fs_mode(target)shutil.move('%snew'%target,target)toedit.append(abspath(target))# cleanuprm(tempdir)# instructions pour la suiteprint'-> regenerated CubicWeb\'s .po catalogs.'print'\nYou can now edit the following files:'print'* '+'\n* '.join(toedit)print'when you are done, run "cubicweb-ctl i18ncube yourcube".'classUpdateTemplateCatalogCommand(Command):"""Update i18n catalogs for cubes. If no cube is specified, update catalogs of all registered cubes. """name='i18ncube'arguments='[<cube>...]'defrun(self,args):"""run the command with its specific arguments"""ifargs:cubes=[DevCubeConfiguration.cube_dir(cube)forcubeinargs]else:cubes=[DevCubeConfiguration.cube_dir(cube)forcubeinDevCubeConfiguration.available_cubes()]cubes=[cubepathforcubepathincubesifexists(join(cubepath,'i18n'))]update_cubes_catalogs(cubes)defupdate_cubes_catalogs(cubes):forcubedirincubes:toedit=[]ifnotisdir(cubedir):print'-> ignoring %s that is not a directory.'%cubedircontinuetry:toedit+=update_cube_catalogs(cubedir)exceptException:importtracebacktraceback.print_exc()print'-> error while updating catalogs for cube',cubedirelse:# instructions pour la suiteprint'-> regenerated .po catalogs for cube %s.'%cubedirprint'\nYou can now edit the following files:'print'* '+'\n* '.join(toedit)print('When you are done, run "cubicweb-ctl i18ninstance ''<yourinstance>" to see changes in your instances.')defupdate_cube_catalogs(cubedir):importshutilimporttempfilefromlogilab.common.fileutilsimportensure_fs_modefromlogilab.common.shellutilsimportfind,rmfromcubicweb.common.i18nimportextract_from_tal,executetoedit=[]cube=basename(normpath(cubedir))tempdir=tempfile.mkdtemp()printunderline_title('Updating i18n catalogs for cube %s'%cube)chdir(cubedir)ifexists(join('i18n','entities.pot')):warn('entities.pot is deprecated, rename file to static-messages.pot (%s)'%join('i18n','entities.pot'),DeprecationWarning)potfiles=[join('i18n','entities.pot')]elifexists(join('i18n','static-messages.pot')):potfiles=[join('i18n','static-messages.pot')]else:potfiles=[]print'-> extract schema messages'schemapot=join(tempdir,'schema.pot')potfiles.append(schemapot)# explicit close necessary else the file may not be yet flushed when# we'll using it belowschemapotstream=file(schemapot,'w')generate_schema_pot(schemapotstream.write,cubedir)schemapotstream.close()print'-> extract TAL messages'tali18nfile=join(tempdir,'tali18n.py')extract_from_tal(find('.',('.py','.pt'),blacklist=STD_BLACKLIST+('test',)),tali18nfile)print'-> extract Javascript messages'jsfiles=[jsfileforjsfileinfind('.','.js')ifbasename(jsfile).startswith('cub')]ifjsfiles:tmppotfile=join(tempdir,'js.pot')execute('xgettext --no-location --omit-header -k_ -L java --from-code=utf-8 -o %s%s'%(tmppotfile,' '.join(jsfiles)))# no pot file created if there are no string to translateifexists(tmppotfile):potfiles.append(tmppotfile)print'-> create cube-specific catalog'tmppotfile=join(tempdir,'generated.pot')cubefiles=find('.','.py',blacklist=STD_BLACKLIST+('test',))cubefiles.append(tali18nfile)execute('xgettext --no-location --omit-header -k_ -o %s%s'%(tmppotfile,' '.join('"%s"'%fforfincubefiles)))ifexists(tmppotfile):# doesn't exists of no translation string foundpotfiles.append(tmppotfile)potfile=join(tempdir,'cube.pot')print'-> merging %i .pot files:'%len(potfiles)execute('msgcat -o %s%s'%(potfile,' '.join('"%s"'%fforfinpotfiles)))print'-> merging main pot file with existing translations:'chdir('i18n')forlanginLANGS:print'-> language',langcubepo='%s.po'%langifnotexists(cubepo):shutil.copy(potfile,cubepo)else:execute('msgmerge -N -s -o %snew %s%s'%(cubepo,cubepo,potfile))ensure_fs_mode(cubepo)shutil.move('%snew'%cubepo,cubepo)toedit.append(abspath(cubepo))# cleanuprm(tempdir)returntoeditclassLiveServerCommand(Command):"""Run a server from within a cube directory. """name='live-server'arguments=''options=()defrun(self,args):"""run the command with its specific arguments"""fromcubicweb.devtools.livetestimportrunserverrunserver()classNewCubeCommand(Command):"""Create a new cube. <cubename> the name of the new cube """name='newcube'arguments='<cubename>'options=(("directory",{'short':'d','type':'string','metavar':'<cubes directory>','help':'directory where the new cube should be created',}),("verbose",{'short':'v','type':'yn','metavar':'<verbose>','default':'n','help':'verbose mode: will ask all possible configuration questions',}),("author",{'short':'a','type':'string','metavar':'<author>','default':'LOGILAB S.A. (Paris, FRANCE)','help':'cube author',}),("author-email",{'short':'e','type':'string','metavar':'<email>','default':'contact@logilab.fr','help':'cube author\'s email',}),("author-web-site",{'short':'w','type':'string','metavar':'<web site>','default':'http://www.logilab.fr','help':'cube author\'s web site',}),)defrun(self,args):iflen(args)!=1:raiseBadCommandUsage("exactly one argument (cube name) is expected")cubename,=argsverbose=self.get('verbose')cubesdir=self.get('directory')ifnotcubesdir:cubespath=ServerConfiguration.cubes_search_path()iflen(cubespath)>1:raiseBadCommandUsage("can't guess directory where to put the new cube."" Please specify it using the --directory option")cubesdir=cubespath[0]ifnotisdir(cubesdir):print"-> creating cubes directory",cubesdirtry:mkdir(cubesdir)exceptOSError,err:self.fail("failed to create directory %r\n(%s)"%(cubesdir,err))cubedir=join(cubesdir,cubename)ifexists(cubedir):self.fail("%s already exists !"%(cubedir))skeldir=join(BASEDIR,'skeleton')default_name='cubicweb-%s'%cubename.lower()ifverbose:distname=raw_input('Debian name for your cube ? [%s]): '%default_name).strip()ifnotdistname:distname=default_nameelifnotdistname.startswith('cubicweb-'):ifASK.confirm('Do you mean cubicweb-%s ?'%distname):distname='cubicweb-'+distnameelse:distname=default_namelongdesc=shortdesc=raw_input('Enter a short description for your cube: ')ifverbose:longdesc=raw_input('Enter a long description (leave empty to reuse the short one): ')ifverbose:includes=self._ask_for_dependancies()iflen(includes)==1:dependancies='%r,'%includes[0]else:dependancies=', '.join(repr(cube)forcubeinincludes)else:dependancies=''context={'cubename':cubename,'distname':distname,'shortdesc':shortdesc,'longdesc':longdescorshortdesc,'dependancies':dependancies,'version':cubicwebversion,'year':str(datetime.now().year),'author':self['author'],'author-email':self['author-email'],'author-web-site':self['author-web-site'],}copy_skeleton(skeldir,cubedir,context)def_ask_for_dependancies(self):includes=[]forstdtypeinServerConfiguration.available_cubes():answer=ASK.ask("Depends on cube %s? "%stdtype,('N','y','skip','type'),'N')ifanswer=='y':includes.append(stdtype)ifanswer=='type':includes=splitstrip(raw_input('type dependancies: '))breakelifanswer=='skip':breakreturnincludesclassExamineLogCommand(Command):"""Examine a rql log file. usage: python exlog.py < rql.log will print out the following table total execution time || number of occurences || rql query sorted by descending total execution time chances are the lines at the top are the ones that will bring the higher benefit after optimisation. Start there. """name='exlog'options=()defrun(self,args):ifargs:raiseBadCommandUsage("no argument expected")importrerequests={}forlineno,lineinenumerate(sys.stdin):ifnot' WHERE 'inline:continue#sys.stderr.write( line )try:rql,time=line.split('--')rql=re.sub("(\'\w+': \d*)",'',rql)if'{'inrql:rql=rql[:rql.index('{')]req=requests.setdefault(rql,[])time.strip()chunks=time.split()clocktime=float(chunks[0][1:])cputime=float(chunks[-3])req.append((clocktime,cputime))exceptException,exc:sys.stderr.write('Line %s: %s (%s)\n'%(lineno,exc,line))stat=[]forrql,timesinrequests.items():stat.append((sum(time[0]fortimeintimes),sum(time[1]fortimeintimes),len(times),rql))stat.sort()stat.reverse()total_time=sum(clocktimeforclocktime,cputime,occ,rqlinstat)*0.01print'Percentage;Cumulative Time (clock);Cumulative Time (CPU);Occurences;Query'forclocktime,cputime,occ,rqlinstat:print'%.2f;%.2f;%.2f;%s;%s'%(clocktime/total_time,clocktime,cputime,occ,rql)register_commands((UpdateCubicWebCatalogCommand,UpdateTemplateCatalogCommand,LiveServerCommand,NewCubeCommand,ExamineLogCommand,))