"""additional cubicweb-ctl commands and command handlers for cubicweb and cubicweb'scubes development:organization: Logilab:copyright: 2001-2009 LOGILAB S.A. (Paris, FRANCE), all rights reserved.:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr"""__docformat__="restructuredtext en"importsysfromdatetimeimportdatetimefromosimportmkdir,chdirfromos.pathimportjoin,exists,abspath,basename,normpath,split,isdirfromlogilab.commonimportSTD_BLACKLISTfromlogilab.common.modutilsimportget_module_filesfromlogilab.common.textutilsimportget_csvfromlogilab.common.clcommandsimportregister_commandsfromcubicwebimportCW_SOFTWARE_ROOTasBASEDIR,BadCommandUsagefromcubicweb.__pkginfo__importversionascubicwebversionfromcubicweb.toolsutilsimportCommand,confirm,copy_skeletonfromcubicweb.web.webconfigimportWebConfigurationfromcubicweb.server.serverconfigimportServerConfigurationclassDevCubeConfiguration(ServerConfiguration,WebConfiguration):"""dummy config to get full library schema and entities"""creating=Truecubicweb_vobject_path=ServerConfiguration.cubicweb_vobject_path|WebConfiguration.cubicweb_vobject_pathcube_vobject_path=ServerConfiguration.cube_vobject_path|WebConfiguration.cube_vobject_pathdef__init__(self,cube):super(DevCubeConfiguration,self).__init__(cube)ifcubeisNone:self._cubes=()else:self._cubes=self.expand_cubes(self.my_cubes(cube))defmy_cubes(self,cube):return(cube,)+self.cube_dependencies(cube)+self.cube_recommends(cube)@propertydefapphome(self):returnNonedefmain_config_file(self):returnNonedefinit_log(self,debug=None):passdefload_configuration(self):passclassDevDepConfiguration(DevCubeConfiguration):"""configuration to use to generate cubicweb po files or to use as "library" configuration to filter out message ids from cubicweb and dependencies of a cube """defmy_cubes(self,cube):returnself.cube_dependencies(cube)+self.cube_recommends(cube)defdefault_log_file(self):returnNonedefcleanup_sys_modules(config):# cleanup sys.modules, required when we're updating multiple cubesforname,modinsys.modules.items():ifmodisNone:# duh ? logilab.common.os for instancedelsys.modules[name]continueifnothasattr(mod,'__file__'):continueforpathinconfig.vregistry_path():ifmod.__file__.startswith(path):delsys.modules[name]breakdefgenerate_schema_pot(w,cubedir=None):"""generate a pot file with schema specific i18n messages notice that relation definitions description and static vocabulary should be marked using '_' and extracted using xgettext """fromcubicweb.cwvregimportCubicWebRegistrycube=cubedirandsplit(cubedir)[-1]config=DevDepConfiguration(cube)cleanup_sys_modules(config)ifcubedir:libschema=config.load_schema()config=DevCubeConfiguration(cube)schema=config.load_schema()else:schema=config.load_schema()libschema=Noneconfig.cleanup_interface_sobjects=Falsevreg=CubicWebRegistry(config)# set_schema triggers objects registrationsvreg.set_schema(schema)w(DEFAULT_POT_HEAD)_generate_schema_pot(w,vreg,schema,libschema=libschema,cube=cube)def_generate_schema_pot(w,vreg,schema,libschema=None,cube=None):fromcubicweb.common.i18nimportadd_msgw('# schema pot file, generated on %s\n'%datetime.now().strftime('%Y-%m-%d %H:%M:%S'))w('# \n')w('# singular and plural forms for each entity type\n')w('\n')iflibschemaisnotNone:entities=[eforeinschema.entities()ifnoteinlibschema]else:entities=schema.entities()done=set()foreschemainsorted(entities):etype=eschema.typeadd_msg(w,etype)add_msg(w,'%s_plural'%etype)ifnoteschema.is_final():add_msg(w,'This %s'%etype)add_msg(w,'New %s'%etype)add_msg(w,'add a %s'%etype)add_msg(w,'remove this %s'%etype)ifeschema.descriptionandnoteschema.descriptionindone:done.add(eschema.description)add_msg(w,eschema.description)w('# subject and object forms for each relation type\n')w('# (no object form for final relation types)\n')w('\n')iflibschemaisnotNone:relations=[rforrinschema.relations()ifnotrinlibschema]else:relations=schema.relations()forrschemainsorted(set(relations)):rtype=rschema.typeadd_msg(w,rtype)done.add(rtype)ifnot(schema.rschema(rtype).is_final()orrschema.symetric):add_msg(w,'%s_object'%rtype)ifrschema.descriptionandrschema.descriptionnotindone:done.add(rschema.description)add_msg(w,rschema.description)w('# add related box generated message\n')w('\n')actionbox=vreg['boxes']['edit_box'][0]foreschemainschema.entities():ifeschema.is_final():continueforx,rschemasin(('subject',eschema.subject_relations()),('object',eschema.object_relations())):forrschemainrschemas:ifrschema.is_final():continueforteschemainrschema.targets(eschema,x):ifdefined_in_library(libschema,eschema,rschema,teschema,x):continueifactionbox.relation_mode(rschema.type,teschema.type,x)=='create':ifx=='subject':label='add %s%s%s%s'%(eschema,rschema,teschema,x)label2="creating %s (%s%%(linkto)s %s%s)"%(teschema,eschema,rschema,teschema)else:label='add %s%s%s%s'%(teschema,rschema,eschema,x)label2="creating %s (%s%s%s%%(linkto)s)"%(teschema,teschema,rschema,eschema)add_msg(w,label)add_msg(w,label2)cube=(cubeand'cubes.%s.'%cubeor'cubicweb.')done=set()forreg,objdictinvreg.items():forobjectsinobjdict.values():forobjinobjects:objid='%s_%s'%(reg,obj.id)ifobjidindone:continueifobj.__module__.startswith(cube)andobj.property_defs:add_msg(w,'%s_description'%objid)add_msg(w,objid)done.add(objid)defdefined_in_library(libschema,etype,rtype,tetype,x):"""return true if the given relation definition exists in cubicweb's library"""iflibschemaisNone:returnFalseifx=='subject':subjtype,objtype=etype,tetypeelse:subjtype,objtype=tetype,etypetry:returnlibschema.rschema(rtype).has_rdef(subjtype,objtype)exceptKeyError:returnFalseLANGS=('en','fr','es')I18NDIR=join(BASEDIR,'i18n')DEFAULT_POT_HEAD=r'''msgid ""msgstr """Project-Id-Version: cubicweb %s\n""PO-Revision-Date: 2008-03-28 18:14+0100\n""Last-Translator: Logilab Team <contact@logilab.fr>\n""Language-Team: fr <contact@logilab.fr>\n""MIME-Version: 1.0\n""Content-Type: text/plain; charset=UTF-8\n""Content-Transfer-Encoding: 8bit\n""Generated-By: cubicweb-devtools\n""Plural-Forms: nplurals=2; plural=(n > 1);\n"'''%cubicwebversionclassUpdateCubicWebCatalogCommand(Command):"""Update i18n catalogs for cubicweb library. It will regenerate cubicweb/i18n/xx.po files. You'll have then to edit those files to add translations of newly added messages. """name='i18nlibupdate'defrun(self,args):"""run the command with its specific arguments"""ifargs:raiseBadCommandUsage('Too much arguments')importshutilfromtempfileimportmktempimportyamsfromlogilab.common.fileutilsimportensure_fs_modefromlogilab.common.shellutilsimportglobfind,find,rmfromcubicweb.common.i18nimportextract_from_tal,executetempdir=mktemp()mkdir(tempdir)potfiles=[join(I18NDIR,'entities.pot')]print'******** extract schema messages'schemapot=join(tempdir,'schema.pot')potfiles.append(schemapot)# explicit close necessary else the file may not be yet flushed when# we'll using it belowschemapotstream=file(schemapot,'w')generate_schema_pot(schemapotstream.write,cubedir=None)schemapotstream.close()print'******** extract TAL messages'tali18nfile=join(tempdir,'tali18n.py')extract_from_tal(find(join(BASEDIR,'web'),('.py','.pt')),tali18nfile)print'******** .pot files generation'forid,files,langin[('pycubicweb',get_module_files(BASEDIR)+list(globfind(join(BASEDIR,'misc','migration'),'*.py')),None),('schemadescr',globfind(join(BASEDIR,'schemas'),'*.py'),None),('yams',get_module_files(yams.__path__[0]),None),('tal',[tali18nfile],None),('js',globfind(join(BASEDIR,'web'),'cub*.js'),'java'),]:cmd='xgettext --no-location --omit-header -k_ -o %s%s'iflangisnotNone:cmd+=' -L %s'%langpotfiles.append(join(tempdir,'%s.pot'%id))execute(cmd%(potfiles[-1],' '.join(files)))print'******** merging .pot files'cubicwebpot=join(tempdir,'cubicweb.pot')execute('msgcat %s > %s'%(' '.join(potfiles),cubicwebpot))print'******** merging main pot file with existing translations'chdir(I18NDIR)toedit=[]forlanginLANGS:target='%s.po'%langexecute('msgmerge -N --sort-output %s%s > %snew'%(target,cubicwebpot,target))ensure_fs_mode(target)shutil.move('%snew'%target,target)toedit.append(abspath(target))# cleanuprm(tempdir)# instructions pour la suiteprint'*'*72print'you can now edit the following files:'print'* '+'\n* '.join(toedit)printprint"then you'll have to update cubes catalogs using the i18nupdate command"classUpdateTemplateCatalogCommand(Command):"""Update i18n catalogs for cubes. If no cube is specified, update catalogs of all registered cubes. """name='i18nupdate'arguments='[<cube>...]'defrun(self,args):"""run the command with its specific arguments"""ifargs:cubes=[DevCubeConfiguration.cube_dir(cube)forcubeinargs]else:cubes=[DevCubeConfiguration.cube_dir(cube)forcubeinDevCubeConfiguration.available_cubes()]cubes=[cubepathforcubepathincubesifexists(join(cubepath,'i18n'))]update_cubes_catalogs(cubes)defupdate_cubes_catalogs(cubes):importshutilfromtempfileimportmktempfromlogilab.common.fileutilsimportensure_fs_modefromlogilab.common.shellutilsimportfind,rmfromcubicweb.common.i18nimportextract_from_tal,executetoedit=[]forcubedirincubes:cube=basename(normpath(cubedir))ifnotisdir(cubedir):print'unknown cube',cubecontinuetempdir=mktemp()mkdir(tempdir)print'*'*72print'updating %s cube...'%cubechdir(cubedir)potfiles=[join('i18n',scfile)forscfilein('entities.pot',)ifexists(join('i18n',scfile))]print'******** extract schema messages'schemapot=join(tempdir,'schema.pot')potfiles.append(schemapot)# explicit close necessary else the file may not be yet flushed when# we'll using it belowschemapotstream=file(schemapot,'w')generate_schema_pot(schemapotstream.write,cubedir)schemapotstream.close()print'******** extract TAL messages'tali18nfile=join(tempdir,'tali18n.py')extract_from_tal(find('.',('.py','.pt'),blacklist=STD_BLACKLIST+('test',)),tali18nfile)print'******** extract Javascript messages'jsfiles=[jsfileforjsfileinfind('.','.js')ifbasename(jsfile).startswith('cub')]ifjsfiles:tmppotfile=join(tempdir,'js.pot')execute('xgettext --no-location --omit-header -k_ -L java --from-code=utf-8 -o %s%s'%(tmppotfile,' '.join(jsfiles)))# no pot file created if there are no string to translateifexists(tmppotfile):potfiles.append(tmppotfile)print'******** create cube specific catalog'tmppotfile=join(tempdir,'generated.pot')cubefiles=find('.','.py',blacklist=STD_BLACKLIST+('test',))cubefiles.append(tali18nfile)execute('xgettext --no-location --omit-header -k_ -o %s%s'%(tmppotfile,' '.join(cubefiles)))ifexists(tmppotfile):# doesn't exists of no translation string foundpotfiles.append(tmppotfile)potfile=join(tempdir,'cube.pot')print'******** merging .pot files'execute('msgcat %s > %s'%(' '.join(potfiles),potfile))print'******** merging main pot file with existing translations'chdir('i18n')forlanginLANGS:print'****',langcubepo='%s.po'%langifnotexists(cubepo):shutil.copy(potfile,cubepo)else:execute('msgmerge -N -s %s%s > %snew'%(cubepo,potfile,cubepo))ensure_fs_mode(cubepo)shutil.move('%snew'%cubepo,cubepo)toedit.append(abspath(cubepo))# cleanuprm(tempdir)# instructions pour la suiteprint'*'*72print'you can now edit the following files:'print'* '+'\n* '.join(toedit)classLiveServerCommand(Command):"""Run a server from within a cube directory. """name='live-server'arguments=''options=()defrun(self,args):"""run the command with its specific arguments"""fromcubicweb.devtools.livetestimportrunserverrunserver()classNewCubeCommand(Command):"""Create a new cube. <cubename> the name of the new cube """name='newcube'arguments='<cubename>'options=(("directory",{'short':'d','type':'string','metavar':'<cubes directory>','help':'directory where the new cube should be created',}),("verbose",{'short':'v','type':'yn','metavar':'<verbose>','default':'n','help':'verbose mode: will ask all possible configuration questions',}),("author",{'short':'a','type':'string','metavar':'<author>','default':'LOGILAB S.A. (Paris, FRANCE)','help':'cube author',}),("author-email",{'short':'e','type':'string','metavar':'<email>','default':'contact@logilab.fr','help':'cube author\'s email',}),("author-web-site",{'short':'w','type':'string','metavar':'<web site>','default':'http://www.logilab.fr','help':'cube author\'s web site',}),)defrun(self,args):iflen(args)!=1:raiseBadCommandUsage("exactly one argument (cube name) is expected")cubename,=args#if ServerConfiguration.mode != "dev":# self.fail("you can only create new cubes in development mode")verbose=self.get('verbose')cubesdir=self.get('directory')ifnotcubesdir:cubespath=ServerConfiguration.cubes_search_path()iflen(cubespath)>1:raiseBadCommandUsage("can't guess directory where to put the new cube."" Please specify it using the --directory option")cubesdir=cubespath[0]ifnotisdir(cubesdir):print"creating cubes directory",cubesdirtry:mkdir(cubesdir)exceptOSError,err:self.fail("failed to create directory %r\n(%s)"%(cubesdir,err))cubedir=join(cubesdir,cubename)ifexists(cubedir):self.fail("%s already exists !"%(cubedir))skeldir=join(BASEDIR,'skeleton')ifverbose:distname=raw_input('Debian name for your cube (just type enter to use the cube name): ').strip()ifnotdistname:distname='cubicweb-%s'%cubename.lower()elifnotdistname.startswith('cubicweb-'):ifconfirm('do you mean cubicweb-%s ?'%distname):distname='cubicweb-'+distnameelse:distname='cubicweb-%s'%cubename.lower()longdesc=shortdesc=raw_input('Enter a short description for your cube: ')ifverbose:longdesc=raw_input('Enter a long description (or nothing if you want to reuse the short one): ')ifverbose:includes=self._ask_for_dependancies()iflen(includes)==1:dependancies='%r,'%includes[0]else:dependancies=', '.join(repr(cube)forcubeinincludes)else:dependancies=''context={'cubename':cubename,'distname':distname,'shortdesc':shortdesc,'longdesc':longdescorshortdesc,'dependancies':dependancies,'version':cubicwebversion,'year':str(datetime.now().year),'author':self['author'],'author-email':self['author-email'],'author-web-site':self['author-web-site'],}copy_skeleton(skeldir,cubedir,context)def_ask_for_dependancies(self):includes=[]forstdtypeinServerConfiguration.available_cubes():ans=raw_input("Depends on cube %s? (N/y/s(kip)/t(ype)"%stdtype).lower().strip()ifans=='y':includes.append(stdtype)ifans=='t':includes=get_csv(raw_input('type dependancies: '))breakelifans=='s':breakreturnincludesclassExamineLogCommand(Command):"""Examine a rql log file. usage: python exlog.py < rql.log will print out the following table total execution time || number of occurences || rql query sorted by descending total execution time chances are the lines at the top are the ones that will bring the higher benefit after optimisation. Start there. """name='exlog'options=()defrun(self,args):ifargs:raiseBadCommandUsage("no argument expected")importrerequests={}forlineno,lineinenumerate(sys.stdin):ifnot' WHERE 'inline:continue#sys.stderr.write( line )try:rql,time=line.split('--')rql=re.sub("(\'\w+': \d*)",'',rql)if'{'inrql:rql=rql[:rql.index('{')]req=requests.setdefault(rql,[])time.strip()chunks=time.split()cputime=float(chunks[-3])req.append(cputime)exceptException,exc:sys.stderr.write('Line %s: %s (%s)\n'%(lineno,exc,line))stat=[]forrql,timesinrequests.items():stat.append((sum(times),len(times),rql))stat.sort()stat.reverse()total_time=sum(timefortime,occ,rqlinstat)*0.01print'Percentage;Cumulative Time;Occurences;Query'fortime,occ,rqlinstat:print'%.2f;%.2f;%s;%s'%(time/total_time,time,occ,rql)register_commands((UpdateCubicWebCatalogCommand,UpdateTemplateCatalogCommand,LiveServerCommand,NewCubeCommand,ExamineLogCommand,))