1 # copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.

     2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr

     3 #

     4 # This file is part of CubicWeb.

     5 #

     6 # CubicWeb is free software: you can redistribute it and/or modify it under the

     7 # terms of the GNU Lesser General Public License as published by the Free

     8 # Software Foundation, either version 2.1 of the License, or (at your option)

     9 # any later version.

    10 #

    11 # CubicWeb is distributed in the hope that it will be useful, but WITHOUT

    12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

    13 # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more

    14 # details.

    15 #

    16 # You should have received a copy of the GNU Lesser General Public License along

    17 # with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.

    18 """additional cubicweb-ctl commands and command handlers for cubicweb and

    19 cubicweb's cubes development

    20 """

    21 from __future__ import print_function

    22 

    23 __docformat__ = "restructuredtext en"

    24 

    25 # *ctl module should limit the number of import to be imported as quickly as

    26 # possible (for cubicweb-ctl reactivity, necessary for instance for usable bash

    27 # completion). So import locally in command helpers.

    28 import sys

    29 from datetime import datetime, date

    30 from os import mkdir, chdir, path as osp

    31 from warnings import warn

    32 

    33 from six.moves import input

    34 

    35 from logilab.common import STD_BLACKLIST

    36 

    37 from cubicweb.__pkginfo__ import version as cubicwebversion

    38 from cubicweb import CW_SOFTWARE_ROOT as BASEDIR, BadCommandUsage, ExecutionError

    39 from cubicweb.cwctl import CWCTL

    40 from cubicweb.cwconfig import CubicWebNoAppConfiguration

    41 from cubicweb.toolsutils import (SKEL_EXCLUDE, Command, copy_skeleton,

    42                                  underline_title)

    43 from cubicweb.web.webconfig import WebConfiguration

    44 from cubicweb.server.serverconfig import ServerConfiguration

    45 

    46 

    47 class DevConfiguration(ServerConfiguration, WebConfiguration):

    48     """dummy config to get full library schema and appobjects for

    49     a cube or for cubicweb (without a home)

    50     """

    51     creating = True

    52     cleanup_unused_appobjects = False

    53 

    54     cubicweb_appobject_path = (ServerConfiguration.cubicweb_appobject_path

    55                                | WebConfiguration.cubicweb_appobject_path)

    56     cube_appobject_path = (ServerConfiguration.cube_appobject_path

    57                            | WebConfiguration.cube_appobject_path)

    58 

    59     def __init__(self, *cubes):

    60         super(DevConfiguration, self).__init__(cubes and cubes[0] or None)

    61         if cubes:

    62             self._cubes = self.reorder_cubes(

    63                 self.expand_cubes(cubes, with_recommends=True))

    64             self.load_site_cubicweb()

    65         else:

    66             self._cubes = ()

    67 

    68     @property

    69     def apphome(self):

    70         return None

    71 

    72     def available_languages(self):

    73         return self.cw_languages()

    74 

    75     def main_config_file(self):

    76         return None

    77     def init_log(self):

    78         pass

    79     def load_configuration(self, **kw):

    80         pass

    81     def default_log_file(self):

    82         return None

    83     def default_stats_file(self):

    84         return None

    85 

    86 

    87 def cleanup_sys_modules(config):

    88     # cleanup sys.modules, required when we're updating multiple cubes

    89     for name, mod in list(sys.modules.items()):

    90         if mod is None:

    91             # duh ? logilab.common.os for instance

    92             del sys.modules[name]

    93             continue

    94         if not hasattr(mod, '__file__'):

    95             continue

    96         if mod.__file__ is None:

    97             # odd/rare but real

    98             continue

    99         for path in config.appobjects_path():

   100             if mod.__file__.startswith(path):

   101                 del sys.modules[name]

   102                 break

   103 

   104 def generate_schema_pot(w, cubedir=None):

   105     """generate a pot file with schema specific i18n messages

   106 

   107     notice that relation definitions description and static vocabulary

   108     should be marked using '_' and extracted using xgettext

   109     """

   110     from cubicweb.cwvreg import CWRegistryStore

   111     if cubedir:

   112         cube = osp.split(cubedir)[-1]

   113         config = DevConfiguration(cube)

   114         depcubes = list(config._cubes)

   115         depcubes.remove(cube)

   116         libconfig = DevConfiguration(*depcubes)

   117     else:

   118         config = DevConfiguration()

   119         cube = libconfig = None

   120     cleanup_sys_modules(config)

   121     schema = config.load_schema(remove_unused_rtypes=False)

   122     vreg = CWRegistryStore(config)

   123     # set_schema triggers objects registrations

   124     vreg.set_schema(schema)

   125     w(DEFAULT_POT_HEAD)

   126     _generate_schema_pot(w, vreg, schema, libconfig=libconfig)

   127 

   128 

   129 def _generate_schema_pot(w, vreg, schema, libconfig=None):

   130     from cubicweb.i18n import add_msg

   131     from cubicweb.schema import NO_I18NCONTEXT, CONSTRAINTS

   132     w('# schema pot file, generated on %s\n'

   133       % datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'))

   134     w('# \n')

   135     w('# singular and plural forms for each entity type\n')

   136     w('\n')

   137     vregdone = set()

   138     afss = vreg['uicfg']['autoform_section']

   139     aiams = vreg['uicfg']['actionbox_appearsin_addmenu']

   140     if libconfig is not None:

   141         # processing a cube, libconfig being a config with all its dependencies

   142         # (cubicweb incl.)

   143         from cubicweb.cwvreg import CWRegistryStore

   144         libschema = libconfig.load_schema(remove_unused_rtypes=False)

   145         cleanup_sys_modules(libconfig)

   146         libvreg = CWRegistryStore(libconfig)

   147         libvreg.set_schema(libschema) # trigger objects registration

   148         libafss = libvreg['uicfg']['autoform_section']

   149         libaiams = libvreg['uicfg']['actionbox_appearsin_addmenu']

   150         # prefill vregdone set

   151         list(_iter_vreg_objids(libvreg, vregdone))

   152 

   153         def is_in_lib(rtags, eschema, rschema, role, tschema, predicate=bool):

   154             return any(predicate(rtag.etype_get(eschema, rschema, role, tschema))

   155                        for rtag in rtags)

   156     else:

   157         # processing cubicweb itself

   158         libschema = {}

   159         for cstrtype in CONSTRAINTS:

   160             add_msg(w, cstrtype)

   161         libafss = libaiams = None

   162         is_in_lib = lambda *args: False

   163     done = set()

   164     for eschema in sorted(schema.entities()):

   165         if eschema.type in libschema:

   166             done.add(eschema.description)

   167     for eschema in sorted(schema.entities()):

   168         etype = eschema.type

   169         if etype not in libschema:

   170             add_msg(w, etype)

   171             add_msg(w, '%s_plural' % etype)

   172             if not eschema.final:

   173                 add_msg(w, 'This %s:' % etype)

   174                 add_msg(w, 'New %s' % etype)

   175                 add_msg(w, 'add a %s' % etype) # AddNewAction

   176                 if libconfig is not None:  # processing a cube

   177                     # As of 3.20.3 we no longer use it, but keeping this string

   178                     # allows developers to run i18ncube with new cubicweb and still

   179                     # have the right translations at runtime for older versions

   180                     add_msg(w, 'This %s' % etype)

   181             if eschema.description and not eschema.description in done:

   182                 done.add(eschema.description)

   183                 add_msg(w, eschema.description)

   184         if eschema.final:

   185             continue

   186         for rschema, targetschemas, role in eschema.relation_definitions(True):

   187             if rschema.final:

   188                 continue

   189             for tschema in targetschemas:

   190 

   191                 for afs in afss:

   192                     fsections = afs.etype_get(eschema, rschema, role, tschema)

   193                     if 'main_inlined' in fsections and not \

   194                             is_in_lib(libafss, eschema, rschema, role, tschema,

   195                                       lambda x: 'main_inlined' in x):

   196                         add_msg(w, 'add a %s' % tschema,

   197                                 'inlined:%s.%s.%s' % (etype, rschema, role))

   198                         add_msg(w, str(tschema),

   199                                 'inlined:%s.%s.%s' % (etype, rschema, role))

   200                         break

   201 

   202                 for aiam in aiams:

   203                     if aiam.etype_get(eschema, rschema, role, tschema) and not \

   204                             is_in_lib(libaiams, eschema, rschema, role, tschema):

   205                         if role == 'subject':

   206                             label = 'add %s %s %s %s' % (eschema, rschema,

   207                                                          tschema, role)

   208                             label2 = "creating %s (%s %%(linkto)s %s %s)" % (

   209                                 tschema, eschema, rschema, tschema)

   210                         else:

   211                             label = 'add %s %s %s %s' % (tschema, rschema,

   212                                                          eschema, role)

   213                             label2 = "creating %s (%s %s %s %%(linkto)s)" % (

   214                                 tschema, tschema, rschema, eschema)

   215                         add_msg(w, label)

   216                         add_msg(w, label2)

   217                         break

   218             # XXX also generate "creating ...' messages for actions in the

   219             # addrelated submenu

   220     w('# subject and object forms for each relation type\n')

   221     w('# (no object form for final or symmetric relation types)\n')

   222     w('\n')

   223     for rschema in sorted(schema.relations()):

   224         if rschema.type in libschema:

   225             done.add(rschema.type)

   226             done.add(rschema.description)

   227     for rschema in sorted(schema.relations()):

   228         rtype = rschema.type

   229         if rtype not in libschema:

   230             # bw compat, necessary until all translation of relation are done

   231             # properly...

   232             add_msg(w, rtype)

   233             done.add(rtype)

   234             if rschema.description and rschema.description not in done:

   235                 add_msg(w, rschema.description)

   236             done.add(rschema.description)

   237             librschema = None

   238         else:

   239             librschema = libschema.rschema(rtype)

   240         # add context information only for non-metadata rtypes

   241         if rschema not in NO_I18NCONTEXT:

   242             libsubjects = librschema and librschema.subjects() or ()

   243             for subjschema in rschema.subjects():

   244                 if not subjschema in libsubjects:

   245                     add_msg(w, rtype, subjschema.type)

   246         if not (rschema.final or rschema.symmetric):

   247             if rschema not in NO_I18NCONTEXT:

   248                 libobjects = librschema and librschema.objects() or ()

   249                 for objschema in rschema.objects():

   250                     if not objschema in libobjects:

   251                         add_msg(w, '%s_object' % rtype, objschema.type)

   252             if rtype not in libschema:

   253                 # bw compat, necessary until all translation of relation are

   254                 # done properly...

   255                 add_msg(w, '%s_object' % rtype)

   256         for rdef in rschema.rdefs.values():

   257             if not rdef.description or rdef.description in done:

   258                 continue

   259             if (librschema is None or

   260                 (rdef.subject, rdef.object) not in librschema.rdefs or

   261                 librschema.rdefs[(rdef.subject, rdef.object)].description != rdef.description):

   262                 add_msg(w, rdef.description)

   263             done.add(rdef.description)

   264     for objid in _iter_vreg_objids(vreg, vregdone):

   265         add_msg(w, '%s_description' % objid)

   266         add_msg(w, objid)

   267 

   268 

   269 def _iter_vreg_objids(vreg, done):

   270     for reg, objdict in vreg.items():

   271         if reg in ('boxes', 'contentnavigation'):

   272             continue

   273         for objects in objdict.values():

   274             for obj in objects:

   275                 objid = '%s_%s' % (reg, obj.__regid__)

   276                 if objid in done:

   277                     break

   278                 pdefs = getattr(obj, 'cw_property_defs', {})

   279                 if pdefs:

   280                     yield objid

   281                     done.add(objid)

   282                     break

   283 

   284 

   285 DEFAULT_POT_HEAD = r'''msgid ""

   286 msgstr ""

   287 "Project-Id-Version: cubicweb %s\n"

   288 "PO-Revision-Date: 2008-03-28 18:14+0100\n"

   289 "Last-Translator: Logilab Team <contact@logilab.fr>\n"

   290 "Language-Team: fr <contact@logilab.fr>\n"

   291 "MIME-Version: 1.0\n"

   292 "Content-Type: text/plain; charset=UTF-8\n"

   293 "Content-Transfer-Encoding: 8bit\n"

   294 "Generated-By: cubicweb-devtools\n"

   295 "Plural-Forms: nplurals=2; plural=(n > 1);\n"

   296 

   297 ''' % cubicwebversion

   298 

   299 

   300 class UpdateCubicWebCatalogCommand(Command):

   301     """Update i18n catalogs for cubicweb library.

   302 

   303     It will regenerate cubicweb/i18n/xx.po files. You'll have then to edit those

   304     files to add translations of newly added messages.

   305     """

   306     name = 'i18ncubicweb'

   307     min_args = max_args = 0

   308 

   309     def run(self, args):

   310         """run the command with its specific arguments"""

   311         import shutil

   312         import tempfile

   313         import yams

   314         from logilab.common.fileutils import ensure_fs_mode

   315         from logilab.common.shellutils import globfind, find, rm

   316         from logilab.common.modutils import get_module_files

   317         from cubicweb.i18n import extract_from_tal, execute2

   318         tempdir = tempfile.mkdtemp(prefix='cw-')

   319         cwi18ndir = WebConfiguration.i18n_lib_dir()

   320         print('-> extract messages:', end=' ')

   321         print('schema', end=' ')

   322         schemapot = osp.join(tempdir, 'schema.pot')

   323         potfiles = [schemapot]

   324         potfiles.append(schemapot)

   325         # explicit close necessary else the file may not be yet flushed when

   326         # we'll using it below

   327         schemapotstream = open(schemapot, 'w')

   328         generate_schema_pot(schemapotstream.write, cubedir=None)

   329         schemapotstream.close()

   330         print('TAL', end=' ')

   331         tali18nfile = osp.join(tempdir, 'tali18n.py')

   332         extract_from_tal(find(osp.join(BASEDIR, 'web'), ('.py', '.pt')),

   333                          tali18nfile)

   334         print('-> generate .pot files.')

   335         pyfiles = get_module_files(BASEDIR)

   336         pyfiles += globfind(osp.join(BASEDIR, 'misc', 'migration'), '*.py')

   337         schemafiles = globfind(osp.join(BASEDIR, 'schemas'), '*.py')

   338         jsfiles = globfind(osp.join(BASEDIR, 'web'), 'cub*.js')

   339         for id, files, lang in [('pycubicweb', pyfiles, None),

   340                                 ('schemadescr', schemafiles, None),

   341                                 ('yams', get_module_files(yams.__path__[0]), None),

   342                                 ('tal', [tali18nfile], None),

   343                                 ('js', jsfiles, 'java'),

   344                                 ]:

   345             potfile = osp.join(tempdir, '%s.pot' % id)

   346             cmd = ['xgettext', '--no-location', '--omit-header', '-k_']

   347             if lang is not None:

   348                 cmd.extend(['-L', lang])

   349             cmd.extend(['-o', potfile])

   350             cmd.extend(files)

   351             execute2(cmd)

   352             if osp.exists(potfile):

   353                 potfiles.append(potfile)

   354             else:

   355                 print('-> WARNING: %s file was not generated' % potfile)

   356         print('-> merging %i .pot files' % len(potfiles))

   357         cubicwebpot = osp.join(tempdir, 'cubicweb.pot')

   358         cmd = ['msgcat', '-o', cubicwebpot] + potfiles

   359         execute2(cmd)

   360         print('-> merging main pot file with existing translations.')

   361         chdir(cwi18ndir)

   362         toedit = []

   363         for lang in CubicWebNoAppConfiguration.cw_languages():

   364             target = '%s.po' % lang

   365             cmd = ['msgmerge', '-N', '--sort-output', '-o',

   366                    target+'new', target, cubicwebpot]

   367             execute2(cmd)

   368             ensure_fs_mode(target)

   369             shutil.move('%snew' % target, target)

   370             toedit.append(osp.abspath(target))

   371         # cleanup

   372         rm(tempdir)

   373         # instructions pour la suite

   374         print('-> regenerated CubicWeb\'s .po catalogs.')

   375         print('\nYou can now edit the following files:')

   376         print('* ' + '\n* '.join(toedit))

   377         print('when you are done, run "cubicweb-ctl i18ncube yourcube".')

   378 

   379 

   380 class UpdateCubeCatalogCommand(Command):

   381     """Update i18n catalogs for cubes. If no cube is specified, update

   382     catalogs of all registered cubes.

   383     """

   384     name = 'i18ncube'

   385     arguments = '[<cube>...]'

   386 

   387     def run(self, args):

   388         """run the command with its specific arguments"""

   389         if args:

   390             cubes = [DevConfiguration.cube_dir(cube) for cube in args]

   391         else:

   392             cubes = [DevConfiguration.cube_dir(cube)

   393                      for cube in DevConfiguration.available_cubes()]

   394             cubes = [cubepath for cubepath in cubes

   395                      if osp.exists(osp.join(cubepath, 'i18n'))]

   396         if not update_cubes_catalogs(cubes):

   397             raise ExecutionError("update cubes i18n catalog failed")

   398 

   399 

   400 def update_cubes_catalogs(cubes):

   401     from subprocess import CalledProcessError

   402     for cubedir in cubes:

   403         if not osp.isdir(cubedir):

   404             print('-> ignoring %s that is not a directory.' % cubedir)

   405             continue

   406         try:

   407             toedit = update_cube_catalogs(cubedir)

   408         except CalledProcessError as exc:

   409             print('\n*** error while updating catalogs for cube', cubedir)

   410             print('cmd:\n%s' % exc.cmd)

   411             print('stdout:\n%s\nstderr:\n%s' % exc.data)

   412         except Exception:

   413             import traceback

   414             traceback.print_exc()

   415             print('*** error while updating catalogs for cube', cubedir)

   416             return False

   417         else:

   418             # instructions pour la suite

   419             if toedit:

   420                 print('-> regenerated .po catalogs for cube %s.' % cubedir)

   421                 print('\nYou can now edit the following files:')

   422                 print('* ' + '\n* '.join(toedit))

   423                 print ('When you are done, run "cubicweb-ctl i18ninstance '

   424                        '<yourinstance>" to see changes in your instances.')

   425             return True

   426 

   427 def update_cube_catalogs(cubedir):

   428     import shutil

   429     import tempfile

   430     from logilab.common.fileutils import ensure_fs_mode

   431     from logilab.common.shellutils import find, rm

   432     from cubicweb.i18n import extract_from_tal, execute2

   433     cube = osp.basename(osp.normpath(cubedir))

   434     tempdir = tempfile.mkdtemp()

   435     print(underline_title('Updating i18n catalogs for cube %s' % cube))

   436     chdir(cubedir)

   437     if osp.exists(osp.join('i18n', 'entities.pot')):

   438         warn('entities.pot is deprecated, rename file to static-messages.pot (%s)'

   439              % osp.join('i18n', 'entities.pot'), DeprecationWarning)

   440         potfiles = [osp.join('i18n', 'entities.pot')]

   441     elif osp.exists(osp.join('i18n', 'static-messages.pot')):

   442         potfiles = [osp.join('i18n', 'static-messages.pot')]

   443     else:

   444         potfiles = []

   445     print('-> extracting messages:', end=' ')

   446     print('schema', end=' ')

   447     schemapot = osp.join(tempdir, 'schema.pot')

   448     potfiles.append(schemapot)

   449     # explicit close necessary else the file may not be yet flushed when

   450     # we'll using it below

   451     schemapotstream = open(schemapot, 'w')

   452     generate_schema_pot(schemapotstream.write, cubedir)

   453     schemapotstream.close()

   454     print('TAL', end=' ')

   455     tali18nfile = osp.join(tempdir, 'tali18n.py')

   456     ptfiles = find('.', ('.py', '.pt'), blacklist=STD_BLACKLIST+('test',))

   457     extract_from_tal(ptfiles, tali18nfile)

   458     print('Javascript')

   459     jsfiles =  [jsfile for jsfile in find('.', '.js')

   460                 if osp.basename(jsfile).startswith('cub')]

   461     if jsfiles:

   462         tmppotfile = osp.join(tempdir, 'js.pot')

   463         cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-L', 'java',

   464                '--from-code=utf-8', '-o', tmppotfile] + jsfiles

   465         execute2(cmd)

   466         # no pot file created if there are no string to translate

   467         if osp.exists(tmppotfile):

   468             potfiles.append(tmppotfile)

   469     print('-> creating cube-specific catalog')

   470     tmppotfile = osp.join(tempdir, 'generated.pot')

   471     cubefiles = find('.', '.py', blacklist=STD_BLACKLIST+('test',))

   472     cubefiles.append(tali18nfile)

   473     cmd = ['xgettext', '--no-location', '--omit-header', '-k_', '-o', tmppotfile]

   474     cmd.extend(cubefiles)

   475     execute2(cmd)

   476     if osp.exists(tmppotfile): # doesn't exists of no translation string found

   477         potfiles.append(tmppotfile)

   478     potfile = osp.join(tempdir, 'cube.pot')

   479     print('-> merging %i .pot files' % len(potfiles))

   480     cmd = ['msgcat', '-o', potfile]

   481     cmd.extend(potfiles)

   482     execute2(cmd)

   483     if not osp.exists(potfile):

   484         print('no message catalog for cube', cube, 'nothing to translate')

   485         # cleanup

   486         rm(tempdir)

   487         return ()

   488     print('-> merging main pot file with existing translations:', end=' ')

   489     chdir('i18n')

   490     toedit = []

   491     for lang in CubicWebNoAppConfiguration.cw_languages():

   492         print(lang, end=' ')

   493         cubepo = '%s.po' % lang

   494         if not osp.exists(cubepo):

   495             shutil.copy(potfile, cubepo)

   496         else:

   497             cmd = ['msgmerge','-N','-s','-o', cubepo+'new', cubepo, potfile]

   498             execute2(cmd)

   499             ensure_fs_mode(cubepo)

   500             shutil.move('%snew' % cubepo, cubepo)

   501         toedit.append(osp.abspath(cubepo))

   502     print()

   503     # cleanup

   504     rm(tempdir)

   505     return toedit

   506 

   507 

   508 # XXX totally broken, fix it

   509 # class LiveServerCommand(Command):

   510 #     """Run a server from within a cube directory.

   511 #     """

   512 #     name = 'live-server'

   513 #     arguments = ''

   514 #     options = ()

   515 

   516 #     def run(self, args):

   517 #         """run the command with its specific arguments"""

   518 #         from cubicweb.devtools.livetest import runserver

   519 #         runserver()

   520 

   521 

   522 class NewCubeCommand(Command):

   523     """Create a new cube.

   524 

   525     <cubename>

   526       the name of the new cube. It should be a valid python module name.

   527     """

   528     name = 'newcube'

   529     arguments = '<cubename>'

   530     min_args = max_args = 1

   531     options = (

   532         ("layout",

   533          {'short': 'L', 'type' : 'choice', 'metavar': '<cube layout>',

   534           'default': 'simple', 'choices': ('simple', 'full'),

   535           'help': 'cube layout. You\'ll get a minimal cube with the "simple" \

   536 layout, and a full featured cube with "full" layout.',

   537           }

   538          ),

   539         ("directory",

   540          {'short': 'd', 'type' : 'string', 'metavar': '<cubes directory>',

   541           'help': 'directory where the new cube should be created',

   542           }

   543          ),

   544         ("verbose",

   545          {'short': 'v', 'type' : 'yn', 'metavar': '<verbose>',

   546           'default': 'n',

   547           'help': 'verbose mode: will ask all possible configuration questions',

   548           }

   549          ),

   550         ("author",

   551          {'short': 'a', 'type' : 'string', 'metavar': '<author>',

   552           'default': 'LOGILAB S.A. (Paris, FRANCE)',

   553           'help': 'cube author',

   554           }

   555          ),

   556         ("author-email",

   557          {'short': 'e', 'type' : 'string', 'metavar': '<email>',

   558           'default': 'contact@logilab.fr',

   559           'help': 'cube author\'s email',

   560           }

   561          ),

   562         ("author-web-site",

   563          {'short': 'w', 'type' : 'string', 'metavar': '<web site>',

   564           'default': 'http://www.logilab.fr',

   565           'help': 'cube author\'s web site',

   566           }

   567          ),

   568         ("license",

   569          {'short': 'l', 'type' : 'choice', 'metavar': '<license>',

   570           'default': 'LGPL', 'choices': ('GPL', 'LGPL', ''),

   571           'help': 'cube license',

   572           }

   573          ),

   574         )

   575 

   576     LICENSES = {

   577         'LGPL': '''\

   578 # This program is free software: you can redistribute it and/or modify it under

   579 # the terms of the GNU Lesser General Public License as published by the Free

   580 # Software Foundation, either version 2.1 of the License, or (at your option)

   581 # any later version.

   582 #

   583 # This program is distributed in the hope that it will be useful, but WITHOUT

   584 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

   585 # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more

   586 # details.

   587 #

   588 # You should have received a copy of the GNU Lesser General Public License

   589 # along with this program. If not, see <http://www.gnu.org/licenses/>.

   590 ''',

   591 

   592         'GPL': '''\

   593 # This program is free software: you can redistribute it and/or modify it under

   594 # the terms of the GNU General Public License as published by the Free Software

   595 # Foundation, either version 2.1 of the License, or (at your option) any later

   596 # version.

   597 #

   598 # This program is distributed in the hope that it will be useful, but WITHOUT

   599 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

   600 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more

   601 # details.

   602 #

   603 # You should have received a copy of the GNU General Public License along with

   604 # this program. If not, see <http://www.gnu.org/licenses/>.

   605 ''',

   606         '': '# INSERT LICENSE HERE'

   607         }

   608 

   609     def run(self, args):

   610         import re

   611         from logilab.common.shellutils import ASK

   612         cubename = args[0]

   613         if not re.match('[_A-Za-z][_A-Za-z0-9]*$', cubename):

   614             raise BadCommandUsage(

   615                 'cube name must be a valid python module name')

   616         verbose = self.get('verbose')

   617         cubesdir = self.get('directory')

   618         if not cubesdir:

   619             cubespath = ServerConfiguration.cubes_search_path()

   620             if len(cubespath) > 1:

   621                 raise BadCommandUsage(

   622                     "can't guess directory where to put the new cube."

   623                     " Please specify it using the --directory option")

   624             cubesdir = cubespath[0]

   625         if not osp.isdir(cubesdir):

   626             print("-> creating cubes directory", cubesdir)

   627             try:

   628                 mkdir(cubesdir)

   629             except OSError as err:

   630                 self.fail("failed to create directory %r\n(%s)"

   631                           % (cubesdir, err))

   632         cubedir = osp.join(cubesdir, cubename)

   633         if osp.exists(cubedir):

   634             self.fail("%s already exists!" % cubedir)

   635         skeldir = osp.join(BASEDIR, 'skeleton')

   636         default_name = 'cubicweb-%s' % cubename.lower().replace('_', '-')

   637         if verbose:

   638             distname = input('Debian name for your cube ? [%s]): '

   639                                  % default_name).strip()

   640             if not distname:

   641                 distname = default_name

   642             elif not distname.startswith('cubicweb-'):

   643                 if ASK.confirm('Do you mean cubicweb-%s ?' % distname):

   644                     distname = 'cubicweb-' + distname

   645         else:

   646             distname = default_name

   647         if not re.match('[a-z][-a-z0-9]*$', distname):

   648             raise BadCommandUsage(

   649                 'cube distname should be a valid debian package name')

   650         longdesc = shortdesc = input(

   651             'Enter a short description for your cube: ')

   652         if verbose:

   653             longdesc = input(

   654                 'Enter a long description (leave empty to reuse the short one): ')

   655         dependencies = {'cubicweb': '>= %s' % cubicwebversion,

   656                         'six': '>= 1.4.0',}

   657         if verbose:

   658             dependencies.update(self._ask_for_dependencies())

   659         context = {'cubename' : cubename,

   660                    'distname' : distname,

   661                    'shortdesc' : shortdesc,

   662                    'longdesc' : longdesc or shortdesc,

   663                    'dependencies' : dependencies,

   664                    'version'  : cubicwebversion,

   665                    'year'  : str(date.today().year),

   666                    'author': self['author'],

   667                    'author-email': self['author-email'],

   668                    'author-web-site': self['author-web-site'],

   669                    'license': self['license'],

   670                    'long-license': self.LICENSES[self['license']],

   671                    }

   672         exclude = SKEL_EXCLUDE

   673         if self['layout'] == 'simple':

   674             exclude += ('sobjects.py*', 'precreate.py*', 'realdb_test*',

   675                         'cubes.*', 'uiprops.py*')

   676         copy_skeleton(skeldir, cubedir, context, exclude=exclude)

   677 

   678     def _ask_for_dependencies(self):

   679         from logilab.common.shellutils import ASK

   680         from logilab.common.textutils import splitstrip

   681         depcubes = []

   682         for cube in ServerConfiguration.available_cubes():

   683             answer = ASK.ask("Depends on cube %s? " % cube,

   684                              ('N','y','skip','type'), 'N')

   685             if answer == 'y':

   686                 depcubes.append(cube)

   687             if answer == 'type':

   688                 depcubes = splitstrip(input('type dependencies: '))

   689                 break

   690             elif answer == 'skip':

   691                 break

   692         return dict(('cubicweb-' + cube, ServerConfiguration.cube_version(cube))

   693                     for cube in depcubes)

   694 

   695 

   696 class ExamineLogCommand(Command):

   697     """Examine a rql log file.

   698 

   699     Will print out the following table

   700 

   701       Percentage; Cumulative Time (clock); Cumulative Time (CPU); Occurences; Query

   702 

   703     sorted by descending cumulative time (clock). Time are expressed in seconds.

   704 

   705     Chances are the lines at the top are the ones that will bring the higher

   706     benefit after optimisation. Start there.

   707     """

   708     arguments = 'rql.log'

   709     name = 'exlog'

   710     options = ()

   711 

   712     def run(self, args):

   713         import re

   714         requests = {}

   715         for filepath in args:

   716             try:

   717                 stream = open(filepath)

   718             except OSError as ex:

   719                 raise BadCommandUsage("can't open rql log file %s: %s"

   720                                       % (filepath, ex))

   721             for lineno, line in enumerate(stream):

   722                 if not ' WHERE ' in line:

   723                     continue

   724                 try:

   725                     rql, time = line.split('--')

   726                     rql = re.sub("(\'\w+': \d*)", '', rql)

   727                     if '{' in rql:

   728                         rql = rql[:rql.index('{')]

   729                     req = requests.setdefault(rql, [])

   730                     time.strip()

   731                     chunks = time.split()

   732                     clocktime = float(chunks[0][1:])

   733                     cputime = float(chunks[-3])

   734                     req.append( (clocktime, cputime) )

   735                 except Exception as exc:

   736                     sys.stderr.write('Line %s: %s (%s)\n' % (lineno, exc, line))

   737         stat = []

   738         for rql, times in requests.items():

   739             stat.append( (sum(time[0] for time in times),

   740                           sum(time[1] for time in times),

   741                           len(times), rql) )

   742         stat.sort()

   743         stat.reverse()

   744         total_time = sum(clocktime for clocktime, cputime, occ, rql in stat) * 0.01

   745         print('Percentage;Cumulative Time (clock);Cumulative Time (CPU);Occurences;Query')

   746         for clocktime, cputime, occ, rql in stat:

   747             print('%.2f;%.2f;%.2f;%s;%s' % (clocktime/total_time, clocktime,

   748                                             cputime, occ, rql))

   749 

   750 

   751 class GenerateSchema(Command):

   752     """Generate schema image for the given cube"""

   753     name = "schema"

   754     arguments = '<cube>'

   755     min_args = max_args = 1

   756     options = [

   757         ('output-file',

   758          {'type':'string', 'default': None,

   759           'metavar': '<file>', 'short':'o', 'help':'output image file',

   760           'input':False,

   761           }),

   762         ('viewer',

   763          {'type': 'string', 'default':None,

   764           'short': "d", 'metavar':'<cmd>',

   765           'help':'command use to view the generated file (empty for none)',

   766           }),

   767         ('show-meta',

   768          {'action': 'store_true', 'default':False,

   769           'short': "m", 'metavar': "<yN>",

   770           'help':'include meta and internal entities in schema',

   771           }),

   772         ('show-workflow',

   773          {'action': 'store_true', 'default':False,

   774           'short': "w", 'metavar': "<yN>",

   775           'help':'include workflow entities in schema',

   776           }),

   777         ('show-cw-user',

   778          {'action': 'store_true', 'default':False,

   779           'metavar': "<yN>",

   780           'help':'include cubicweb user entities in schema',

   781           }),

   782         ('exclude-type',

   783          {'type':'string', 'default':'',

   784           'short': "x", 'metavar': "<types>",

   785           'help':'coma separated list of entity types to remove from view',

   786           }),

   787         ('include-type',

   788          {'type':'string', 'default':'',

   789           'short': "i", 'metavar': "<types>",

   790           'help':'coma separated list of entity types to include in view',

   791           }),

   792         ('show-etype',

   793          {'type':'string', 'default':'',

   794           'metavar': '<etype>',

   795           'help':'show graph of this etype and its neighbours'

   796           }),

   797         ]

   798 

   799     def run(self, args):

   800         from subprocess import Popen

   801         from tempfile import NamedTemporaryFile

   802         from logilab.common.textutils import splitstrip

   803         from logilab.common.graph import GraphGenerator, DotBackend

   804         from yams import schema2dot as s2d, BASE_TYPES

   805         from cubicweb.schema import (META_RTYPES, SCHEMA_TYPES, SYSTEM_RTYPES,

   806                                      WORKFLOW_TYPES, INTERNAL_TYPES)

   807         cubes = splitstrip(args[0])

   808         dev_conf = DevConfiguration(*cubes)

   809         schema = dev_conf.load_schema()

   810         out, viewer = self['output-file'], self['viewer']

   811         if out is None:

   812             tmp_file = NamedTemporaryFile(suffix=".svg")

   813             out = tmp_file.name

   814         skiptypes = BASE_TYPES | SCHEMA_TYPES

   815         if not self['show-meta']:

   816             skiptypes |=  META_RTYPES | SYSTEM_RTYPES | INTERNAL_TYPES

   817         if not self['show-workflow']:

   818             skiptypes |= WORKFLOW_TYPES

   819         if not self['show-cw-user']:

   820             skiptypes |= set(('CWUser', 'CWGroup', 'EmailAddress'))

   821         skiptypes |= set(self['exclude-type'].split(','))

   822         skiptypes -= set(self['include-type'].split(','))

   823 

   824         if not self['show-etype']:

   825             s2d.schema2dot(schema, out, skiptypes=skiptypes)

   826         else:

   827             etype = self['show-etype']

   828             visitor = s2d.OneHopESchemaVisitor(schema[etype], skiptypes=skiptypes)

   829             propshdlr = s2d.SchemaDotPropsHandler(visitor)

   830             backend = DotBackend('schema', 'BT',

   831                                  ratio='compress',size=None,

   832                                  renderer='dot',

   833                                  additionnal_param={'overlap' : 'false',

   834                                                     'splines' : 'true',

   835                                                     'sep' : '0.2'})

   836             generator = s2d.GraphGenerator(backend)

   837             generator.generate(visitor, propshdlr, out)

   838 

   839         if viewer:

   840             p = Popen((viewer, out))

   841             p.wait()

   842 

   843 

   844 for cmdcls in (UpdateCubicWebCatalogCommand,

   845                UpdateCubeCatalogCommand,

   846                #LiveServerCommand,

   847                NewCubeCommand,

   848                ExamineLogCommand,

   849                GenerateSchema,

   850                ):

   851     CWCTL.register(cmdcls)