1 """Internationalization and localization support.

     2 

     3 This module provides internationalization (I18N) and localization (L10N)

     4 support for your Python programs by providing an interface to the GNU gettext

     5 message catalog library.

     6 

     7 I18N refers to the operation by which a program is made aware of multiple

     8 languages.  L10N refers to the adaptation of your program, once

     9 internationalized, to the local language and cultural habits.

    10 

    11 """

    12 

    13 # This module represents the integration of work, contributions, feedback, and

    14 # suggestions from the following people:

    15 #

    16 # Martin von Loewis, who wrote the initial implementation of the underlying

    17 # C-based libintlmodule (later renamed _gettext), along with a skeletal

    18 # gettext.py implementation.

    19 #

    20 # Peter Funk, who wrote fintl.py, a fairly complete wrapper around intlmodule,

    21 # which also included a pure-Python implementation to read .mo files if

    22 # intlmodule wasn't available.

    23 #

    24 # James Henstridge, who also wrote a gettext.py module, which has some

    25 # interesting, but currently unsupported experimental features: the notion of

    26 # a Catalog class and instances, and the ability to add to a catalog file via

    27 # a Python API.

    28 #

    29 # Barry Warsaw integrated these modules, wrote the .install() API and code,

    30 # and conformed all C and Python code to Python's coding standards.

    31 #

    32 # Francois Pinard and Marc-Andre Lemburg also contributed valuably to this

    33 # module.

    34 #

    35 # J. David Ibanez implemented plural forms. Bruno Haible fixed some bugs.

    36 #

    37 # TODO:

    38 # - Lazy loading of .mo files.  Currently the entire catalog is loaded into

    39 #   memory, but that's probably bad for large translated programs.  Instead,

    40 #   the lexical sort of original strings in GNU .mo files should be exploited

    41 #   to do binary searches and lazy initializations.  Or you might want to use

    42 #   the undocumented double-hash algorithm for .mo files with hash tables, but

    43 #   you'll need to study the GNU gettext code to do this.

    44 #

    45 # - Support Solaris .mo file formats.  Unfortunately, we've been unable to

    46 #   find this format documented anywhere.

    47 

    48 

    49 import copy, os, re, struct, sys

    50 from errno import ENOENT

    51 

    52 

    53 __all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',

    54            'find', 'translation', 'install', 'textdomain', 'bindtextdomain',

    55            'dgettext', 'dngettext', 'gettext', 'ngettext',

    56            ]

    57 

    58 _default_localedir = os.path.join(sys.prefix, 'share', 'locale')

    59 

    60 

    61 def test(condition, true, false):

    62     """

    63     Implements the C expression:

    64 

    65       condition ? true : false

    66 

    67     Required to correctly interpret plural forms.

    68     """

    69     if condition:

    70         return true

    71     else:

    72         return false

    73 

    74 

    75 def c2py(plural):

    76     """Gets a C expression as used in PO files for plural forms and returns a

    77     Python lambda function that implements an equivalent expression.

    78     """

    79     # Security check, allow only the "n" identifier

    80     from StringIO import StringIO

    81     import token, tokenize

    82     tokens = tokenize.generate_tokens(StringIO(plural).readline)

    83     try:

    84         danger = [x for x in tokens if x[0] == token.NAME and x[1] != 'n']

    85     except tokenize.TokenError:

    86         raise ValueError, \

    87               'plural forms expression error, maybe unbalanced parenthesis'

    88     else:

    89         if danger:

    90             raise ValueError, 'plural forms expression could be dangerous'

    91 

    92     # Replace some C operators by their Python equivalents

    93     plural = plural.replace('&&', ' and ')

    94     plural = plural.replace('||', ' or ')

    95 

    96     expr = re.compile(r'\!([^=])')

    97     plural = expr.sub(' not \\1', plural)

    98 

    99     # Regular expression and replacement function used to transform

   100     # "a?b:c" to "test(a,b,c)".

   101     expr = re.compile(r'(.*?)\?(.*?):(.*)')

   102     def repl(x):

   103         return "test(%s, %s, %s)" % (x.group(1), x.group(2),

   104                                      expr.sub(repl, x.group(3)))

   105 

   106     # Code to transform the plural expression, taking care of parentheses

   107     stack = ['']

   108     for c in plural:

   109         if c == '(':

   110             stack.append('')

   111         elif c == ')':

   112             if len(stack) == 1:

   113                 # Actually, we never reach this code, because unbalanced

   114                 # parentheses get caught in the security check at the

   115                 # beginning.

   116                 raise ValueError, 'unbalanced parenthesis in plural form'

   117             s = expr.sub(repl, stack.pop())

   118             stack[-1] += '(%s)' % s

   119         else:

   120             stack[-1] += c

   121     plural = expr.sub(repl, stack.pop())

   122 

   123     return eval('lambda n: int(%s)' % plural)

   124 

   125 

   126 

   127 def _expand_lang(locale):

   128     from locale import normalize

   129     locale = normalize(locale)

   130     COMPONENT_CODESET   = 1 << 0

   131     COMPONENT_TERRITORY = 1 << 1

   132     COMPONENT_MODIFIER  = 1 << 2

   133     # split up the locale into its base components

   134     mask = 0

   135     pos = locale.find('@')

   136     if pos >= 0:

   137         modifier = locale[pos:]

   138         locale = locale[:pos]

   139         mask |= COMPONENT_MODIFIER

   140     else:

   141         modifier = ''

   142     pos = locale.find('.')

   143     if pos >= 0:

   144         codeset = locale[pos:]

   145         locale = locale[:pos]

   146         mask |= COMPONENT_CODESET

   147     else:

   148         codeset = ''

   149     pos = locale.find('_')

   150     if pos >= 0:

   151         territory = locale[pos:]

   152         locale = locale[:pos]

   153         mask |= COMPONENT_TERRITORY

   154     else:

   155         territory = ''

   156     language = locale

   157     ret = []

   158     for i in range(mask+1):

   159         if not (i & ~mask):  # if all components for this combo exist ...

   160             val = language

   161             if i & COMPONENT_TERRITORY: val += territory

   162             if i & COMPONENT_CODESET:   val += codeset

   163             if i & COMPONENT_MODIFIER:  val += modifier

   164             ret.append(val)

   165     ret.reverse()

   166     return ret

   167 

   168 

   169 

   170 class NullTranslations:

   171     def __init__(self, fp=None):

   172         self._info = {}

   173         self._charset = None

   174         self._fallback = None

   175         if fp is not None:

   176             self._parse(fp)

   177 

   178     def _parse(self, fp):

   179         pass

   180 

   181     def add_fallback(self, fallback):

   182         if self._fallback:

   183             self._fallback.add_fallback(fallback)

   184         else:

   185             self._fallback = fallback

   186 

   187     def gettext(self, message):

   188         if self._fallback:

   189             return self._fallback.gettext(message)

   190         return message

   191 

   192     def ngettext(self, msgid1, msgid2, n):

   193         if self._fallback:

   194             return self._fallback.ngettext(msgid1, msgid2, n)

   195         if n == 1:

   196             return msgid1

   197         else:

   198             return msgid2

   199 

   200     def ugettext(self, message):

   201         if self._fallback:

   202             return self._fallback.ugettext(message)

   203         return unicode(message)

   204 

   205     def ungettext(self, msgid1, msgid2, n):

   206         if self._fallback:

   207             return self._fallback.ungettext(msgid1, msgid2, n)

   208         if n == 1:

   209             return unicode(msgid1)

   210         else:

   211             return unicode(msgid2)

   212 

   213     def info(self):

   214         return self._info

   215 

   216     def charset(self):

   217         return self._charset

   218 

   219     def install(self, unicode=False):

   220         import __builtin__

   221         __builtin__.__dict__['_'] = unicode and self.ugettext or self.gettext

   222 

   223 

   224 class GNUTranslations(NullTranslations):

   225     # Magic number of .mo files

   226     LE_MAGIC = 0x950412deL

   227     BE_MAGIC = 0xde120495L

   228 

   229     def _parse(self, fp):

   230         """Override this method to support alternative .mo formats."""

   231         unpack = struct.unpack

   232         filename = getattr(fp, 'name', '')

   233         # Parse the .mo file header, which consists of 5 little endian 32

   234         # bit words.

   235         self._catalog = catalog = {}

   236         self.plural = lambda n: int(n != 1) # germanic plural by default

   237         buf = fp.read()

   238         buflen = len(buf)

   239         # Are we big endian or little endian?

   240         magic = unpack('<I', buf[:4])[0]

   241         if magic == self.LE_MAGIC:

   242             version, msgcount, masteridx, transidx = unpack('<4I', buf[4:20])

   243             ii = '<II'

   244         elif magic == self.BE_MAGIC:

   245             version, msgcount, masteridx, transidx = unpack('>4I', buf[4:20])

   246             ii = '>II'

   247         else:

   248             raise IOError(0, 'Bad magic number', filename)

   249         # Now put all messages from the .mo file buffer into the catalog

   250         # dictionary.

   251         for i in xrange(0, msgcount):

   252             mlen, moff = unpack(ii, buf[masteridx:masteridx+8])

   253             mend = moff + mlen

   254             tlen, toff = unpack(ii, buf[transidx:transidx+8])

   255             tend = toff + tlen

   256             if mend < buflen and tend < buflen:

   257                 msg = buf[moff:mend]

   258                 tmsg = buf[toff:tend]

   259             else:

   260                 raise IOError(0, 'File is corrupt', filename)

   261             # See if we're looking at GNU .mo conventions for metadata

   262             if mlen == 0:

   263                 # Catalog description

   264                 # don't handle multi-lines fields here, and skip

   265                 # lines which don't look like a header description

   266                 # (e.g. "header: value")

   267                 lastk = k = None

   268                 for item in tmsg.splitlines():

   269                     item = item.strip()

   270                     if not item or not ':' in item:

   271                         continue

   272                     k, v = item.split(':', 1)

   273                     k = k.strip().lower()

   274                     v = v.strip()

   275                     self._info[k] = v

   276                     if k == 'content-type':

   277                         self._charset = v.split('charset=')[1]

   278                     elif k == 'plural-forms':

   279                         v = v.split(';')

   280                         plural = v[1].split('plural=')[1]

   281                         self.plural = c2py(plural)

   282             # Note: we unconditionally convert both msgids and msgstrs to

   283             # Unicode using the character encoding specified in the charset

   284             # parameter of the Content-Type header.  The gettext documentation

   285             # strongly encourages msgids to be us-ascii, but some appliations

   286             # require alternative encodings (e.g. Zope's ZCML and ZPT).  For

   287             # traditional gettext applications, the msgid conversion will

   288             # cause no problems since us-ascii should always be a subset of

   289             # the charset encoding.  We may want to fall back to 8-bit msgids

   290             # if the Unicode conversion fails.

   291             if msg.find('\x00') >= 0:

   292                 # Plural forms

   293                 msgid1, msgid2 = msg.split('\x00')

   294                 tmsg = tmsg.split('\x00')

   295                 if self._charset:

   296                     msgid1 = unicode(msgid1, self._charset)

   297                     tmsg = [unicode(x, self._charset) for x in tmsg]

   298                 for i in range(len(tmsg)):

   299                     catalog[(msgid1, i)] = tmsg[i]

   300             else:

   301                 if self._charset:

   302                     msg = unicode(msg, self._charset)

   303                     tmsg = unicode(tmsg, self._charset)

   304                 catalog[msg] = tmsg

   305             # advance to next entry in the seek tables

   306             masteridx += 8

   307             transidx += 8

   308 

   309     def gettext(self, message):

   310         missing = object()

   311         tmsg = self._catalog.get(message, missing)

   312         if tmsg is missing:

   313             if self._fallback:

   314                 return self._fallback.gettext(message)

   315             return message

   316         # Encode the Unicode tmsg back to an 8-bit string, if possible

   317         if self._charset:

   318             return tmsg.encode(self._charset)

   319         return tmsg

   320 

   321     def ngettext(self, msgid1, msgid2, n):

   322         try:

   323             tmsg = self._catalog[(msgid1, self.plural(n))]

   324             if self._charset:

   325                 return tmsg.encode(self._charset)

   326             return tmsg

   327         except KeyError:

   328             if self._fallback:

   329                 return self._fallback.ngettext(msgid1, msgid2, n)

   330             if n == 1:

   331                 return msgid1

   332             else:

   333                 return msgid2

   334 

   335     def ugettext(self, message):

   336         missing = object()

   337         tmsg = self._catalog.get(message, missing)

   338         if tmsg is missing:

   339             if self._fallback:

   340                 return self._fallback.ugettext(message)

   341             return unicode(message)

   342         return tmsg

   343 

   344     def ungettext(self, msgid1, msgid2, n):

   345         try:

   346             tmsg = self._catalog[(msgid1, self.plural(n))]

   347         except KeyError:

   348             if self._fallback:

   349                 return self._fallback.ungettext(msgid1, msgid2, n)

   350             if n == 1:

   351                 tmsg = unicode(msgid1)

   352             else:

   353                 tmsg = unicode(msgid2)

   354         return tmsg

   355 

   356 

   357 # Locate a .mo file using the gettext strategy

   358 def find(domain, localedir=None, languages=None, all=0):

   359     # Get some reasonable defaults for arguments that were not supplied

   360     if localedir is None:

   361         localedir = _default_localedir

   362     if languages is None:

   363         languages = []

   364         for envar in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'):

   365             val = os.environ.get(envar)

   366             if val:

   367                 languages = val.split(':')

   368                 break

   369         if 'C' not in languages:

   370             languages.append('C')

   371     # now normalize and expand the languages

   372     nelangs = []

   373     for lang in languages:

   374         for nelang in _expand_lang(lang):

   375             if nelang not in nelangs:

   376                 nelangs.append(nelang)

   377     # select a language

   378     if all:

   379         result = []

   380     else:

   381         result = None

   382     for lang in nelangs:

   383         if lang == 'C':

   384             break

   385         mofile = os.path.join(localedir, lang, 'LC_MESSAGES', '%s.mo' % domain)

   386         if os.path.exists(mofile):

   387             if all:

   388                 result.append(mofile)

   389             else:

   390                 return mofile

   391     return result

   392 

   393 

   394 

   395 # a mapping between absolute .mo file path and Translation object

   396 _translations = {}

   397 

   398 def translation(domain, localedir=None, languages=None,

   399                 class_=None, fallback=False):

   400     if class_ is None:

   401         class_ = GNUTranslations

   402     mofiles = find(domain, localedir, languages, all=1)

   403     if not mofiles:

   404         if fallback:

   405             return NullTranslations()

   406         raise IOError(ENOENT, 'No translation file found for domain', domain)

   407     # TBD: do we need to worry about the file pointer getting collected?

   408     # Avoid opening, reading, and parsing the .mo file after it's been done

   409     # once.

   410     result = None

   411     for mofile in mofiles:

   412         key = os.path.abspath(mofile)

   413         t = _translations.get(key)

   414         if t is None:

   415             t = _translations.setdefault(key, class_(open(mofile, 'rb')))

   416         # Copy the translation object to allow setting fallbacks.

   417         # All other instance data is shared with the cached object.

   418         t = copy.copy(t)

   419         if result is None:

   420             result = t

   421         else:

   422             result.add_fallback(t)

   423     return result

   424 

   425 

   426 def install(domain, localedir=None, unicode=False):

   427     translation(domain, localedir, fallback=True).install(unicode)

   428 

   429 

   430 

   431 # a mapping b/w domains and locale directories

   432 _localedirs = {}

   433 # current global domain, `messages' used for compatibility w/ GNU gettext

   434 _current_domain = 'messages'

   435 

   436 

   437 def textdomain(domain=None):

   438     global _current_domain

   439     if domain is not None:

   440         _current_domain = domain

   441     return _current_domain

   442 

   443 

   444 def bindtextdomain(domain, localedir=None):

   445     global _localedirs

   446     if localedir is not None:

   447         _localedirs[domain] = localedir

   448     return _localedirs.get(domain, _default_localedir)

   449 

   450 

   451 def dgettext(domain, message):

   452     try:

   453         t = translation(domain, _localedirs.get(domain, None))

   454     except IOError:

   455         return message

   456     return t.gettext(message)

   457 

   458 

   459 def dngettext(domain, msgid1, msgid2, n):

   460     try:

   461         t = translation(domain, _localedirs.get(domain, None))

   462     except IOError:

   463         if n == 1:

   464             return msgid1

   465         else:

   466             return msgid2

   467     return t.ngettext(msgid1, msgid2, n)

   468 

   469 

   470 def gettext(message):

   471     return dgettext(_current_domain, message)

   472 

   473 

   474 def ngettext(msgid1, msgid2, n):

   475     return dngettext(_current_domain, msgid1, msgid2, n)

   476 

   477 

   478 # dcgettext() has been deemed unnecessary and is not implemented.

   479 

   480 # James Henstridge's Catalog constructor from GNOME gettext.  Documented usage

   481 # was:

   482 #

   483 #    import gettext

   484 #    cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR)

   485 #    _ = cat.gettext

   486 #    print _('Hello World')

   487 

   488 # The resulting catalog object currently don't support access through a

   489 # dictionary API, which was supported (but apparently unused) in GNOME

   490 # gettext.

   491 

   492 Catalog = translation