1 """Internationalization and localization support.
3 This module provides internationalization (I18N) and localization (L10N)
4 support for your Python programs by providing an interface to the GNU gettext
5 message catalog library.
7 I18N refers to the operation by which a program is made aware of multiple
8 languages. L10N refers to the adaptation of your program, once
9 internationalized, to the local language and cultural habits.
13 # This module represents the integration of work, contributions, feedback, and
14 # suggestions from the following people:
16 # Martin von Loewis, who wrote the initial implementation of the underlying
17 # C-based libintlmodule (later renamed _gettext), along with a skeletal
18 # gettext.py implementation.
20 # Peter Funk, who wrote fintl.py, a fairly complete wrapper around intlmodule,
21 # which also included a pure-Python implementation to read .mo files if
22 # intlmodule wasn't available.
24 # James Henstridge, who also wrote a gettext.py module, which has some
25 # interesting, but currently unsupported experimental features: the notion of
26 # a Catalog class and instances, and the ability to add to a catalog file via
29 # Barry Warsaw integrated these modules, wrote the .install() API and code,
30 # and conformed all C and Python code to Python's coding standards.
32 # Francois Pinard and Marc-Andre Lemburg also contributed valuably to this
35 # J. David Ibanez implemented plural forms. Bruno Haible fixed some bugs.
38 # - Lazy loading of .mo files. Currently the entire catalog is loaded into
39 # memory, but that's probably bad for large translated programs. Instead,
40 # the lexical sort of original strings in GNU .mo files should be exploited
41 # to do binary searches and lazy initializations. Or you might want to use
42 # the undocumented double-hash algorithm for .mo files with hash tables, but
43 # you'll need to study the GNU gettext code to do this.
45 # - Support Solaris .mo file formats. Unfortunately, we've been unable to
46 # find this format documented anywhere.
49 import locale, copy, os, re, struct, sys
50 from errno import ENOENT
53 __all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',
54 'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
55 'dgettext', 'dngettext', 'gettext', 'ngettext',
58 _default_localedir = os.path.join(sys.prefix, 'share', 'locale')
61 def test(condition, true, false):
63 Implements the C expression:
65 condition ? true : false
67 Required to correctly interpret plural forms.
76 """Gets a C expression as used in PO files for plural forms and returns a
77 Python lambda function that implements an equivalent expression.
79 # Security check, allow only the "n" identifier
81 from cStringIO import StringIO
83 from StringIO import StringIO
84 import token, tokenize
85 tokens = tokenize.generate_tokens(StringIO(plural).readline)
87 danger = [x for x in tokens if x[0] == token.NAME and x[1] != 'n']
88 except tokenize.TokenError:
90 'plural forms expression error, maybe unbalanced parenthesis'
93 raise ValueError, 'plural forms expression could be dangerous'
95 # Replace some C operators by their Python equivalents
96 plural = plural.replace('&&', ' and ')
97 plural = plural.replace('||', ' or ')
99 expr = re.compile(r'\!([^=])')
100 plural = expr.sub(' not \\1', plural)
102 # Regular expression and replacement function used to transform
103 # "a?b:c" to "test(a,b,c)".
104 expr = re.compile(r'(.*?)\?(.*?):(.*)')
106 return "test(%s, %s, %s)" % (x.group(1), x.group(2),
107 expr.sub(repl, x.group(3)))
109 # Code to transform the plural expression, taking care of parentheses
116 # Actually, we never reach this code, because unbalanced
117 # parentheses get caught in the security check at the
119 raise ValueError, 'unbalanced parenthesis in plural form'
120 s = expr.sub(repl, stack.pop())
121 stack[-1] += '(%s)' % s
124 plural = expr.sub(repl, stack.pop())
126 return eval('lambda n: int(%s)' % plural)
130 def _expand_lang(locale):
131 from locale import normalize
132 locale = normalize(locale)
133 COMPONENT_CODESET = 1 << 0
134 COMPONENT_TERRITORY = 1 << 1
135 COMPONENT_MODIFIER = 1 << 2
136 # split up the locale into its base components
138 pos = locale.find('@')
140 modifier = locale[pos:]
141 locale = locale[:pos]
142 mask |= COMPONENT_MODIFIER
145 pos = locale.find('.')
147 codeset = locale[pos:]
148 locale = locale[:pos]
149 mask |= COMPONENT_CODESET
152 pos = locale.find('_')
154 territory = locale[pos:]
155 locale = locale[:pos]
156 mask |= COMPONENT_TERRITORY
161 for i in range(mask+1):
162 if not (i & ~mask): # if all components for this combo exist ...
164 if i & COMPONENT_TERRITORY: val += territory
165 if i & COMPONENT_CODESET: val += codeset
166 if i & COMPONENT_MODIFIER: val += modifier
173 class NullTranslations:
174 def __init__(self, fp=None):
177 self._output_charset = None
178 self._fallback = None
182 def _parse(self, fp):
185 def add_fallback(self, fallback):
187 self._fallback.add_fallback(fallback)
189 self._fallback = fallback
191 def gettext(self, message):
193 return self._fallback.gettext(message)
196 def lgettext(self, message):
198 return self._fallback.lgettext(message)
201 def ngettext(self, msgid1, msgid2, n):
203 return self._fallback.ngettext(msgid1, msgid2, n)
209 def lngettext(self, msgid1, msgid2, n):
211 return self._fallback.lngettext(msgid1, msgid2, n)
217 def ugettext(self, message):
219 return self._fallback.ugettext(message)
220 return unicode(message)
222 def ungettext(self, msgid1, msgid2, n):
224 return self._fallback.ungettext(msgid1, msgid2, n)
226 return unicode(msgid1)
228 return unicode(msgid2)
236 def output_charset(self):
237 return self._output_charset
239 def set_output_charset(self, charset):
240 self._output_charset = charset
242 def install(self, unicode=False, names=None):
244 __builtin__.__dict__['_'] = unicode and self.ugettext or self.gettext
245 if hasattr(names, "__contains__"):
246 if "gettext" in names:
247 __builtin__.__dict__['gettext'] = __builtin__.__dict__['_']
248 if "ngettext" in names:
249 __builtin__.__dict__['ngettext'] = (unicode and self.ungettext
251 if "lgettext" in names:
252 __builtin__.__dict__['lgettext'] = self.lgettext
253 if "lngettext" in names:
254 __builtin__.__dict__['lngettext'] = self.lngettext
257 class GNUTranslations(NullTranslations):
258 # Magic number of .mo files
259 LE_MAGIC = 0x950412deL
260 BE_MAGIC = 0xde120495L
262 def _parse(self, fp):
263 """Override this method to support alternative .mo formats."""
264 unpack = struct.unpack
265 filename = getattr(fp, 'name', '')
266 # Parse the .mo file header, which consists of 5 little endian 32
268 self._catalog = catalog = {}
269 self.plural = lambda n: int(n != 1) # germanic plural by default
272 # Are we big endian or little endian?
273 magic = unpack('<I', buf[:4])[0]
274 if magic == self.LE_MAGIC:
275 version, msgcount, masteridx, transidx = unpack('<4I', buf[4:20])
277 elif magic == self.BE_MAGIC:
278 version, msgcount, masteridx, transidx = unpack('>4I', buf[4:20])
281 raise IOError(0, 'Bad magic number', filename)
282 # Now put all messages from the .mo file buffer into the catalog
284 for i in xrange(0, msgcount):
285 mlen, moff = unpack(ii, buf[masteridx:masteridx+8])
287 tlen, toff = unpack(ii, buf[transidx:transidx+8])
289 if mend < buflen and tend < buflen:
291 tmsg = buf[toff:tend]
293 raise IOError(0, 'File is corrupt', filename)
294 # See if we're looking at GNU .mo conventions for metadata
296 # Catalog description
298 for item in tmsg.splitlines():
303 k, v = item.split(':', 1)
304 k = k.strip().lower()
309 self._info[lastk] += '\n' + item
310 if k == 'content-type':
311 self._charset = v.split('charset=')[1]
312 elif k == 'plural-forms':
314 plural = v[1].split('plural=')[1]
315 self.plural = c2py(plural)
316 # Note: we unconditionally convert both msgids and msgstrs to
317 # Unicode using the character encoding specified in the charset
318 # parameter of the Content-Type header. The gettext documentation
319 # strongly encourages msgids to be us-ascii, but some appliations
320 # require alternative encodings (e.g. Zope's ZCML and ZPT). For
321 # traditional gettext applications, the msgid conversion will
322 # cause no problems since us-ascii should always be a subset of
323 # the charset encoding. We may want to fall back to 8-bit msgids
324 # if the Unicode conversion fails.
327 msgid1, msgid2 = msg.split('\x00')
328 tmsg = tmsg.split('\x00')
330 msgid1 = unicode(msgid1, self._charset)
331 tmsg = [unicode(x, self._charset) for x in tmsg]
332 for i in range(len(tmsg)):
333 catalog[(msgid1, i)] = tmsg[i]
336 msg = unicode(msg, self._charset)
337 tmsg = unicode(tmsg, self._charset)
339 # advance to next entry in the seek tables
343 def gettext(self, message):
345 tmsg = self._catalog.get(message, missing)
348 return self._fallback.gettext(message)
350 # Encode the Unicode tmsg back to an 8-bit string, if possible
351 if self._output_charset:
352 return tmsg.encode(self._output_charset)
354 return tmsg.encode(self._charset)
357 def lgettext(self, message):
359 tmsg = self._catalog.get(message, missing)
362 return self._fallback.lgettext(message)
364 if self._output_charset:
365 return tmsg.encode(self._output_charset)
366 return tmsg.encode(locale.getpreferredencoding())
368 def ngettext(self, msgid1, msgid2, n):
370 tmsg = self._catalog[(msgid1, self.plural(n))]
371 if self._output_charset:
372 return tmsg.encode(self._output_charset)
374 return tmsg.encode(self._charset)
378 return self._fallback.ngettext(msgid1, msgid2, n)
384 def lngettext(self, msgid1, msgid2, n):
386 tmsg = self._catalog[(msgid1, self.plural(n))]
387 if self._output_charset:
388 return tmsg.encode(self._output_charset)
389 return tmsg.encode(locale.getpreferredencoding())
392 return self._fallback.lngettext(msgid1, msgid2, n)
398 def ugettext(self, message):
400 tmsg = self._catalog.get(message, missing)
403 return self._fallback.ugettext(message)
404 return unicode(message)
407 def ungettext(self, msgid1, msgid2, n):
409 tmsg = self._catalog[(msgid1, self.plural(n))]
412 return self._fallback.ungettext(msgid1, msgid2, n)
414 tmsg = unicode(msgid1)
416 tmsg = unicode(msgid2)
420 # Locate a .mo file using the gettext strategy
421 def find(domain, localedir=None, languages=None, all=0):
422 # Get some reasonable defaults for arguments that were not supplied
423 if localedir is None:
424 localedir = _default_localedir
425 if languages is None:
427 for envar in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'):
428 val = os.environ.get(envar)
430 languages = val.split(':')
432 if 'C' not in languages:
433 languages.append('C')
434 # now normalize and expand the languages
436 for lang in languages:
437 for nelang in _expand_lang(lang):
438 if nelang not in nelangs:
439 nelangs.append(nelang)
448 mofile = os.path.join(localedir, lang, 'LC_MESSAGES', '%s.mo' % domain)
449 if os.path.exists(mofile):
451 result.append(mofile)
458 # a mapping between absolute .mo file path and Translation object
461 def translation(domain, localedir=None, languages=None,
462 class_=None, fallback=False, codeset=None):
464 class_ = GNUTranslations
465 mofiles = find(domain, localedir, languages, all=1)
468 return NullTranslations()
469 raise IOError(ENOENT, 'No translation file found for domain', domain)
470 # TBD: do we need to worry about the file pointer getting collected?
471 # Avoid opening, reading, and parsing the .mo file after it's been done
474 for mofile in mofiles:
475 key = os.path.abspath(mofile)
476 t = _translations.get(key)
478 t = _translations.setdefault(key, class_(open(mofile, 'rb')))
479 # Copy the translation object to allow setting fallbacks and
480 # output charset. All other instance data is shared with the
484 t.set_output_charset(codeset)
488 result.add_fallback(t)
492 def install(domain, localedir=None, unicode=False, codeset=None, names=None):
493 t = translation(domain, localedir, fallback=True, codeset=codeset)
494 t.install(unicode, names)
498 # a mapping b/w domains and locale directories
500 # a mapping b/w domains and codesets
502 # current global domain, `messages' used for compatibility w/ GNU gettext
503 _current_domain = 'messages'
506 def textdomain(domain=None):
507 global _current_domain
508 if domain is not None:
509 _current_domain = domain
510 return _current_domain
513 def bindtextdomain(domain, localedir=None):
515 if localedir is not None:
516 _localedirs[domain] = localedir
517 return _localedirs.get(domain, _default_localedir)
520 def bind_textdomain_codeset(domain, codeset=None):
521 global _localecodesets
522 if codeset is not None:
523 _localecodesets[domain] = codeset
524 return _localecodesets.get(domain)
527 def dgettext(domain, message):
529 t = translation(domain, _localedirs.get(domain, None),
530 codeset=_localecodesets.get(domain))
533 return t.gettext(message)
535 def ldgettext(domain, message):
537 t = translation(domain, _localedirs.get(domain, None),
538 codeset=_localecodesets.get(domain))
541 return t.lgettext(message)
543 def dngettext(domain, msgid1, msgid2, n):
545 t = translation(domain, _localedirs.get(domain, None),
546 codeset=_localecodesets.get(domain))
552 return t.ngettext(msgid1, msgid2, n)
554 def ldngettext(domain, msgid1, msgid2, n):
556 t = translation(domain, _localedirs.get(domain, None),
557 codeset=_localecodesets.get(domain))
563 return t.lngettext(msgid1, msgid2, n)
565 def gettext(message):
566 return dgettext(_current_domain, message)
568 def lgettext(message):
569 return ldgettext(_current_domain, message)
571 def ngettext(msgid1, msgid2, n):
572 return dngettext(_current_domain, msgid1, msgid2, n)
574 def lngettext(msgid1, msgid2, n):
575 return ldngettext(_current_domain, msgid1, msgid2, n)
577 # dcgettext() has been deemed unnecessary and is not implemented.
579 # James Henstridge's Catalog constructor from GNOME gettext. Documented usage
583 # cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR)
585 # print _('Hello World')
587 # The resulting catalog object currently don't support access through a
588 # dictionary API, which was supported (but apparently unused) in GNOME
591 Catalog = translation