Package translate :: Package storage :: Module base

[hide private]
[frames] | no frames]

# Source Code for Module translate.storage.base

      1  #!/usr/bin/env python
      2  # -*- coding: utf-8 -*-
      3  #
      4  # Copyright 2006-2009 Zuza Software Foundation
      5  #
      6  # This file is part of the Translate Toolkit.
      7  #
      8  # This program is free software; you can redistribute it and/or modify
      9  # it under the terms of the GNU General Public License as published by
     10  # the Free Software Foundation; either version 2 of the License, or
     11  # (at your option) any later version.
     12  #
     13  # This program is distributed in the hope that it will be useful,
     14  # but WITHOUT ANY WARRANTY; without even the implied warranty of
     15  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     16  # GNU General Public License for more details.
     17  #
     18  # You should have received a copy of the GNU General Public License
     19  # along with this program; if not, see <http://www.gnu.org/licenses/>.
     20
     21  """Base classes for storage interfaces.
     22
     23  @organization: Zuza Software Foundation
     24  @copyright: 2006-2009 Zuza Software Foundation
     25  @license: U{GPL <http://www.fsf.org/licensing/licenses/gpl.html>}
     26  """
     27
     28  try:
     29      import cPickle as pickle
     30  except:
     31      import pickle
     32  from exceptions import NotImplementedError
     33  import translate.i18n
     34  from translate.storage.placeables import StringElem, general, parse as rich_parse
     35  from translate.misc.typecheck import accepts, Self, IsOneOf
     36  from translate.misc.multistring import multistring
     37
     38 -def force_override(method, baseclass):
     39      """Forces derived classes to override method."""
     40
     41      if type(method.im_self) == type(baseclass):
     42          # then this is a classmethod and im_self is the actual class
     43          actualclass = method.im_self
     44      else:
     45          actualclass = method.im_class
     46      if actualclass != baseclass:
     47          raise NotImplementedError(
     48              "%s does not reimplement %s as required by %s" % \
     49              (actualclass.__name__, method.__name__, baseclass.__name__)
     50          )
     51
     52
     53 -class ParseError(Exception):
     54 -    def __init__(self, inner_exc):
     55          self.inner_exc = inner_exc
     56
     57 -    def __str__(self):
     58          return repr(self.inner_exc)
     59
     60
     61 -class TranslationUnit(object):
     62      """Base class for translation units.
     63
     64      Our concept of a I{translation unit} is influenced heavily by XLIFF:
     65      U{http://www.oasis-open.org/committees/xliff/documents/xliff-specification.htm}
     66
     67      As such most of the method- and variable names borrows from XLIFF terminology.
     68
     69      A translation unit consists of the following:
     70        - A I{source} string. This is the original translatable text.
     71        - A I{target} string. This is the translation of the I{source}.
     72        - Zero or more I{notes} on the unit. Notes would typically be some
     73          comments from a translator on the unit, or some comments originating from
     74          the source code.
     75        - Zero or more I{locations}. Locations indicate where in the original
     76          source code this unit came from.
     77        - Zero or more I{errors}. Some tools (eg. L{pofilter <filters.pofilter>}) can run checks on
     78          translations and produce error messages.
     79
     80      @group Source: *source*
     81      @group Target: *target*
     82      @group Notes: *note*
     83      @group Locations: *location*
     84      @group Errors: *error*
     85      """
     86
     87      rich_parsers = []
     88      """A list of functions to use for parsing a string into a rich string tree."""
     89
     90 -    def __init__(self, source):
     91          """Constructs a TranslationUnit containing the given source string."""
     92          self.notes = ""
     93          self._store = None
     94          self.source = source
     95          self._target = None
     96          self._rich_source = None
     97          self._rich_target = None
     98
     99 -    def __eq__(self, other):
    100          """Compares two TranslationUnits.
    101
    102          @type other: L{TranslationUnit}
    103          @param other: Another L{TranslationUnit}
    104          @rtype: Boolean
    105          @return: Returns True if the supplied TranslationUnit equals this unit.
    106          """
    107          return self.source == other.source and self.target == other.target
    108
    109 -    def rich_to_multistring(cls, elem_list):
    110          """Convert a "rich" string tree to a C{multistring}:
    111
    112             >>> from translate.storage.placeables.interfaces import X
    113             >>> rich = [StringElem(['foo', X(id='xxx', sub=[' ']), 'bar'])]
    114             >>> TranslationUnit.rich_to_multistring(rich)
    115             multistring(u'foo bar')
    116          """
    117          return multistring([unicode(elem) for elem in elem_list])
    118      rich_to_multistring = classmethod(rich_to_multistring)
    119
    120 -    def multistring_to_rich(cls, mulstring):
    121          """Convert a multistring to a list of "rich" string trees:
    122
    123             >>> target = multistring([u'foo', u'bar', u'baz'])
    124             >>> TranslationUnit.multistring_to_rich(target)
    125             [<StringElem([<StringElem([u'foo'])>])>,
    126              <StringElem([<StringElem([u'bar'])>])>,
    127              <StringElem([<StringElem([u'baz'])>])>]
    128          """
    129          if isinstance(mulstring, multistring):
    130              return [rich_parse(s, cls.rich_parsers) for s in mulstring.strings]
    131          return [rich_parse(mulstring, cls.rich_parsers)]
    132
    133 -    def setsource(self, source):
    134          """Sets the source string to the given value."""
    135          self._rich_source = None
    136          self._source = source
    137      source = property(lambda self: self._source, setsource)
    138
    139 -    def settarget(self, target):
    140          """Sets the target string to the given value."""
    141          self._rich_target = None
    142          self._target = target
    143      target = property(lambda self: self._target, settarget)
    144
    145 -    def _get_rich_source(self):
    146          if self._rich_source is None:
    147              self._rich_source = self.multistring_to_rich(self.source)
    148          return self._rich_source
    149 -    def _set_rich_source(self, value):
    150          if not hasattr(value, '__iter__'):
    151              raise ValueError('value must be iterable')
    152          if len(value) < 1:
    153              raise ValueError('value must have at least one element.')
    154          if not isinstance(value[0], StringElem):
    155              raise ValueError('value[0] must be of type StringElem.')
    156          self._rich_source = list(value)
    157          self.source = self.rich_to_multistring(value)
    158      rich_source = property(_get_rich_source, _set_rich_source)
    159      """ @see: rich_to_multistring
    160          @see: multistring_to_rich"""
    161
    162 -    def _get_rich_target(self):
    163          if self._rich_target is None:
    164              self._rich_target = self.multistring_to_rich(self.target)
    165          return self._rich_target
    166 -    def _set_rich_target(self, value):
    167          if not hasattr(value, '__iter__'):
    168              raise ValueError('value must be iterable')
    169          if len(value) < 1:
    170              raise ValueError('value must have at least one element.')
    171          if not isinstance(value[0], StringElem):
    172              raise ValueError('value[0] must be of type StringElem.')
    173          self._rich_target = list(value)
    174          self.target = self.rich_to_multistring(value)
    175      rich_target = property(_get_rich_target, _set_rich_target)
    176      """ @see: rich_to_multistring
    177          @see: multistring_to_rich"""
    178
    179 -    def gettargetlen(self):
    180          """Returns the length of the target string.
    181
    182          @note: Plural forms might be combined.
    183          @rtype: Integer
    184          """
    185          length = len(self.target or "")
    186          strings = getattr(self.target, "strings", [])
    187          if strings:
    188              length += sum([len(pluralform) for pluralform in strings[1:]])
    189          return length
    190
    191 -    def getid(self):
    192          """A unique identifier for this unit.
    193
    194          @rtype: string
    195          @return: an identifier for this unit that is unique in the store
    196
    197          Derived classes should override this in a way that guarantees a unique
    198          identifier for each unit in the store.
    199          """
    200          return self.source
    201
    202 -    def getlocations(self):
    203          """A list of source code locations.
    204
    205          @note: Shouldn't be implemented if the format doesn't support it.
    206          @rtype: List
    207          """
    208          return []
    209
    210 -    def addlocation(self, location):
    211          """Add one location to the list of locations.
    212
    213          @note: Shouldn't be implemented if the format doesn't support it.
    214          """
    215          pass
    216
    217 -    def addlocations(self, location):
    218          """Add a location or a list of locations.
    219
    220          @note: Most classes shouldn't need to implement this,
    221                 but should rather implement L{addlocation()}.
    222          @warning: This method might be removed in future.
    223          """
    224          if isinstance(location, list):
    225              for item in location:
    226                  self.addlocation(item)
    227          else:
    228              self.addlocation(location)
    229
    230 -    def getcontext(self):
    231          """Get the message context."""
    232          return ""
    233
    234 -    def getnotes(self, origin=None):
    235          """Returns all notes about this unit.
    236
    237          It will probably be freeform text or something reasonable that can be
    238          synthesised by the format.
    239          It should not include location comments (see L{getlocations()}).
    240          """
    241          return getattr(self, "notes", "")
    242
    243 -    def addnote(self, text, origin=None):
    244          """Adds a note (comment).
    245
    246          @type text: string
    247          @param text: Usually just a sentence or two.
    248          @type origin: string
    249          @param origin: Specifies who/where the comment comes from.
    250                         Origin can be one of the following text strings:
    251                           - 'translator'
    252                           - 'developer', 'programmer', 'source code' (synonyms)
    253          """
    254          if getattr(self, "notes", None):
    255              self.notes += '\n'+text
    256          else:
    257              self.notes = text
    258
    259 -    def removenotes(self):
    260          """Remove all the translator's notes."""
    261          self.notes = u''
    262
    263 -    def adderror(self, errorname, errortext):
    264          """Adds an error message to this unit.
    265
    266          @type errorname: string
    267          @param errorname: A single word to id the error.
    268          @type errortext: string
    269          @param errortext: The text describing the error.
    270          """
    271          pass
    272
    273 -    def geterrors(self):
    274          """Get all error messages.
    275
    276          @rtype: Dictionary
    277          """
    278          return {}
    279
    280 -    def markreviewneeded(self, needsreview=True, explanation=None):
    281          """Marks the unit to indicate whether it needs review.
    282
    283          @keyword needsreview: Defaults to True.
    284          @keyword explanation: Adds an optional explanation as a note.
    285          """
    286          pass
    287
    288 -    def istranslated(self):
    289          """Indicates whether this unit is translated.
    290
    291          This should be used rather than deducing it from .target,
    292          to ensure that other classes can implement more functionality
    293          (as XLIFF does).
    294          """
    295          return bool(self.target) and not self.isfuzzy()
    296
    297 -    def istranslatable(self):
    298          """Indicates whether this unit can be translated.
    299
    300          This should be used to distinguish real units for translation from
    301          header, obsolete, binary or other blank units.
    302          """
    303          return True
    304
    305 -    def isfuzzy(self):
    306          """Indicates whether this unit is fuzzy."""
    307          return False
    308
    309 -    def markfuzzy(self, value=True):
    310          """Marks the unit as fuzzy or not."""
    311          pass
    312
    313 -    def isheader(self):
    314          """Indicates whether this unit is a header."""
    315          return False
    316
    317 -    def isreview(self):
    318          """Indicates whether this unit needs review."""
    319          return False
    320
    321 -    def isblank(self):
    322          """Used to see if this unit has no source or target string.
    323
    324          @note: This is probably used more to find translatable units,
    325          and we might want to move in that direction rather and get rid of this.
    326          """
    327          return not (self.source or self.target)
    328
    329 -    def hasplural(self):
    330          """Tells whether or not this specific unit has plural strings."""
    331          #TODO: Reconsider
    332          return False
    333
    334 -    def getsourcelanguage(self):
    335          return getattr(self._store, "sourcelanguage", "en")
    336
    337 -    def gettargetlanguage(self):
    338          return getattr(self._store, "targetlanguage", None)
    339
    340 -    def merge(self, otherunit, overwrite=False, comments=True):
    341          """Do basic format agnostic merging."""
    342          if not self.target or overwrite:
    343              self.rich_target = otherunit.rich_target
    344
    345 -    def unit_iter(self):
    346          """Iterator that only returns this unit."""
    347          yield self
    348
    349 -    def getunits(self):
    350          """This unit in a list."""
    351          return [self]
    352
    353 -    def buildfromunit(cls, unit):
    354          """Build a native unit from a foreign unit, preserving as much
    355          information as possible."""
    356          if type(unit) == cls and hasattr(unit, "copy") and callable(unit.copy):
    357              return unit.copy()
    358          newunit = cls(unit.source)
    359          newunit.target = unit.target
    360          newunit.markfuzzy(unit.isfuzzy())
    361          locations = unit.getlocations()
    362          if locations:
    363              newunit.addlocations(locations)
    364          notes = unit.getnotes()
    365          if notes:
    366              newunit.addnote(notes)
    367          return newunit
    368      buildfromunit = classmethod(buildfromunit)
    369
    370      xid = property(lambda self: None, lambda self, value: None)
    371      rid = property(lambda self: None, lambda self, value: None)
    372
    373
    374 -class TranslationStore(object):
    375      """Base class for stores for multiple translation units of type UnitClass."""
    376
    377      UnitClass = TranslationUnit
    378      """The class of units that will be instantiated and used by this class"""
    379      Name = "Base translation store"
    380      """The human usable name of this store type"""
    381      Mimetypes = None
    382      """A list of MIME types associated with this store type"""
    383      Extensions = None
    384      """A list of file extentions associated with this store type"""
    385      _binary = False
    386      """Indicates whether a file should be accessed as a binary file."""
    387      suggestions_in_format = False
    388      """Indicates if format can store suggestions and alternative translation for a unit"""
    389
    390 -    def __init__(self, unitclass=None):
    391          """Constructs a blank TranslationStore."""
    392          self.units = []
    393          self.sourcelanguage = None
    394          self.targetlanguage = None
    395          if unitclass:
    396              self.UnitClass = unitclass
    397          super(TranslationStore, self).__init__()
    398
    399 -    def getsourcelanguage(self):
    400          """Gets the target language for this store"""
    401          return self.sourcelanguage
    402
    403 -    def setsourcelanguage(self, sourcelanguage):
    404          """Sets the source language for this store"""
    405          self.sourcelanguage = sourcelanguage
    406
    407 -    def gettargetlanguage(self):
    408          """Gets the target language for this store"""
    409          return self.targetlanguage
    410
    411 -    def settargetlanguage(self, targetlanguage):
    412          """Sets the target language for this store"""
    413          self.targetlanguage = targetlanguage
    414
    415 -    def unit_iter(self):
    416          """Iterator over all the units in this store."""
    417          for unit in self.units:
    418              yield unit
    419
    420 -    def getunits(self):
    421          """Return a list of all units in this store."""
    422          return [unit for unit in self.unit_iter()]
    423
    424 -    def addunit(self, unit):
    425          """Appends the given unit to the object's list of units.
    426
    427          This method should always be used rather than trying to modify the
    428          list manually.
    429
    430          @type unit: L{TranslationUnit}
    431          @param unit: The unit that will be added.
    432          """
    433          unit._store = self
    434          self.units.append(unit)
    435
    436 -    def addsourceunit(self, source):
    437          """Adds and returns a new unit with the given source string.
    438
    439          @rtype: L{TranslationUnit}
    440          """
    441          unit = self.UnitClass(source)
    442          self.addunit(unit)
    443          return unit
    444
    445 -    def findunit(self, source):
    446          """Finds the unit with the given source string.
    447
    448          @rtype: L{TranslationUnit} or None
    449          """
    450          if len(getattr(self, "sourceindex", [])):
    451              if source in self.sourceindex:
    452                  return self.sourceindex[source][0]
    453          else:
    454              for unit in self.units:
    455                  if unit.source == source:
    456                      return unit
    457          return None
    458
    459
    460 -    def findunits(self, source):
    461          """Finds the unit with the given source string.
    462
    463          @rtype: L{TranslationUnit} or None
    464          """
    465          if len(getattr(self, "sourceindex", [])):
    466              if source in self.sourceindex:
    467                  return self.sourceindex[source]
    468          else:
    469              #FIXME: maybe we should generate index here instead since
    470              #we'll scan all units anyway
    471              result = []
    472              for unit in self.units:
    473                  if unit.source == source:
    474                      result.append(unit)
    475              return result
    476          return None
    477
    478 -    def translate(self, source):
    479          """Returns the translated string for a given source string.
    480
    481          @rtype: String or None
    482          """
    483          unit = self.findunit(source)
    484          if unit and unit.target:
    485              return unit.target
    486          else:
    487              return None
    488
    489 -    def remove_unit_from_index(self, unit):
    490          """Remove a unit from source and locaton indexes"""
    491          def remove_unit(source):
    492              if source in self.sourceindex:
    493                  try:
    494                      self.sourceindex[source].remove(unit)
    495                      if len(self.sourceindex[source]) == 0:
    496                          del(self.sourceindex[source])
    497                  except ValueError:
    498                      pass
    499
    500          if unit.hasplural():
    501              for source in unit.source.strings:
    502                  remove_unit(source)
    503          else:
    504              remove_unit(unit.source)
    505
    506          for location in unit.getlocations():
    507              if location in self.locationindex and self.locationindex[location] is not None \
    508                     and self.locationindex[location] == unit:
    509                  del(self.locationindex[location])
    510
    511
    512 -    def add_unit_to_index(self, unit):
    513          """Add a unit to source and location idexes"""
    514          def insert_unit(source):
    515              if not source in self.sourceindex:
    516                  self.sourceindex[source] = [unit]
    517              else:
    518                  self.sourceindex[source].append(unit)
    519
    520          if unit.hasplural():
    521              for source in unit.source.strings:
    522                  insert_unit(source)
    523          else:
    524              insert_unit(unit.source)
    525
    526          for location in unit.getlocations():
    527              if location in self.locationindex:
    528                  # if sources aren't unique, don't use them
    529                  #FIXME: maybe better store a list of units like sourceindex
    530                  self.locationindex[location] = None
    531              else:
    532                  self.locationindex[location] = unit
    533
    534 -    def makeindex(self):
    535          """Indexes the items in this store. At least .sourceindex should be usefull."""
    536          self.locationindex = {}
    537          self.sourceindex = {}
    538          for unit in self.units:
    539              # Do we need to test if unit.source exists?
    540              self.add_unit_to_index(unit)
    541
    542 -    def require_index(self):
    543          """make sure source index exists"""
    544          if not hasattr(self, "sourceindex"):
    545              self.makeindex()
    546
    547
    548 -    def __getstate__(self):
    549          odict = self.__dict__.copy()
    550          odict['fileobj'] = None
    551          return odict
    552
    553 -    def __setstate__(self, dict):
    554          self.__dict__.update(dict)
    555          if getattr(self, "filename", False):
    556              self.fileobj = open(self.filename)
    557
    558 -    def __str__(self):
    559          """Converts to a string representation that can be parsed back using L{parsestring()}."""
    560          # We can't pickle fileobj if it is there, so let's hide it for a while.
    561          fileobj = getattr(self, "fileobj", None)
    562          self.fileobj = None
    563          dump = pickle.dumps(self)
    564          self.fileobj = fileobj
    565          return dump
    566
    567 -    def isempty(self):
    568          """Returns True if the object doesn't contain any translation units."""
    569          if len(self.units) == 0:
    570              return True
    571          for unit in self.units:
    572              if unit.istranslatable():
    573                  return False
    574          return True
    575
    576 -    def _assignname(self):
    577          """Tries to work out what the name of the filesystem file is and
    578          assigns it to .filename."""
    579          fileobj = getattr(self, "fileobj", None)
    580          if fileobj:
    581              filename = getattr(fileobj, "name", getattr(fileobj, "filename", None))
    582              if filename:
    583                  self.filename = filename
    584
    585 -    def parsestring(cls, storestring):
    586          """Converts the string representation back to an object."""
    587          newstore = cls()
    588          if storestring:
    589              newstore.parse(storestring)
    590          return newstore
    591      parsestring = classmethod(parsestring)
    592
    593 -    def parse(self, data):
    594          """parser to process the given source string"""
    595          self.units = pickle.loads(data).units
    596
    597 -    def savefile(self, storefile):
    598          """Writes the string representation to the given file (or filename)."""
    599          if isinstance(storefile, basestring):
    600              mode = 'w'
    601              if self._binary:
    602                  mode = 'wb'
    603              storefile = open(storefile, mode)
    604          self.fileobj = storefile
    605          self._assignname()
    606          storestring = str(self)
    607          storefile.write(storestring)
    608          storefile.close()
    609
    610 -    def save(self):
    611          """Save to the file that data was originally read from, if available."""
    612          fileobj = getattr(self, "fileobj", None)
    613          mode = 'w'
    614          if self._binary:
    615              mode = 'wb'
    616          if not fileobj:
    617              filename = getattr(self, "filename", None)
    618              if filename:
    619                  fileobj = file(filename, mode)
    620          else:
    621              fileobj.close()
    622              filename = getattr(fileobj, "name", getattr(fileobj, "filename", None))
    623              if not filename:
    624                  raise ValueError("No file or filename to save to")
    625              fileobj = fileobj.__class__(filename, mode)
    626          self.savefile(fileobj)
    627
    628 -    def parsefile(cls, storefile):
    629          """Reads the given file (or opens the given filename) and parses back to an object."""
    630          mode = 'r'
    631          if cls._binary:
    632              mode = 'rb'
    633          if isinstance(storefile, basestring):
    634              storefile = open(storefile, mode)
    635          mode = getattr(storefile, "mode", mode)
    636          #For some reason GzipFile returns 1, so we have to test for that here
    637          if mode == 1 or "r" in mode:
    638              storestring = storefile.read()
    639              storefile.close()
    640          else:
    641              storestring = ""
    642          newstore = cls.parsestring(storestring)
    643          newstore.fileobj = storefile
    644          newstore._assignname()
    645          return newstore
    646      parsefile = classmethod(parsefile)
    647