Package translate :: Package storage :: Module base

[hide private]
[frames] | no frames]

# Source Code for Module translate.storage.base

      1  #!/usr/bin/env python
      2  # -*- coding: utf-8 -*-
      3  #
      4  # Copyright 2006-2009 Zuza Software Foundation
      5  #
      6  # This file is part of the Translate Toolkit.
      7  #
      8  # This program is free software; you can redistribute it and/or modify
      9  # it under the terms of the GNU General Public License as published by
     10  # the Free Software Foundation; either version 2 of the License, or
     11  # (at your option) any later version.
     12  #
     13  # This program is distributed in the hope that it will be useful,
     14  # but WITHOUT ANY WARRANTY; without even the implied warranty of
     15  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     16  # GNU General Public License for more details.
     17  #
     18  # You should have received a copy of the GNU General Public License
     19  # along with this program; if not, see <http://www.gnu.org/licenses/>.
     20
     21  """Base classes for storage interfaces.
     22
     23  @organization: Zuza Software Foundation
     24  @copyright: 2006-2009 Zuza Software Foundation
     25  @license: U{GPL <http://www.fsf.org/licensing/licenses/gpl.html>}
     26  """
     27
     28  try:
     29      import cPickle as pickle
     30  except:
     31      import pickle
     32  from exceptions import NotImplementedError
     33  import translate.i18n
     34  from translate.storage.placeables import StringElem, general, parse as rich_parse
     35  from translate.misc.typecheck import accepts, Self, IsOneOf
     36  from translate.misc.multistring import multistring
     37
     38 -def force_override(method, baseclass):
     39      """Forces derived classes to override method."""
     40
     41      if type(method.im_self) == type(baseclass):
     42          # then this is a classmethod and im_self is the actual class
     43          actualclass = method.im_self
     44      else:
     45          actualclass = method.im_class
     46      if actualclass != baseclass:
     47          raise NotImplementedError(
     48              "%s does not reimplement %s as required by %s" % \
     49              (actualclass.__name__, method.__name__, baseclass.__name__)
     50          )
     51
     52
     53 -class ParseError(Exception):
     54 -    def __init__(self, inner_exc):
     55          self.inner_exc = inner_exc
     56
     57 -    def __str__(self):
     58          return repr(self.inner_exc)
     59
     60
     61 -class TranslationUnit(object):
     62      """Base class for translation units.
     63
     64      Our concept of a I{translation unit} is influenced heavily by XLIFF:
     65      U{http://www.oasis-open.org/committees/xliff/documents/xliff-specification.htm}
     66
     67      As such most of the method- and variable names borrows from XLIFF terminology.
     68
     69      A translation unit consists of the following:
     70        - A I{source} string. This is the original translatable text.
     71        - A I{target} string. This is the translation of the I{source}.
     72        - Zero or more I{notes} on the unit. Notes would typically be some
     73          comments from a translator on the unit, or some comments originating from
     74          the source code.
     75        - Zero or more I{locations}. Locations indicate where in the original
     76          source code this unit came from.
     77        - Zero or more I{errors}. Some tools (eg. L{pofilter <filters.pofilter>}) can run checks on
     78          translations and produce error messages.
     79
     80      @group Source: *source*
     81      @group Target: *target*
     82      @group Notes: *note*
     83      @group Locations: *location*
     84      @group Errors: *error*
     85      """
     86
     87      rich_parsers = []
     88      """A list of functions to use for parsing a string into a rich string tree."""
     89
     90 -    def __init__(self, source):
     91          """Constructs a TranslationUnit containing the given source string."""
     92          self.notes = ""
     93          self._store = None
     94          self.source = source
     95          self._target = None
     96          self._rich_source = None
     97          self._rich_target = None
     98
     99 -    def __eq__(self, other):
    100          """Compares two TranslationUnits.
    101
    102          @type other: L{TranslationUnit}
    103          @param other: Another L{TranslationUnit}
    104          @rtype: Boolean
    105          @return: Returns True if the supplied TranslationUnit equals this unit.
    106          """
    107          return self.source == other.source and self.target == other.target
    108
    109 -    def rich_to_multistring(cls, elem_list):
    110          """Convert a "rich" string tree to a C{multistring}:
    111
    112             >>> from translate.storage.placeables.interfaces import X
    113             >>> rich = [StringElem(['foo', X(id='xxx', sub=[' ']), 'bar'])]
    114             >>> TranslationUnit.rich_to_multistring(rich)
    115             multistring(u'foo bar')
    116          """
    117          return multistring([unicode(elem) for elem in elem_list])
    118      rich_to_multistring = classmethod(rich_to_multistring)
    119
    120 -    def multistring_to_rich(cls, mulstring):
    121          """Convert a multistring to a list of "rich" string trees:
    122
    123             >>> target = multistring([u'foo', u'bar', u'baz'])
    124             >>> TranslationUnit.multistring_to_rich(target)
    125             [<StringElem([<StringElem([u'foo'])>])>,
    126              <StringElem([<StringElem([u'bar'])>])>,
    127              <StringElem([<StringElem([u'baz'])>])>]
    128          """
    129          if isinstance(mulstring, multistring):
    130              return [rich_parse(s, cls.rich_parsers) for s in mulstring.strings]
    131          return [rich_parse(mulstring, cls.rich_parsers)]
    132
    133 -    def setsource(self, source):
    134          """Sets the source string to the given value."""
    135          self._rich_source = None
    136          self._source = source
    137      source = property(lambda self: self._source, setsource)
    138
    139 -    def settarget(self, target):
    140          """Sets the target string to the given value."""
    141          self._rich_target = None
    142          self._target = target
    143      target = property(lambda self: self._target, settarget)
    144
    145 -    def _get_rich_source(self):
    146          if self._rich_source is None:
    147              self._rich_source = self.multistring_to_rich(self.source)
    148          return self._rich_source
    149 -    def _set_rich_source(self, value):
    150          if not hasattr(value, '__iter__'):
    151              raise ValueError('value must be iterable')
    152          if len(value) < 1:
    153              raise ValueError('value must have at least one element.')
    154          if not isinstance(value[0], StringElem):
    155              raise ValueError('value[0] must be of type StringElem.')
    156          self._rich_source = list(value)
    157          self.source = self.rich_to_multistring(value)
    158      rich_source = property(_get_rich_source, _set_rich_source)
    159      """ @see: rich_to_multistring
    160          @see: multistring_to_rich"""
    161
    162 -    def _get_rich_target(self):
    163          if self._rich_target is None:
    164              self._rich_target = self.multistring_to_rich(self.target)
    165          return self._rich_target
    166 -    def _set_rich_target(self, value):
    167          if not hasattr(value, '__iter__'):
    168              raise ValueError('value must be iterable')
    169          if len(value) < 1:
    170              raise ValueError('value must have at least one element.')
    171          if not isinstance(value[0], StringElem):
    172              raise ValueError('value[0] must be of type StringElem.')
    173          self._rich_target = list(value)
    174          self.target = self.rich_to_multistring(value)
    175      rich_target = property(_get_rich_target, _set_rich_target)
    176      """ @see: rich_to_multistring
    177          @see: multistring_to_rich"""
    178
    179 -    def gettargetlen(self):
    180          """Returns the length of the target string.
    181
    182          @note: Plural forms might be combined.
    183          @rtype: Integer
    184          """
    185          length = len(self.target or "")
    186          strings = getattr(self.target, "strings", [])
    187          if strings:
    188              length += sum([len(pluralform) for pluralform in strings[1:]])
    189          return length
    190
    191 -    def getid(self):
    192          """A unique identifier for this unit.
    193
    194          @rtype: string
    195          @return: an identifier for this unit that is unique in the store
    196
    197          Derived classes should override this in a way that guarantees a unique
    198          identifier for each unit in the store.
    199          """
    200          return self.source
    201
    202 -    def getlocations(self):
    203          """A list of source code locations.
    204
    205          @note: Shouldn't be implemented if the format doesn't support it.
    206          @rtype: List
    207          """
    208          return []
    209
    210 -    def addlocation(self, location):
    211          """Add one location to the list of locations.
    212
    213          @note: Shouldn't be implemented if the format doesn't support it.
    214          """
    215          pass
    216
    217 -    def addlocations(self, location):
    218          """Add a location or a list of locations.
    219
    220          @note: Most classes shouldn't need to implement this,
    221                 but should rather implement L{addlocation()}.
    222          @warning: This method might be removed in future.
    223          """
    224          if isinstance(location, list):
    225              for item in location:
    226                  self.addlocation(item)
    227          else:
    228              self.addlocation(location)
    229
    230 -    def getcontext(self):
    231          """Get the message context."""
    232          return ""
    233
    234 -    def getnotes(self, origin=None):
    235          """Returns all notes about this unit.
    236
    237          It will probably be freeform text or something reasonable that can be
    238          synthesised by the format.
    239          It should not include location comments (see L{getlocations()}).
    240          """
    241          return getattr(self, "notes", "")
    242
    243 -    def addnote(self, text, origin=None):
    244          """Adds a note (comment).
    245
    246          @type text: string
    247          @param text: Usually just a sentence or two.
    248          @type origin: string
    249          @param origin: Specifies who/where the comment comes from.
    250                         Origin can be one of the following text strings:
    251                           - 'translator'
    252                           - 'developer', 'programmer', 'source code' (synonyms)
    253          """
    254          if getattr(self, "notes", None):
    255              self.notes += '\n'+text
    256          else:
    257              self.notes = text
    258
    259 -    def removenotes(self):
    260          """Remove all the translator's notes."""
    261          self.notes = u''
    262
    263 -    def adderror(self, errorname, errortext):
    264          """Adds an error message to this unit.
    265
    266          @type errorname: string
    267          @param errorname: A single word to id the error.
    268          @type errortext: string
    269          @param errortext: The text describing the error.
    270          """
    271          pass
    272
    273 -    def geterrors(self):
    274          """Get all error messages.
    275
    276          @rtype: Dictionary
    277          """
    278          return {}
    279
    280 -    def markreviewneeded(self, needsreview=True, explanation=None):
    281          """Marks the unit to indicate whether it needs review.
    282
    283          @keyword needsreview: Defaults to True.
    284          @keyword explanation: Adds an optional explanation as a note.
    285          """
    286          pass
    287
    288 -    def istranslated(self):
    289          """Indicates whether this unit is translated.
    290
    291          This should be used rather than deducing it from .target,
    292          to ensure that other classes can implement more functionality
    293          (as XLIFF does).
    294          """
    295          return bool(self.target) and not self.isfuzzy()
    296
    297 -    def istranslatable(self):
    298          """Indicates whether this unit can be translated.
    299
    300          This should be used to distinguish real units for translation from
    301          header, obsolete, binary or other blank units.
    302          """
    303          return True
    304
    305 -    def isfuzzy(self):
    306          """Indicates whether this unit is fuzzy."""
    307          return False
    308
    309 -    def markfuzzy(self, value=True):
    310          """Marks the unit as fuzzy or not."""
    311          pass
    312
    313 -    def isobsolete(self):
    314          """indicate whether a unit is obsolete"""
    315          return False
    316
    317 -    def makeobsolete(self):
    318          """Make a unit obsolete"""
    319          pass
    320
    321 -    def isheader(self):
    322          """Indicates whether this unit is a header."""
    323          return False
    324
    325 -    def isreview(self):
    326          """Indicates whether this unit needs review."""
    327          return False
    328
    329 -    def isblank(self):
    330          """Used to see if this unit has no source or target string.
    331
    332          @note: This is probably used more to find translatable units,
    333          and we might want to move in that direction rather and get rid of this.
    334          """
    335          return not (self.source or self.target)
    336
    337 -    def hasplural(self):
    338          """Tells whether or not this specific unit has plural strings."""
    339          #TODO: Reconsider
    340          return False
    341
    342 -    def getsourcelanguage(self):
    343          return getattr(self._store, "sourcelanguage", "en")
    344
    345 -    def gettargetlanguage(self):
    346          return getattr(self._store, "targetlanguage", None)
    347
    348 -    def merge(self, otherunit, overwrite=False, comments=True):
    349          """Do basic format agnostic merging."""
    350          if not self.target or overwrite:
    351              self.rich_target = otherunit.rich_target
    352
    353 -    def unit_iter(self):
    354          """Iterator that only returns this unit."""
    355          yield self
    356
    357 -    def getunits(self):
    358          """This unit in a list."""
    359          return [self]
    360
    361 -    def buildfromunit(cls, unit):
    362          """Build a native unit from a foreign unit, preserving as much
    363          information as possible."""
    364          if type(unit) == cls and hasattr(unit, "copy") and callable(unit.copy):
    365              return unit.copy()
    366          newunit = cls(unit.source)
    367          newunit.target = unit.target
    368          newunit.markfuzzy(unit.isfuzzy())
    369          locations = unit.getlocations()
    370          if locations:
    371              newunit.addlocations(locations)
    372          notes = unit.getnotes()
    373          if notes:
    374              newunit.addnote(notes)
    375          return newunit
    376      buildfromunit = classmethod(buildfromunit)
    377
    378      xid = property(lambda self: None, lambda self, value: None)
    379      rid = property(lambda self: None, lambda self, value: None)
    380
    381
    382 -class TranslationStore(object):
    383      """Base class for stores for multiple translation units of type UnitClass."""
    384
    385      UnitClass = TranslationUnit
    386      """The class of units that will be instantiated and used by this class"""
    387      Name = "Base translation store"
    388      """The human usable name of this store type"""
    389      Mimetypes = None
    390      """A list of MIME types associated with this store type"""
    391      Extensions = None
    392      """A list of file extentions associated with this store type"""
    393      _binary = False
    394      """Indicates whether a file should be accessed as a binary file."""
    395      suggestions_in_format = False
    396      """Indicates if format can store suggestions and alternative translation for a unit"""
    397
    398 -    def __init__(self, unitclass=None):
    399          """Constructs a blank TranslationStore."""
    400          self.units = []
    401          self.sourcelanguage = None
    402          self.targetlanguage = None
    403          if unitclass:
    404              self.UnitClass = unitclass
    405          super(TranslationStore, self).__init__()
    406
    407 -    def getsourcelanguage(self):
    408          """Gets the source language for this store"""
    409          return self.sourcelanguage
    410
    411 -    def setsourcelanguage(self, sourcelanguage):
    412          """Sets the source language for this store"""
    413          self.sourcelanguage = sourcelanguage
    414
    415 -    def gettargetlanguage(self):
    416          """Gets the target language for this store"""
    417          return self.targetlanguage
    418
    419 -    def settargetlanguage(self, targetlanguage):
    420          """Sets the target language for this store"""
    421          self.targetlanguage = targetlanguage
    422
    423 -    def unit_iter(self):
    424          """Iterator over all the units in this store."""
    425          for unit in self.units:
    426              yield unit
    427
    428 -    def getunits(self):
    429          """Return a list of all units in this store."""
    430          return [unit for unit in self.unit_iter()]
    431
    432 -    def addunit(self, unit):
    433          """Appends the given unit to the object's list of units.
    434
    435          This method should always be used rather than trying to modify the
    436          list manually.
    437
    438          @type unit: L{TranslationUnit}
    439          @param unit: The unit that will be added.
    440          """
    441          unit._store = self
    442          self.units.append(unit)
    443
    444 -    def addsourceunit(self, source):
    445          """Adds and returns a new unit with the given source string.
    446
    447          @rtype: L{TranslationUnit}
    448          """
    449          unit = self.UnitClass(source)
    450          self.addunit(unit)
    451          return unit
    452
    453 -    def findid(self, id):
    454          """find unit with matching id by checking id_index"""
    455          self.require_index()
    456          return self.id_index.get(id, None)
    457
    458 -    def findunit(self, source):
    459          """Finds the unit with the given source string.
    460
    461          @rtype: L{TranslationUnit} or None
    462          """
    463          if len(getattr(self, "sourceindex", [])):
    464              if source in self.sourceindex:
    465                  return self.sourceindex[source][0]
    466          else:
    467              for unit in self.units:
    468                  if unit.source == source:
    469                      return unit
    470          return None
    471
    472
    473 -    def findunits(self, source):
    474          """Finds the units with the given source string.
    475
    476          @rtype: L{TranslationUnit} or None
    477          """
    478          if len(getattr(self, "sourceindex", [])):
    479              if source in self.sourceindex:
    480                  return self.sourceindex[source]
    481          else:
    482              #FIXME: maybe we should generate index here instead since
    483              #we'll scan all units anyway
    484              result = []
    485              for unit in self.units:
    486                  if unit.source == source:
    487                      result.append(unit)
    488              return result
    489          return None
    490
    491 -    def translate(self, source):
    492          """Returns the translated string for a given source string.
    493
    494          @rtype: String or None
    495          """
    496          unit = self.findunit(source)
    497          if unit and unit.target:
    498              return unit.target
    499          else:
    500              return None
    501
    502 -    def remove_unit_from_index(self, unit):
    503          """Remove a unit from source and locaton indexes"""
    504          def remove_unit(source):
    505              if source in self.sourceindex:
    506                  try:
    507                      self.sourceindex[source].remove(unit)
    508                      if len(self.sourceindex[source]) == 0:
    509                          del(self.sourceindex[source])
    510                  except ValueError:
    511                      pass
    512
    513          if unit.hasplural():
    514              for source in unit.source.strings:
    515                  remove_unit(source)
    516          else:
    517              remove_unit(unit.source)
    518
    519          for location in unit.getlocations():
    520              if location in self.locationindex and self.locationindex[location] is not None \
    521                     and self.locationindex[location] == unit:
    522                  del(self.locationindex[location])
    523
    524
    525 -    def add_unit_to_index(self, unit):
    526          """Add a unit to source and location idexes"""
    527          self.id_index[unit.getid()] = unit
    528
    529          def insert_unit(source):
    530              if not source in self.sourceindex:
    531                  self.sourceindex[source] = [unit]
    532              else:
    533                  self.sourceindex[source].append(unit)
    534
    535          if unit.hasplural():
    536              for source in unit.source.strings:
    537                  insert_unit(source)
    538          else:
    539              insert_unit(unit.source)
    540
    541          for location in unit.getlocations():
    542              if location in self.locationindex:
    543                  # if sources aren't unique, don't use them
    544                  #FIXME: maybe better store a list of units like sourceindex
    545                  self.locationindex[location] = None
    546              else:
    547                  self.locationindex[location] = unit
    548
    549 -    def makeindex(self):
    550          """Indexes the items in this store. At least .sourceindex should be usefull."""
    551          self.locationindex = {}
    552          self.sourceindex = {}
    553          self.id_index = {}
    554          for index, unit in enumerate(self.units):
    555              unit.index = index
    556              if unit.istranslatable():
    557                  self.add_unit_to_index(unit)
    558
    559 -    def require_index(self):
    560          """make sure source index exists"""
    561          if not hasattr(self, "sourceindex"):
    562              self.makeindex()
    563
    564 -    def getids(self):
    565          """return a list of unit ids"""
    566          self.require_index()
    567          return self.id_index.keys()
    568
    569 -    def __getstate__(self):
    570          odict = self.__dict__.copy()
    571          odict['fileobj'] = None
    572          return odict
    573
    574 -    def __setstate__(self, dict):
    575          self.__dict__.update(dict)
    576          if getattr(self, "filename", False):
    577              self.fileobj = open(self.filename)
    578
    579 -    def __str__(self):
    580          """Converts to a string representation that can be parsed back using L{parsestring()}."""
    581          # We can't pickle fileobj if it is there, so let's hide it for a while.
    582          fileobj = getattr(self, "fileobj", None)
    583          self.fileobj = None
    584          dump = pickle.dumps(self)
    585          self.fileobj = fileobj
    586          return dump
    587
    588 -    def isempty(self):
    589          """Returns True if the object doesn't contain any translation units."""
    590          if len(self.units) == 0:
    591              return True
    592          for unit in self.units:
    593              if unit.istranslatable():
    594                  return False
    595          return True
    596
    597 -    def _assignname(self):
    598          """Tries to work out what the name of the filesystem file is and
    599          assigns it to .filename."""
    600          fileobj = getattr(self, "fileobj", None)
    601          if fileobj:
    602              filename = getattr(fileobj, "name", getattr(fileobj, "filename", None))
    603              if filename:
    604                  self.filename = filename
    605
    606 -    def parsestring(cls, storestring):
    607          """Converts the string representation back to an object."""
    608          newstore = cls()
    609          if storestring:
    610              newstore.parse(storestring)
    611          return newstore
    612      parsestring = classmethod(parsestring)
    613
    614 -    def parse(self, data):
    615          """parser to process the given source string"""
    616          self.units = pickle.loads(data).units
    617
    618 -    def savefile(self, storefile):
    619          """Writes the string representation to the given file (or filename)."""
    620          if isinstance(storefile, basestring):
    621              mode = 'w'
    622              if self._binary:
    623                  mode = 'wb'
    624              storefile = open(storefile, mode)
    625          self.fileobj = storefile
    626          self._assignname()
    627          storestring = str(self)
    628          storefile.write(storestring)
    629          storefile.close()
    630
    631 -    def save(self):
    632          """Save to the file that data was originally read from, if available."""
    633          fileobj = getattr(self, "fileobj", None)
    634          mode = 'w'
    635          if self._binary:
    636              mode = 'wb'
    637          if not fileobj:
    638              filename = getattr(self, "filename", None)
    639              if filename:
    640                  fileobj = file(filename, mode)
    641          else:
    642              fileobj.close()
    643              filename = getattr(fileobj, "name", getattr(fileobj, "filename", None))
    644              if not filename:
    645                  raise ValueError("No file or filename to save to")
    646              fileobj = fileobj.__class__(filename, mode)
    647          self.savefile(fileobj)
    648
    649 -    def parsefile(cls, storefile):
    650          """Reads the given file (or opens the given filename) and parses back to an object."""
    651          mode = 'r'
    652          if cls._binary:
    653              mode = 'rb'
    654          if isinstance(storefile, basestring):
    655              storefile = open(storefile, mode)
    656          mode = getattr(storefile, "mode", mode)
    657          #For some reason GzipFile returns 1, so we have to test for that here
    658          if mode == 1 or "r" in mode:
    659              storestring = storefile.read()
    660              storefile.close()
    661          else:
    662              storestring = ""
    663          newstore = cls.parsestring(storestring)
    664          newstore.fileobj = storefile
    665          newstore._assignname()
    666          return newstore
    667      parsefile = classmethod(parsefile)
    668