Package translate :: Package storage :: Module base |
[hide private]
[frames] | no frames]

# Source Code for Module translate.storage.base

      1  #!/usr/bin/env python
      2  # -*- coding: utf-8 -*-
      3  #
      4  # Copyright 2006-2009 Zuza Software Foundation
      5  #
      6  # This file is part of the Translate Toolkit.
      7  #
      8  # This program is free software; you can redistribute it and/or modify
      9  # it under the terms of the GNU General Public License as published by
     10  # the Free Software Foundation; either version 2 of the License, or
     11  # (at your option) any later version.
     12  #
     13  # This program is distributed in the hope that it will be useful,
     14  # but WITHOUT ANY WARRANTY; without even the implied warranty of
     15  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     16  # GNU General Public License for more details.
     17  #
     18  # You should have received a copy of the GNU General Public License
     19  # along with this program; if not, see <http://www.gnu.org/licenses/>.
     20
     21  """Base classes for storage interfaces.
     22
     23  @organization: Zuza Software Foundation
     24  @copyright: 2006-2009 Zuza Software Foundation
     25  @license: U{GPL <http://www.fsf.org/licensing/licenses/gpl.html>}
     26  """
     27
     28  try:
     29      import cPickle as pickle
     30  except:
     31      import pickle
     32  from exceptions import NotImplementedError
     33  import translate.i18n
     34  from translate.storage.placeables import StringElem, general, parse as rich_parse
     35  from translate.misc.typecheck import accepts, Self, IsOneOf
     36  from translate.misc.multistring import multistring
     37
     38 -def force_override(method, baseclass):
     39      """Forces derived classes to override method."""
     40
     41      if type(method.im_self) == type(baseclass):
     42          # then this is a classmethod and im_self is the actual class
     43          actualclass = method.im_self
     44      else:
     45          actualclass = method.im_class
     46      if actualclass != baseclass:
     47          raise NotImplementedError(
     48              "%s does not reimplement %s as required by %s" % \
     49              (actualclass.__name__, method.__name__, baseclass.__name__)
     50          )
     51
     52
     53 -class ParseError(Exception):
     54 -    def __init__(self, inner_exc):
     55          self.inner_exc = inner_exc
     56
     57 -    def __str__(self):
     58          return repr(self.inner_exc)
     59
     60
     61 -class TranslationUnit(object):
     62      """Base class for translation units.
     63
     64      Our concept of a I{translation unit} is influenced heavily by XLIFF:
     65      U{http://www.oasis-open.org/committees/xliff/documents/xliff-specification.htm}
     66
     67      As such most of the method- and variable names borrows from XLIFF terminology.
     68
     69      A translation unit consists of the following:
     70        - A I{source} string. This is the original translatable text.
     71        - A I{target} string. This is the translation of the I{source}.
     72        - Zero or more I{notes} on the unit. Notes would typically be some
     73          comments from a translator on the unit, or some comments originating from
     74          the source code.
     75        - Zero or more I{locations}. Locations indicate where in the original
     76          source code this unit came from.
     77        - Zero or more I{errors}. Some tools (eg. L{pofilter <filters.pofilter>}) can run checks on
     78          translations and produce error messages.
     79
     80      @group Source: *source*
     81      @group Target: *target*
     82      @group Notes: *note*
     83      @group Locations: *location*
     84      @group Errors: *error*
     85      """
     86
     87      rich_parsers = []
     88      """A list of functions to use for parsing a string into a rich string tree."""
     89
     90 -    def __init__(self, source):
     91          """Constructs a TranslationUnit containing the given source string."""
     92          self.notes = ""
     93          self._store = None
     94          self.source = source
     95          self._target = None
     96          self._rich_source = None
     97          self._rich_target = None
     98
     99 -    def __eq__(self, other):
    100          """Compares two TranslationUnits.
    101
    102          @type other: L{TranslationUnit}
    103          @param other: Another L{TranslationUnit}
    104          @rtype: Boolean
    105          @return: Returns True if the supplied TranslationUnit equals this unit.
    106          """
    107          return self.source == other.source and self.target == other.target
    108
    109 -    def rich_to_multistring(cls, elem_list):
    110          """Convert a "rich" string tree to a C{multistring}:
    111
    112             >>> from translate.storage.placeables.interfaces import X
    113             >>> rich = [StringElem(['foo', X(id='xxx', sub=[' ']), 'bar'])]
    114             >>> TranslationUnit.rich_to_multistring(rich)
    115             multistring(u'foo bar')
    116          """
    117          return multistring([unicode(elem) for elem in elem_list])
    118      rich_to_multistring = classmethod(rich_to_multistring)
    119
    120 -    def multistring_to_rich(cls, mulstring):
    121          """Convert a multistring to a list of "rich" string trees:
    122
    123             >>> target = multistring([u'foo', u'bar', u'baz'])
    124             >>> TranslationUnit.multistring_to_rich(target)
    125             [<StringElem([<StringElem([u'foo'])>])>,
    126              <StringElem([<StringElem([u'bar'])>])>,
    127              <StringElem([<StringElem([u'baz'])>])>]
    128          """
    129          if isinstance(mulstring, multistring):
    130              return [rich_parse(s, cls.rich_parsers) for s in mulstring.strings]
    131          return [rich_parse(mulstring, cls.rich_parsers)]
    132
    133 -    def setsource(self, source):
    134          """Sets the source string to the given value."""
    135          self._rich_source = None
    136          self._source = source
    137      source = property(lambda self: self._source, setsource)
    138
    139 -    def settarget(self, target):
    140          """Sets the target string to the given value."""
    141          self._rich_target = None
    142          self._target = target
    143      target = property(lambda self: self._target, settarget)
    144
    145 -    def _get_rich_source(self):
    146          if self._rich_source is None:
    147              self._rich_source = self.multistring_to_rich(self.source)
    148          return self._rich_source
    149 -    def _set_rich_source(self, value):
    150          if not hasattr(value, '__iter__'):
    151              raise ValueError('value must be iterable')
    152          if len(value) < 1:
    153              raise ValueError('value must have at least one element.')
    154          if not isinstance(value[0], StringElem):
    155              raise ValueError('value[0] must be of type StringElem.')
    156          self._rich_source = list(value)
    157          self.source = self.rich_to_multistring(value)
    158      rich_source = property(_get_rich_source, _set_rich_source)
    159      """ @see: rich_to_multistring
    160          @see: multistring_to_rich"""
    161
    162 -    def _get_rich_target(self):
    163          if self._rich_target is None:
    164              self._rich_target = self.multistring_to_rich(self.target)
    165          return self._rich_target
    166 -    def _set_rich_target(self, value):
    167          if not hasattr(value, '__iter__'):
    168              raise ValueError('value must be iterable')
    169          if len(value) < 1:
    170              raise ValueError('value must have at least one element.')
    171          if not isinstance(value[0], StringElem):
    172              raise ValueError('value[0] must be of type StringElem.')
    173          self._rich_target = list(value)
    174          self.target = self.rich_to_multistring(value)
    175      rich_target = property(_get_rich_target, _set_rich_target)
    176      """ @see: rich_to_multistring
    177          @see: multistring_to_rich"""
    178
    179 -    def gettargetlen(self):
    180          """Returns the length of the target string.
    181
    182          @note: Plural forms might be combined.
    183          @rtype: Integer
    184          """
    185          length = len(self.target or "")
    186          strings = getattr(self.target, "strings", [])
    187          if strings:
    188              length += sum([len(pluralform) for pluralform in strings[1:]])
    189          return length
    190
    191 -    def getid(self):
    192          """A unique identifier for this unit.
    193
    194          @rtype: string
    195          @return: an identifier for this unit that is unique in the store
    196
    197          Derived classes should override this in a way that guarantees a unique
    198          identifier for each unit in the store.
    199          """
    200          return self.source
    201
    202 -    def getlocations(self):
    203          """A list of source code locations.
    204
    205          @note: Shouldn't be implemented if the format doesn't support it.
    206          @rtype: List
    207          """
    208          return []
    209
    210 -    def addlocation(self, location):
    211          """Add one location to the list of locations.
    212
    213          @note: Shouldn't be implemented if the format doesn't support it.
    214          """
    215          pass
    216
    217 -    def addlocations(self, location):
    218          """Add a location or a list of locations.
    219
    220          @note: Most classes shouldn't need to implement this,
    221                 but should rather implement L{addlocation()}.
    222          @warning: This method might be removed in future.
    223          """
    224          if isinstance(location, list):
    225              for item in location:
    226                  self.addlocation(item)
    227          else:
    228              self.addlocation(location)
    229
    230 -    def getcontext(self):
    231          """Get the message context."""
    232          return ""
    233
    234 -    def getnotes(self, origin=None):
    235          """Returns all notes about this unit.
    236
    237          It will probably be freeform text or something reasonable that can be
    238          synthesised by the format.
    239          It should not include location comments (see L{getlocations()}).
    240          """
    241          return getattr(self, "notes", "")
    242
    243 -    def addnote(self, text, origin=None):
    244          """Adds a note (comment).
    245
    246          @type text: string
    247          @param text: Usually just a sentence or two.
    248          @type origin: string
    249          @param origin: Specifies who/where the comment comes from.
    250                         Origin can be one of the following text strings:
    251                           - 'translator'
    252                           - 'developer', 'programmer', 'source code' (synonyms)
    253          """
    254          if getattr(self, "notes", None):
    255              self.notes += '\n'+text
    256          else:
    257              self.notes = text
    258
    259 -    def removenotes(self):
    260          """Remove all the translator's notes."""
    261          self.notes = u''
    262
    263 -    def adderror(self, errorname, errortext):
    264          """Adds an error message to this unit.
    265
    266          @type errorname: string
    267          @param errorname: A single word to id the error.
    268          @type errortext: string
    269          @param errortext: The text describing the error.
    270          """
    271          pass
    272
    273 -    def geterrors(self):
    274          """Get all error messages.
    275
    276          @rtype: Dictionary
    277          """
    278          return {}
    279
    280 -    def markreviewneeded(self, needsreview=True, explanation=None):
    281          """Marks the unit to indicate whether it needs review.
    282
    283          @keyword needsreview: Defaults to True.
    284          @keyword explanation: Adds an optional explanation as a note.
    285          """
    286          pass
    287
    288 -    def istranslated(self):
    289          """Indicates whether this unit is translated.
    290
    291          This should be used rather than deducing it from .target,
    292          to ensure that other classes can implement more functionality
    293          (as XLIFF does).
    294          """
    295          return bool(self.target) and not self.isfuzzy()
    296
    297 -    def istranslatable(self):
    298          """Indicates whether this unit can be translated.
    299
    300          This should be used to distinguish real units for translation from
    301          header, obsolete, binary or other blank units.
    302          """
    303          return True
    304
    305 -    def isfuzzy(self):
    306          """Indicates whether this unit is fuzzy."""
    307          return False
    308
    309 -    def markfuzzy(self, value=True):
    310          """Marks the unit as fuzzy or not."""
    311          pass
    312
    313 -    def isheader(self):
    314          """Indicates whether this unit is a header."""
    315          return False
    316
    317 -    def isreview(self):
    318          """Indicates whether this unit needs review."""
    319          return False
    320
    321 -    def isblank(self):
    322          """Used to see if this unit has no source or target string.
    323
    324          @note: This is probably used more to find translatable units,
    325          and we might want to move in that direction rather and get rid of this.
    326          """
    327          return not (self.source or self.target)
    328
    329 -    def hasplural(self):
    330          """Tells whether or not this specific unit has plural strings."""
    331          #TODO: Reconsider
    332          return False
    333
    334 -    def getsourcelanguage(self):
    335          return getattr(self._store, "sourcelanguage", "en")
    336
    337 -    def gettargetlanguage(self):
    338          return getattr(self._store, "targetlanguage", None)
    339
    340 -    def merge(self, otherunit, overwrite=False, comments=True):
    341          """Do basic format agnostic merging."""
    342          if not self.target or overwrite:
    343              self.rich_target = otherunit.rich_target
    344
    345 -    def unit_iter(self):
    346          """Iterator that only returns this unit."""
    347          yield self
    348
    349 -    def getunits(self):
    350          """This unit in a list."""
    351          return [self]
    352
    353 -    def buildfromunit(cls, unit):
    354          """Build a native unit from a foreign unit, preserving as much
    355          information as possible."""
    356          if type(unit) == cls and hasattr(unit, "copy") and callable(unit.copy):
    357              return unit.copy()
    358          newunit = cls(unit.source)
    359          newunit.target = unit.target
    360          newunit.markfuzzy(unit.isfuzzy())
    361          locations = unit.getlocations()
    362          if locations:
    363              newunit.addlocations(locations)
    364          notes = unit.getnotes()
    365          if notes:
    366              newunit.addnote(notes)
    367          return newunit
    368      buildfromunit = classmethod(buildfromunit)
    369
    370      xid = property(lambda self: None, lambda self, value: None)
    371      rid = property(lambda self: None, lambda self, value: None)
    372
    373
    374 -class TranslationStore(object):
    375      """Base class for stores for multiple translation units of type UnitClass."""
    376
    377      UnitClass = TranslationUnit
    378      """The class of units that will be instantiated and used by this class"""
    379      Name = "Base translation store"
    380      """The human usable name of this store type"""
    381      Mimetypes = None
    382      """A list of MIME types associated with this store type"""
    383      Extensions = None
    384      """A list of file extentions associated with this store type"""
    385      _binary = False
    386      """Indicates whether a file should be accessed as a binary file."""
    387      suggestions_in_format = False
    388      """Indicates if format can store suggestions and alternative translation for a unit"""
    389
    390 -    def __init__(self, unitclass=None):
    391          """Constructs a blank TranslationStore."""
    392          self.units = []
    393          self.sourcelanguage = None
    394          self.targetlanguage = None
    395          if unitclass:
    396              self.UnitClass = unitclass
    397          super(TranslationStore, self).__init__()
    398
    399 -    def getsourcelanguage(self):
    400          """Gets the source language for this store"""
    401          return self.sourcelanguage
    402
    403 -    def setsourcelanguage(self, sourcelanguage):
    404          """Sets the source language for this store"""
    405          self.sourcelanguage = sourcelanguage
    406
    407 -    def gettargetlanguage(self):
    408          """Gets the target language for this store"""
    409          return self.targetlanguage
    410
    411 -    def settargetlanguage(self, targetlanguage):
    412          """Sets the target language for this store"""
    413          self.targetlanguage = targetlanguage
    414
    415 -    def unit_iter(self):
    416          """Iterator over all the units in this store."""
    417          for unit in self.units:
    418              yield unit
    419
    420 -    def getunits(self):
    421          """Return a list of all units in this store."""
    422          return [unit for unit in self.unit_iter()]
    423
    424 -    def addunit(self, unit):
    425          """Appends the given unit to the object's list of units.
    426
    427          This method should always be used rather than trying to modify the
    428          list manually.
    429
    430          @type unit: L{TranslationUnit}
    431          @param unit: The unit that will be added.
    432          """
    433          unit._store = self
    434          self.units.append(unit)
    435
    436 -    def addsourceunit(self, source):
    437          """Adds and returns a new unit with the given source string.
    438
    439          @rtype: L{TranslationUnit}
    440          """
    441          unit = self.UnitClass(source)
    442          self.addunit(unit)
    443          return unit
    444
    445 -    def findid(self, id):
    446          """find unit with matching id by checking id_index"""
    447          self.require_index()
    448          return self.id_index.get(id, None)
    449
    450 -    def findunit(self, source):
    451          """Finds the unit with the given source string.
    452
    453          @rtype: L{TranslationUnit} or None
    454          """
    455          if len(getattr(self, "sourceindex", [])):
    456              if source in self.sourceindex:
    457                  return self.sourceindex[source][0]
    458          else:
    459              for unit in self.units:
    460                  if unit.source == source:
    461                      return unit
    462          return None
    463
    464
    465 -    def findunits(self, source):
    466          """Finds the unit with the given source string.
    467
    468          @rtype: L{TranslationUnit} or None
    469          """
    470          if len(getattr(self, "sourceindex", [])):
    471              if source in self.sourceindex:
    472                  return self.sourceindex[source]
    473          else:
    474              #FIXME: maybe we should generate index here instead since
    475              #we'll scan all units anyway
    476              result = []
    477              for unit in self.units:
    478                  if unit.source == source:
    479                      result.append(unit)
    480              return result
    481          return None
    482
    483 -    def translate(self, source):
    484          """Returns the translated string for a given source string.
    485
    486          @rtype: String or None
    487          """
    488          unit = self.findunit(source)
    489          if unit and unit.target:
    490              return unit.target
    491          else:
    492              return None
    493
    494 -    def remove_unit_from_index(self, unit):
    495          """Remove a unit from source and locaton indexes"""
    496          def remove_unit(source):
    497              if source in self.sourceindex:
    498                  try:
    499                      self.sourceindex[source].remove(unit)
    500                      if len(self.sourceindex[source]) == 0:
    501                          del(self.sourceindex[source])
    502                  except ValueError:
    503                      pass
    504
    505          if unit.hasplural():
    506              for source in unit.source.strings:
    507                  remove_unit(source)
    508          else:
    509              remove_unit(unit.source)
    510
    511          for location in unit.getlocations():
    512              if location in self.locationindex and self.locationindex[location] is not None \
    513                     and self.locationindex[location] == unit:
    514                  del(self.locationindex[location])
    515
    516
    517 -    def add_unit_to_index(self, unit):
    518          """Add a unit to source and location idexes"""
    519          self.id_index[unit.getid()] = unit
    520
    521          def insert_unit(source):
    522              if not source in self.sourceindex:
    523                  self.sourceindex[source] = [unit]
    524              else:
    525                  self.sourceindex[source].append(unit)
    526
    527          if unit.hasplural():
    528              for source in unit.source.strings:
    529                  insert_unit(source)
    530          else:
    531              insert_unit(unit.source)
    532
    533          for location in unit.getlocations():
    534              if location in self.locationindex:
    535                  # if sources aren't unique, don't use them
    536                  #FIXME: maybe better store a list of units like sourceindex
    537                  self.locationindex[location] = None
    538              else:
    539                  self.locationindex[location] = unit
    540
    541 -    def makeindex(self):
    542          """Indexes the items in this store. At least .sourceindex should be usefull."""
    543          self.locationindex = {}
    544          self.sourceindex = {}
    545          self.id_index = {}
    546          for unit in self.units:
    547              # Do we need to test if unit.source exists?
    548              if unit.istranslatable():
    549                  self.add_unit_to_index(unit)
    550
    551 -    def require_index(self):
    552          """make sure source index exists"""
    553          if not hasattr(self, "sourceindex"):
    554              self.makeindex()
    555
    556
    557 -    def __getstate__(self):
    558          odict = self.__dict__.copy()
    559          odict['fileobj'] = None
    560          return odict
    561
    562 -    def __setstate__(self, dict):
    563          self.__dict__.update(dict)
    564          if getattr(self, "filename", False):
    565              self.fileobj = open(self.filename)
    566
    567 -    def __str__(self):
    568          """Converts to a string representation that can be parsed back using L{parsestring()}."""
    569          # We can't pickle fileobj if it is there, so let's hide it for a while.
    570          fileobj = getattr(self, "fileobj", None)
    571          self.fileobj = None
    572          dump = pickle.dumps(self)
    573          self.fileobj = fileobj
    574          return dump
    575
    576 -    def isempty(self):
    577          """Returns True if the object doesn't contain any translation units."""
    578          if len(self.units) == 0:
    579              return True
    580          for unit in self.units:
    581              if unit.istranslatable():
    582                  return False
    583          return True
    584
    585 -    def _assignname(self):
    586          """Tries to work out what the name of the filesystem file is and
    587          assigns it to .filename."""
    588          fileobj = getattr(self, "fileobj", None)
    589          if fileobj:
    590              filename = getattr(fileobj, "name", getattr(fileobj, "filename", None))
    591              if filename:
    592                  self.filename = filename
    593
    594 -    def parsestring(cls, storestring):
    595          """Converts the string representation back to an object."""
    596          newstore = cls()
    597          if storestring:
    598              newstore.parse(storestring)
    599          return newstore
    600      parsestring = classmethod(parsestring)
    601
    602 -    def parse(self, data):
    603          """parser to process the given source string"""
    604          self.units = pickle.loads(data).units
    605
    606 -    def savefile(self, storefile):
    607          """Writes the string representation to the given file (or filename)."""
    608          if isinstance(storefile, basestring):
    609              mode = 'w'
    610              if self._binary:
    611                  mode = 'wb'
    612              storefile = open(storefile, mode)
    613          self.fileobj = storefile
    614          self._assignname()
    615          storestring = str(self)
    616          storefile.write(storestring)
    617          storefile.close()
    618
    619 -    def save(self):
    620          """Save to the file that data was originally read from, if available."""
    621          fileobj = getattr(self, "fileobj", None)
    622          mode = 'w'
    623          if self._binary:
    624              mode = 'wb'
    625          if not fileobj:
    626              filename = getattr(self, "filename", None)
    627              if filename:
    628                  fileobj = file(filename, mode)
    629          else:
    630              fileobj.close()
    631              filename = getattr(fileobj, "name", getattr(fileobj, "filename", None))
    632              if not filename:
    633                  raise ValueError("No file or filename to save to")
    634              fileobj = fileobj.__class__(filename, mode)
    635          self.savefile(fileobj)
    636
    637 -    def parsefile(cls, storefile):
    638          """Reads the given file (or opens the given filename) and parses back to an object."""
    639          mode = 'r'
    640          if cls._binary:
    641              mode = 'rb'
    642          if isinstance(storefile, basestring):
    643              storefile = open(storefile, mode)
    644          mode = getattr(storefile, "mode", mode)
    645          #For some reason GzipFile returns 1, so we have to test for that here
    646          if mode == 1 or "r" in mode:
    647              storestring = storefile.read()
    648              storefile.close()
    649          else:
    650              storestring = ""
    651          newstore = cls.parsestring(storestring)
    652          newstore.fileobj = storefile
    653          newstore._assignname()
    654          return newstore
    655      parsefile = classmethod(parsefile)
    656