Package translate :: Package search :: Package indexing :: Module PyLuceneIndexer

[hide private]
[frames] | no frames]

# Source Code for Module translate.search.indexing.PyLuceneIndexer

      1  # -*- coding: utf-8 -*-
      2  #
      3  # Copyright 2008 Zuza Software Foundation
      4  #
      5  # This file is part of translate.
      6  #
      7  # translate is free software; you can redistribute it and/or modify
      8  # it under the terms of the GNU General Public License as published by
      9  # the Free Software Foundation; either version 2 of the License, or
     10  # (at your option) any later version.
     11  #
     12  # translate is distributed in the hope that it will be useful,
     13  # but WITHOUT ANY WARRANTY; without even the implied warranty of
     14  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     15  # GNU General Public License for more details.
     16  #
     17  # You should have received a copy of the GNU General Public License
     18  # along with translate; if not, write to the Free Software
     19  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
     20  #
     21
     22
     23  """
     24  interface for the PyLucene (v2.x) indexing engine
     25
     26  take a look at PyLuceneIndexer1.py for the PyLucene v1.x interface
     27  """
     28
     29  __revision__ = "$Id: PyLuceneIndexer.py 14174 2010-04-08 23:14:10Z alaaosh $"
     30
     31  import CommonIndexer
     32  import re
     33  import os
     34  import time
     35  import logging
     36
     37  # try to import the PyLucene package (with the two possible names)
     38  # remember the type of the detected package (compiled with jcc (>=v2.3) or
     39  # with gcj (<=v2.2)
     40  try:
     41      import PyLucene
     42      _COMPILER = 'gcj'
     43  except ImportError:
     44      # if this fails, then there is no pylucene installed
     45      import lucene
     46      PyLucene = lucene
     47      PyLucene.initVM(PyLucene.CLASSPATH)
     48      _COMPILER = 'jcc'
     49
     50
     51  UNNAMED_FIELD_NAME = "FieldWithoutAName"
     52  MAX_FIELD_SIZE = 1048576
     53
     54
     55 -def is_available():
     56      return _get_pylucene_version() == 2
     57
     58
     59 -class PyLuceneDatabase(CommonIndexer.CommonDatabase):
     60      """manage and use a pylucene indexing database"""
     61
     62      QUERY_TYPE = PyLucene.Query
     63      INDEX_DIRECTORY_NAME = "lucene"
     64
     65 -    def __init__(self, basedir, analyzer=None, create_allowed=True):
     66          """initialize or open an indexing database
     67
     68          Any derived class must override __init__.
     69
     70          @raise ValueError: the given location exists, but the database type
     71                  is incompatible (e.g. created by a different indexing engine)
     72          @raise OSError: the database failed to initialize
     73
     74          @param basedir: the parent directory of the database
     75          @type basedir: str
     76          @param analyzer: bitwise combination of possible analyzer flags
     77              to be used as the default analyzer for this database. Leave it empty
     78              to use the system default analyzer (self.ANALYZER_DEFAULT).
     79              see self.ANALYZER_TOKENIZE, self.ANALYZER_PARTIAL, ...
     80          @type analyzer: int
     81          @param create_allowed: create the database, if necessary; default: True
     82          @type create_allowed: bool
     83          """
     84          jvm = PyLucene.getVMEnv()
     85          jvm.attachCurrentThread()
     86          super(PyLuceneDatabase, self).__init__(basedir, analyzer=analyzer,
     87                  create_allowed=create_allowed)
     88          self.pyl_analyzer = PyLucene.StandardAnalyzer()
     89          self.writer = None
     90          self.reader = None
     91          self.index_version = None
     92          try:
     93              # try to open an existing database
     94              tempreader = PyLucene.IndexReader.open(self.location)
     95              tempreader.close()
     96          except PyLucene.JavaError, err_msg:
     97              # Write an error out, in case this is a real problem instead of an absence of an index
     98              # TODO: turn the following two lines into debug output
     99              #errorstr = str(e).strip() + "\n" + self.errorhandler.traceback_str()
    100              #DEBUG_FOO("could not open index, so going to create: " + errorstr)
    101              # Create the index, so we can open cached readers on it
    102              if not create_allowed:
    103                  raise OSError("Indexer: skipping database creation")
    104              try:
    105                  # create the parent directory if it does not exist
    106                  parent_path = os.path.dirname(self.location)
    107                  if not os.path.isdir(parent_path):
    108                      # recursively create all directories up to parent_path
    109                      os.makedirs(parent_path)
    110              except IOError, err_msg:
    111                  raise OSError("Indexer: failed to create the parent " \
    112                          + "directory (%s) of the indexing database: %s" \
    113                          % (parent_path, err_msg))
    114              try:
    115                  tempwriter = PyLucene.IndexWriter(self.location,
    116                          self.pyl_analyzer, True)
    117                  tempwriter.close()
    118              except PyLucene.JavaError, err_msg:
    119                  raise OSError("Indexer: failed to open or create a Lucene" \
    120                          + " database (%s): %s" % (self.location, err_msg))
    121          # the indexer is initialized - now we prepare the searcher
    122          # windows file locking seems inconsistent, so we try 10 times
    123          numtries = 0
    124          #self.dir_lock.acquire(blocking=True)
    125          # read "self.reader", "self.indexVersion" and "self.searcher"
    126          try:
    127              while numtries < 10:
    128                  try:
    129                      self.reader = PyLucene.IndexReader.open(self.location)
    130                      self.indexVersion = self.reader.getCurrentVersion(
    131                          self.location)
    132                      self.searcher = PyLucene.IndexSearcher(self.reader)
    133                      break
    134                  except PyLucene.JavaError, e:
    135                      # store error message for possible later re-raise (below)
    136                      lock_error_msg = e
    137                      time.sleep(0.01)
    138                      numtries += 1
    139              else:
    140                  # locking failed for 10 times
    141                  raise OSError("Indexer: failed to lock index database" \
    142                                + " (%s)" % lock_error_msg)
    143          finally:
    144              pass
    145          #    self.dir_lock.release()
    146          # initialize the searcher and the reader
    147          self._index_refresh()
    148
    149 -    def __del__(self):
    150          """remove lock and close writer after loosing the last reference"""
    151          self._writer_close()
    152
    153 -    def flush(self, optimize=False):
    154          """flush the content of the database - to force changes to be written
    155          to disk
    156
    157          some databases also support index optimization
    158
    159          @param optimize: should the index be optimized if possible?
    160          @type optimize: bool
    161          """
    162          if self._writer_is_open():
    163              try:
    164                  if optimize:
    165                      self.writer.optimize()
    166              finally:
    167                  # close the database even if optimizing failed
    168                  self._writer_close()
    169          # the reader/searcher needs an update, too
    170          self._index_refresh()
    171
    172 -    def _create_query_for_query(self, query):
    173          """generate a query based on an existing query object
    174
    175          basically this function should just create a copy of the original
    176
    177          @param query: the original query object
    178          @type query: PyLucene.Query
    179          @return: resulting query object
    180          @rtype: PyLucene.Query
    181          """
    182          # TODO: a deep copy or a clone would be safer
    183          # somehow not working (returns "null"): copy.deepcopy(query)
    184          return query
    185
    186 -    def _create_query_for_string(self, text, require_all=True,
    187              analyzer=None):
    188          """generate a query for a plain term of a string query
    189
    190          basically this function parses the string and returns the resulting
    191          query
    192
    193          @param text: the query string
    194          @type text: str
    195          @param require_all: boolean operator
    196              (True -> AND (default) / False -> OR)
    197          @type require_all: bool
    198          @param analyzer: the analyzer to be used
    199              possible analyzers are:
    200               -  L{CommonDatabase.ANALYZER_TOKENIZE}
    201                      the field value is splitted to be matched word-wise
    202               -  L{CommonDatabase.ANALYZER_PARTIAL}
    203                      the field value must start with the query string
    204               -  L{CommonDatabase.ANALYZER_EXACT}
    205                      keep special characters and the like
    206          @type analyzer: bool
    207          @return: resulting query object
    208          @rtype: PyLucene.Query
    209          """
    210          if analyzer is None:
    211              analyzer = self.analyzer
    212          if analyzer == self.ANALYZER_EXACT:
    213              analyzer_obj = PyLucene.KeywordAnalyzer()
    214          else:
    215              text = _escape_term_value(text)
    216              analyzer_obj = PyLucene.StandardAnalyzer()
    217          qp = PyLucene.QueryParser(UNNAMED_FIELD_NAME, analyzer_obj)
    218          if (analyzer & self.ANALYZER_PARTIAL > 0):
    219              # PyLucene uses explicit wildcards for partial matching
    220              text += "*"
    221          if require_all:
    222              qp.setDefaultOperator(qp.Operator.AND)
    223          else:
    224              qp.setDefaultOperator(qp.Operator.OR)
    225          return qp.parse(text)
    226
    227 -    def _create_query_for_field(self, field, value, analyzer=None):
    228          """generate a field query
    229
    230          this functions creates a field->value query
    231
    232          @param field: the fieldname to be used
    233          @type field: str
    234          @param value: the wanted value of the field
    235          @type value: str
    236          @param analyzer: the analyzer to be used
    237              possible analyzers are:
    238                - L{CommonDatabase.ANALYZER_TOKENIZE}
    239                      the field value is splitted to be matched word-wise
    240                - L{CommonDatabase.ANALYZER_PARTIAL}
    241                      the field value must start with the query string
    242                - L{CommonDatabase.ANALYZER_EXACT}
    243                      keep special characters and the like
    244          @type analyzer: bool
    245          @return: resulting query object
    246          @rtype: PyLucene.Query
    247          """
    248          if analyzer is None:
    249              analyzer = self.analyzer
    250          if analyzer == self.ANALYZER_EXACT:
    251              analyzer_obj = PyLucene.KeywordAnalyzer()
    252          else:
    253              value = _escape_term_value(value)
    254              analyzer_obj = PyLucene.StandardAnalyzer()
    255          qp = PyLucene.QueryParser(field, analyzer_obj)
    256          if (analyzer & self.ANALYZER_PARTIAL > 0):
    257              # PyLucene uses explicit wildcards for partial matching
    258              value += "*"
    259          return qp.parse(value)
    260
    261 -    def _create_query_combined(self, queries, require_all=True):
    262          """generate a combined query
    263
    264          @param queries: list of the original queries
    265          @type queries: list of PyLucene.Query
    266          @param require_all: boolean operator
    267              (True -> AND (default) / False -> OR)
    268          @type require_all: bool
    269          @return: the resulting combined query object
    270          @rtype: PyLucene.Query
    271          """
    272          combined_query = PyLucene.BooleanQuery()
    273          for query in queries:
    274              combined_query.add(
    275                      PyLucene.BooleanClause(query, _occur(require_all, False)))
    276          return combined_query
    277
    278 -    def _create_empty_document(self):
    279          """create an empty document to be filled and added to the index later
    280
    281          @return: the new document object
    282          @rtype: PyLucene.Document
    283          """
    284          return PyLucene.Document()
    285
    286 -    def _add_plain_term(self, document, term, tokenize=True):
    287          """add a term to a document
    288
    289          @param document: the document to be changed
    290          @type document: PyLucene.Document
    291          @param term: a single term to be added
    292          @type term: str
    293          @param tokenize: should the term be tokenized automatically
    294          @type tokenize: bool
    295          """
    296          if tokenize:
    297              token_flag = PyLucene.Field.Index.TOKENIZED
    298          else:
    299              token_flag = PyLucene.Field.Index.UN_TOKENIZED
    300          document.add(PyLucene.Field(str(UNNAMED_FIELD_NAME), term,
    301                  PyLucene.Field.Store.YES, token_flag))
    302
    303 -    def _add_field_term(self, document, field, term, tokenize=True):
    304          """add a field term to a document
    305
    306          @param document: the document to be changed
    307          @type document: PyLucene.Document
    308          @param field: name of the field
    309          @type field: str
    310          @param term: term to be associated to the field
    311          @type term: str
    312          @param tokenize: should the term be tokenized automatically
    313          @type tokenize: bool
    314          """
    315          if tokenize:
    316              token_flag = PyLucene.Field.Index.TOKENIZED
    317          else:
    318              token_flag = PyLucene.Field.Index.UN_TOKENIZED
    319          document.add(PyLucene.Field(str(field), term,
    320                  PyLucene.Field.Store.YES, token_flag))
    321
    322 -    def _add_document_to_index(self, document):
    323          """add a prepared document to the index database
    324
    325          @param document: the document to be added
    326          @type document: PyLucene.Document
    327          """
    328          self._writer_open()
    329          self.writer.addDocument(document)
    330
    331 -    def begin_transaction(self):
    332          """PyLucene does not support transactions
    333
    334          Thus this function just opens the database for write access.
    335          Call "cancel_transaction" or "commit_transaction" to close write
    336          access in order to remove the exclusive lock from the database
    337          directory.
    338          """
    339          self._writer_open()
    340
    341 -    def cancel_transaction(self):
    342          """PyLucene does not support transactions
    343
    344          Thus this function just closes the database write access and removes
    345          the exclusive lock.
    346
    347          See 'start_transaction' for details.
    348          """
    349          self._writer_close()
    350
    351 -    def commit_transaction(self):
    352          """PyLucene does not support transactions
    353
    354          Thus this function just closes the database write access and removes
    355          the exclusive lock.
    356
    357          See 'start_transaction' for details.
    358          """
    359          self._writer_close()
    360          self._index_refresh()
    361
    362 -    def get_query_result(self, query):
    363          """return an object containing the results of a query
    364
    365          @param query: a pre-compiled query
    366          @type query: a query object of the real implementation
    367          @return: an object that allows access to the results
    368          @rtype: subclass of CommonEnquire
    369          """
    370          return PyLuceneHits(self.searcher.search(query))
    371
    372 -    def delete_document_by_id(self, docid):
    373          """delete a specified document
    374
    375          @param docid: the document ID to be deleted
    376          @type docid: int
    377          """
    378          self._delete_stale_lock()
    379          self.reader.deleteDocument(docid)
    380          self.reader.flush()
    381          # TODO: check the performance impact of calling "refresh" for each id
    382          self._index_refresh()
    383
    384 -    def search(self, query, fieldnames):
    385          """return a list of the contents of specified fields for all matches of
    386          a query
    387
    388          @param query: the query to be issued
    389          @type query: a query object of the real implementation
    390          @param fieldnames: the name(s) of a field of the document content
    391          @type fieldnames: string | list of strings
    392          @return: a list of dicts containing the specified field(s)
    393          @rtype: list of dicts
    394          """
    395          if isinstance(fieldnames, basestring):
    396              fieldnames = [fieldnames]
    397          hits = self.searcher.search(query)
    398          if _COMPILER == 'jcc':
    399              # add the ranking number and the retrieved document to the array
    400              hits = [(hit, hits.doc(hit)) for hit in range(hits.length())]
    401          result = []
    402          for hit, doc in hits:
    403              fields = {}
    404              for fieldname in fieldnames:
    405                  # take care for the special field "None"
    406                  if fieldname is None:
    407                      pyl_fieldname = UNNAMED_FIELD_NAME
    408                  else:
    409                      pyl_fieldname = fieldname
    410                  fields[fieldname] = doc.getValues(pyl_fieldname)
    411              result.append(fields)
    412          return result
    413
    414 -    def _delete_stale_lock(self):
    415          if self.reader.isLocked(self.location):
    416              #HACKISH: there is a lock but Lucene api can't tell us how old it
    417              # is, will have to check the filesystem
    418              try:
    419                  # in try block just in case lock disappears on us while testing it
    420                  stat = os.stat(os.path.join(self.location, 'write.lock'))
    421                  age = (time.time() - stat.st_mtime) / 60
    422                  if age > 15:
    423                      logging.warning("stale lock found in %s, removing.", self.location)
    424                      self.reader.unlock(self.reader.directory())
    425              except:
    426                  pass
    427
    428 -    def _writer_open(self):
    429          """open write access for the indexing database and acquire an
    430          exclusive lock
    431          """
    432          if not self._writer_is_open():
    433              self._delete_stale_lock()
    434              self.writer = PyLucene.IndexWriter(self.location, self.pyl_analyzer,
    435                      False)
    436              # "setMaxFieldLength" is available since PyLucene v2
    437              # we must stay compatible to v1 for the derived class
    438              # (PyLuceneIndexer1) - thus we make this step optional
    439              if hasattr(self.writer, "setMaxFieldLength"):
    440                  self.writer.setMaxFieldLength(MAX_FIELD_SIZE)
    441          # do nothing, if it is already open
    442
    443 -    def _writer_close(self):
    444          """close indexing write access and remove the database lock"""
    445          if self._writer_is_open():
    446              self.writer.commit()
    447              self.writer.close()
    448              self.writer = None
    449
    450 -    def _writer_is_open(self):
    451          """check if the indexing write access is currently open"""
    452          return not self.writer is None
    453
    454 -    def _index_refresh(self):
    455          """re-read the indexer database"""
    456          try:
    457              if self.reader is None or self.searcher is None:
    458                  self.reader = PyLucene.IndexReader.open(self.location)
    459                  self.searcher = PyLucene.IndexSearcher(self.reader)
    460              elif self.index_version != self.reader.getCurrentVersion( \
    461                      self.location):
    462                  self.searcher.close()
    463                  self.reader.close()
    464                  self.reader = PyLucene.IndexReader.open(self.location)
    465                  self.searcher = PyLucene.IndexSearcher(self.reader)
    466                  self.index_version = self.reader.getCurrentVersion(self.location)
    467          except PyLucene.JavaError, e:
    468              # TODO: add some debugging output?
    469              #self.errorhandler.logerror("Error attempting to read index - try reindexing: "+str(e))
    470              pass
    471
    472
    473 -class PyLuceneHits(CommonIndexer.CommonEnquire):
    474      """an enquire object contains the information about the result of a request
    475      """
    476
    477 -    def get_matches(self, start, number):
    478          """return a specified number of qualified matches of a previous query
    479
    480          @param start: index of the first match to return (starting from zero)
    481          @type start: int
    482          @param number: the number of matching entries to return
    483          @type number: int
    484          @return: a set of matching entries and some statistics
    485          @rtype: tuple of (returned number, available number, matches)
    486                  "matches" is a dictionary of::
    487                      ["rank", "percent", "document", "docid"]
    488          """
    489          # check if requested results do not exist
    490          # stop is the lowest index number to be ommitted
    491          stop = start + number
    492          if stop > self.enquire.length():
    493              stop = self.enquire.length()
    494          # invalid request range
    495          if stop <= start:
    496              return (0, self.enquire.length(), [])
    497          result = []
    498          for index in range(start, stop):
    499              item = {}
    500              item["rank"] = index
    501              item["docid"] = self.enquire.id(index)
    502              item["percent"] = self.enquire.score(index)
    503              item["document"] = self.enquire.doc(index)
    504              result.append(item)
    505          return (stop-start, self.enquire.length(), result)
    506
    507 -def _occur(required, prohibited):
    508      if required == True and prohibited == False:
    509          return PyLucene.BooleanClause.Occur.MUST
    510      elif required == False and prohibited == False:
    511          return PyLucene.BooleanClause.Occur.SHOULD
    512      elif required == False and prohibited == True:
    513          return PyLucene.BooleanClause.Occur.MUST_NOT
    514      else:
    515          # It is an error to specify a clause as both required
    516          # and prohibited
    517          return None
    518
    519 -def _get_pylucene_version():
    520      """get the installed pylucene version
    521
    522      @return: 1 -> PyLucene v1.x / 2 -> PyLucene v2.x / 0 -> unknown
    523      @rtype: int
    524      """
    525      version = PyLucene.VERSION
    526      if version.startswith("1."):
    527          return 1
    528      elif version.startswith("2."):
    529          return 2
    530      else:
    531          return 0
    532
    533
    534 -def _escape_term_value(text):
    535      return re.sub("\*", "", text)
    536