Package translate :: Package convert :: Module convert

[hide private]
[frames] | no frames]

# Source Code for Module translate.convert.convert

      1  #!/usr/bin/env python
      2  # -*- coding: utf-8 -*-
      3  #
      4  # Copyright 2004-2006 Zuza Software Foundation
      5  #
      6  # This file is part of translate.
      7  #
      8  # translate is free software; you can redistribute it and/or modify
      9  # it under the terms of the GNU General Public License as published by
     10  # the Free Software Foundation; either version 2 of the License, or
     11  # (at your option) any later version.
     12  #
     13  # translate is distributed in the hope that it will be useful,
     14  # but WITHOUT ANY WARRANTY; without even the implied warranty of
     15  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     16  # GNU General Public License for more details.
     17  #
     18  # You should have received a copy of the GNU General Public License
     19  # along with translate; if not, write to the Free Software
     20  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
     21
     22  """Handles converting of files between formats (used by translate.convert tools)"""
     23
     24  import os.path
     25  from translate.misc import optrecurse
     26  # don't import optparse ourselves, get the version from optrecurse
     27  optparse = optrecurse.optparse
     28  try:
     29      from cStringIO import StringIO
     30  except ImportError:
     31      from StringIO import StringIO
     32
     33 -class ConvertOptionParser(optrecurse.RecursiveOptionParser, object):
     34      """a specialized Option Parser for convertor tools..."""
     35 -    def __init__(self, formats, usetemplates=False, usepots=False, allowmissingtemplate=False, description=None):
     36          """construct the specialized Option Parser"""
     37          optrecurse.RecursiveOptionParser.__init__(self, formats, usetemplates,
     38                  allowmissingtemplate=allowmissingtemplate, description=description)
     39          self.usepots = usepots
     40          self.setpotoption()
     41          self.set_usage()
     42
     43 -    def add_fuzzy_option(self, default=False):
     44          """adds an option to include / exclude fuzzy translations"""
     45          fuzzyhelp = "use translations marked fuzzy"
     46          nofuzzyhelp = "don't use translations marked fuzzy"
     47          if default:
     48              fuzzyhelp += " (default)"
     49          else:
     50              nofuzzyhelp += " (default)"
     51          self.add_option("", "--fuzzy", dest="includefuzzy", action="store_true", default=default, help=fuzzyhelp)
     52          self.add_option("", "--nofuzzy", dest="includefuzzy", action="store_false", default=default, help=nofuzzyhelp)
     53          self.passthrough.append("includefuzzy")
     54
     55 -    def add_duplicates_option(self, default="msgctxt"):
     56          """adds an option to say what to do with duplicate strings"""
     57          self.add_option("", "--duplicates", dest="duplicatestyle", default=default,
     58              type="choice", choices=["msgid_comment", "msgctxt", "merge", "keep", "msgid_comment_all"],
     59              help="what to do with duplicate strings (identical source text): merge, msgid_comment, msgctxt, keep, msgid_comment_all (default: '%s')" % default, metavar="DUPLICATESTYLE")
     60          self.passthrough.append("duplicatestyle")
     61
     62 -    def add_multifile_option(self, default="single"):
     63          """adds an option to say how to split the po/pot files"""
     64          self.add_option("", "--multifile", dest="multifilestyle", default=default,
     65              type="choice", choices=["single", "toplevel", "onefile"],
     66              help="how to split po/pot files (single, toplevel or onefile)", metavar="MULTIFILESTYLE")
     67          self.passthrough.append("multifilestyle")
     68
     69 -    def potifyformat(self, fileformat):
     70          """converts a .po to a .pot where required"""
     71          if fileformat is None:
     72              return fileformat
     73          elif fileformat == "po":
     74              return "pot"
     75          elif fileformat.endswith(os.extsep + "po"):
     76              return fileformat + "t"
     77          else:
     78              return fileformat
     79
     80 -    def getformathelp(self, formats):
     81          """make a nice help string for describing formats..."""
     82          # include implicit pot options...
     83          helpformats = []
     84          for fileformat in formats:
     85              helpformats.append(fileformat)
     86              potformat = self.potifyformat(fileformat)
     87              if potformat != fileformat:
     88                  helpformats.append(potformat)
     89          return super(ConvertOptionParser, self).getformathelp(helpformats)
     90
     91 -    def filterinputformats(self, options):
     92          """filters input formats, processing relevant switches in options"""
     93          if self.usepots and options.pot:
     94              return [self.potifyformat(inputformat) for inputformat in self.inputformats]
     95          else:
     96              return self.inputformats
     97
     98 -    def filteroutputoptions(self, options):
     99          """filters output options, processing relevant switches in options"""
    100          if self.usepots and options.pot:
    101              outputoptions = {}
    102              for (inputformat, templateformat), (outputformat, convertor) in self.outputoptions.iteritems():
    103                  inputformat = self.potifyformat(inputformat)
    104                  templateformat = self.potifyformat(templateformat)
    105                  outputformat = self.potifyformat(outputformat)
    106                  outputoptions[(inputformat, templateformat)] = (outputformat, convertor)
    107              return outputoptions
    108          else:
    109              return self.outputoptions
    110
    111 -    def setpotoption(self):
    112          """sets the -P/--pot option depending on input/output formats etc"""
    113          if self.usepots:
    114              potoption = optparse.Option("-P", "--pot", \
    115                      action="store_true", dest="pot", default=False, \
    116                      help="output PO Templates (.pot) rather than PO files (.po)")
    117              self.define_option(potoption)
    118
    119 -    def verifyoptions(self, options):
    120          """verifies that the options are valid (required options are present, etc)"""
    121          pass
    122
    123 -    def run(self, argv=None):
    124          """parses the command line options and runs the conversion"""
    125          (options, args) = self.parse_args(argv)
    126          options.inputformats = self.filterinputformats(options)
    127          options.outputoptions = self.filteroutputoptions(options)
    128          self.usepsyco(options)
    129          self.verifyoptions(options)
    130          self.recursiveprocess(options)
    131
    132 -def copyinput(inputfile, outputfile, templatefile, **kwargs):
    133      """copies the input file to the output file"""
    134      outputfile.write(inputfile.read())
    135      return True
    136
    137 -def copytemplate(inputfile, outputfile, templatefile, **kwargs):
    138      """copies the template file to the output file"""
    139      outputfile.write(templatefile.read())
    140      return True
    141
    142 -class Replacer:
    143      """an object that knows how to replace strings in files"""
    144 -    def __init__(self, searchstring, replacestring):
    145          self.searchstring = searchstring
    146          self.replacestring = replacestring
    147
    148 -    def doreplace(self, text):
    149          """actually replace the text"""
    150          if self.searchstring is not None and self.replacestring is not None:
    151              return text.replace(self.searchstring, self.replacestring)
    152          else:
    153              return text
    154
    155 -    def searchreplaceinput(self, inputfile, outputfile, templatefile, **kwargs):
    156          """copies the input file to the output file, searching and replacing"""
    157          outputfile.write(self.doreplace(inputfile.read()))
    158          return True
    159
    160 -    def searchreplacetemplate(self, inputfile, outputfile, templatefile, **kwargs):
    161          """copies the template file to the output file, searching and replacing"""
    162          outputfile.write(self.doreplace(templatefile.read()))
    163          return True
    164
    165  # archive files need to know how to:
    166  # - openarchive: creates an archive object for the archivefilename
    167  #   * requires a constructor that takes the filename
    168  # - iterarchivefile: iterate through the names in the archivefile
    169  #   * requires the default iterator to do this
    170  # - archivefileexists: check if a given pathname exists inside the archivefile
    171  #   * uses the in operator - requires __contains__ (or will use __iter__ by default)
    172  # - openarchiveinputfile: returns an open input file from the archive, given the path
    173  #   * requires an archivefile.openinputfile method that takes the pathname
    174  # - openarchiveoutputfile: returns an open output file from the archive, given the path
    175  #   * requires an archivefile.openoutputfile method that takes the pathname
    176
    177 -class ArchiveConvertOptionParser(ConvertOptionParser):
    178      """ConvertOptionParser that can handle recursing into single archive files.
    179      archiveformats maps extension to class. if the extension doesn't matter, it can be None.
    180      if the extension is only valid for input/output/template, it can be given as (extension, filepurpose)"""
    181 -    def __init__(self, formats, usetemplates=False, usepots=False, description=None, archiveformats=None):
    182          if archiveformats is None:
    183              self.archiveformats = {}
    184          else:
    185              self.archiveformats = archiveformats
    186          self.archiveoptions = {}
    187          ConvertOptionParser.__init__(self, formats, usetemplates, usepots, description=description)
    188
    189 -    def setarchiveoptions(self, **kwargs):
    190          """allows setting options that will always be passed to openarchive"""
    191          self.archiveoptions = kwargs
    192
    193 -    def isrecursive(self, fileoption, filepurpose='input'):
    194          """checks if fileoption is a recursive file"""
    195          if self.isarchive(fileoption, filepurpose): return True
    196          return super(ArchiveConvertOptionParser, self).isrecursive(fileoption, filepurpose)
    197
    198 -    def isarchive(self, fileoption, filepurpose='input'):
    199          """returns whether the file option is an archive file"""
    200          if not isinstance(fileoption, (str, unicode)):
    201              return False
    202          mustexist = (filepurpose != 'output')
    203          if mustexist and not os.path.isfile(fileoption):
    204              return False
    205          fileext = self.splitext(fileoption)[1]
    206          # if None is in the archive formats, then treat all non-directory inputs as archives
    207          return self.getarchiveclass(fileext, filepurpose, os.path.isdir(fileoption)) is not None
    208
    209 -    def getarchiveclass(self, fileext, filepurpose, isdir=False):
    210          """returns the archiveclass for the given fileext and filepurpose"""
    211          archiveclass = self.archiveformats.get(fileext, None)
    212          if archiveclass is not None:
    213              return archiveclass
    214          archiveclass = self.archiveformats.get((fileext, filepurpose), None)
    215          if archiveclass is not None:
    216              return archiveclass
    217          if not isdir:
    218              archiveclass = self.archiveformats.get(None, None)
    219              if archiveclass is not None:
    220                  return archiveclass
    221              archiveclass = self.archiveformats.get((None, filepurpose), None)
    222              if archiveclass is not None:
    223                  return archiveclass
    224          return None
    225
    226 -    def openarchive(self, archivefilename, filepurpose, **kwargs):
    227          """creates an archive object for the given file"""
    228          archiveext = self.splitext(archivefilename)[1]
    229          archiveclass = self.getarchiveclass(archiveext, filepurpose, os.path.isdir(archivefilename))
    230          archiveoptions = self.archiveoptions.copy()
    231          archiveoptions.update(kwargs)
    232          return archiveclass(archivefilename, **archiveoptions)
    233
    234 -    def recurseinputfiles(self, options):
    235          """recurse through archive file / directories and return files to be converted"""
    236          if self.isarchive(options.input, 'input'):
    237              options.inputarchive = self.openarchive(options.input, 'input')
    238              return self.recursearchivefiles(options)
    239          else:
    240              return super(ArchiveConvertOptionParser, self).recurseinputfiles(options)
    241
    242 -    def recursearchivefiles(self, options):
    243          """recurse through archive files and convert files"""
    244          inputfiles = []
    245          for inputpath in options.inputarchive:
    246              if self.isexcluded(options, inputpath):
    247                  continue
    248              top, name = os.path.split(inputpath)
    249              if not self.isvalidinputname(options, name):
    250                  continue
    251              inputfiles.append(inputpath)
    252          return inputfiles
    253
    254 -    def openinputfile(self, options, fullinputpath):
    255          """opens the input file"""
    256          if self.isarchive(options.input, 'input'):
    257              return options.inputarchive.openinputfile(fullinputpath)
    258          else:
    259              return super(ArchiveConvertOptionParser, self).openinputfile(options, fullinputpath)
    260
    261 -    def getfullinputpath(self, options, inputpath):
    262          """gets the absolute path to an input file"""
    263          if self.isarchive(options.input, 'input'):
    264              return inputpath
    265          else:
    266              return os.path.join(options.input, inputpath)
    267
    268 -    def opentemplatefile(self, options, fulltemplatepath):
    269          """opens the template file (if required)"""
    270          if fulltemplatepath is not None:
    271              if options.recursivetemplate and self.isarchive(options.template, 'template'):
    272                  # TODO: deal with different names in input/template archives
    273                  if fulltemplatepath in options.templatearchive:
    274                      return options.templatearchive.openinputfile(fulltemplatepath)
    275                  else:
    276                      self.warning("missing template file %s" % fulltemplatepath)
    277          return super(ArchiveConvertOptionParser, self).opentemplatefile(options, fulltemplatepath)
    278
    279 -    def getfulltemplatepath(self, options, templatepath):
    280          """gets the absolute path to a template file"""
    281          if templatepath is not None and self.usetemplates and options.template:
    282              if self.isarchive(options.template, 'template'):
    283                  return templatepath
    284              elif not options.recursivetemplate:
    285                  return templatepath
    286              else:
    287                  return os.path.join(options.template, templatepath)
    288          else:
    289              return None
    290
    291 -    def templateexists(self, options, templatepath):
    292          """returns whether the given template exists..."""
    293          if templatepath is not None:
    294              if self.isarchive(options.template, 'template'):
    295                  # TODO: deal with different names in input/template archives
    296                  return templatepath in options.templatearchive
    297          return super(ArchiveConvertOptionParser, self).templateexists(options, templatepath)
    298
    299 -    def getfulloutputpath(self, options, outputpath):
    300          """gets the absolute path to an output file"""
    301          if self.isarchive(options.output, 'output'):
    302              return outputpath
    303          elif options.recursiveoutput and options.output:
    304              return os.path.join(options.output, outputpath)
    305          else:
    306              return outputpath
    307
    308 -    def checkoutputsubdir(self, options, subdir):
    309          """checks to see if subdir under options.output needs to be created, creates if neccessary"""
    310          if not self.isarchive(options.output, 'output'):
    311              super(ArchiveConvertOptionParser, self).checkoutputsubdir(options, subdir)
    312
    313 -    def openoutputfile(self, options, fulloutputpath):
    314          """opens the output file"""
    315          if self.isarchive(options.output, 'output'):
    316              outputstream = options.outputarchive.openoutputfile(fulloutputpath)
    317              if outputstream is None:
    318                  self.warning("Could not find where to put %s in output archive; writing to tmp" % fulloutputpath)
    319                  return StringIO()
    320              return outputstream
    321          else:
    322              return super(ArchiveConvertOptionParser, self).openoutputfile(options, fulloutputpath)
    323
    324 -    def inittemplatearchive(self, options):
    325          """opens the templatearchive if not already open"""
    326          if not self.usetemplates:
    327              return
    328          if options.template and self.isarchive(options.template, 'template') and not hasattr(options, "templatearchive"):
    329              options.templatearchive = self.openarchive(options.template, 'template')
    330
    331 -    def initoutputarchive(self, options):
    332          """creates an outputarchive if required"""
    333          if options.output and self.isarchive(options.output, 'output'):
    334              options.outputarchive = self.openarchive(options.output, 'output', mode="w")
    335
    336 -    def recursiveprocess(self, options):
    337          """recurse through directories and convert files"""
    338          if hasattr(options, "multifilestyle"):
    339              self.setarchiveoptions(multifilestyle=options.multifilestyle)
    340              for filetype in ("input", "output", "template"):
    341                  allowoption = "allowrecursive%s" % filetype
    342                  if options.multifilestyle == "onefile" and getattr(options, allowoption, True):
    343                      setattr(options, allowoption, False)
    344          self.inittemplatearchive(options)
    345          self.initoutputarchive(options)
    346          return super(ArchiveConvertOptionParser, self).recursiveprocess(options)
    347
    348 -    def processfile(self, fileprocessor, options, fullinputpath, fulloutputpath, fulltemplatepath):
    349          """run an invidividual conversion"""
    350          if self.isarchive(options.output, 'output'):
    351              inputfile = self.openinputfile(options, fullinputpath)
    352              # TODO: handle writing back to same archive as input/template
    353              templatefile = self.opentemplatefile(options, fulltemplatepath)
    354              outputfile = self.openoutputfile(options, fulloutputpath)
    355              passthroughoptions = self.getpassthroughoptions(options)
    356              if fileprocessor(inputfile, outputfile, templatefile, **passthroughoptions):
    357                  if not outputfile.isatty():
    358                      outputfile.close()
    359                  return True
    360              else:
    361                  if fulloutputpath and os.path.isfile(fulloutputpath):
    362                      outputfile.close()
    363                      os.unlink(fulloutputpath)
    364                  return False
    365          else:
    366              return super(ArchiveConvertOptionParser, self).processfile(fileprocessor, options, fullinputpath, fulloutputpath, fulltemplatepath)
    367
    368 -def main(argv=None):
    369      parser = ArchiveConvertOptionParser({}, description=__doc__)
    370      parser.run(argv)
    371