Split up main file into components

.gitignore

@@ -1,2 +1,5 @@
-.idea
-RimTranslate.log
+logs/
+po/
+output/
+.tool-versions
+__pycache__/

Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3.13-rc-bookworm
+
+WORKDIR /app
+COPY . .
+
+RUN python -m pip install -r requirements.txt
+CMD ["python", "RimTranslate.py", "-p", "po/", "-o", "output/"]

config/config.py

@@ -0,0 +1,13 @@
+pot_metadata = {
+    "language_team": {
+        "name": "English",
+        "email": "[email protected]"
+    },
+    "last_translator": {
+        "name": "Some Translator",
+        "email": "[email protected]"
+    },
+    "report_msgid_bugs_to": "[email protected]"
+}
+
+version = "0.7.0"

requirements.txt

@@ -0,0 +1,2 @@
+lxml==5.2.1
+polib==1.2.0

src/builders/output_dir_builder.py

@@ -0,0 +1,65 @@
+from lxml import etree
+import os
+import polib
+
+from config.config import version
+from ..logger import Logger
+from ..parser import Parser
+
+
+class OutputDirBuilder():
+    def build_output_dir(self):
+        Logger.logger.info('Beginning to generate DefInjected files')
+        fuzzy = 0
+        total = 0
+        translated = 0
+        untranslated = 0
+
+        for root, dirs, files in os.walk(Parser.args.po_dir):
+            for file in files:
+                if file.endswith('.po'):
+                    full_filename = os.path.join(root, file)
+                    Logger.logger.info("Processing " + full_filename)
+                    xml_filename = full_filename.split(Parser.args.po_dir, 1)[1]
+                    xml_filename = xml_filename.strip('.po')
+                    xml_filename = os.path.join(Parser.args.output_dir, xml_filename)
+                    directory = os.path.dirname(xml_filename)
+
+                    po = polib.pofile(full_filename)
+                    translated_po_entries = len(po.translated_entries())
+                    fuzzy_po_entries = len(po.fuzzy_entries())
+                    untranslated_po_entries = len(po.untranslated_entries())
+
+                    translated = translated + translated_po_entries
+                    fuzzy = fuzzy + fuzzy_po_entries
+                    untranslated = untranslated + untranslated_po_entries
+
+                    # Do we have translated entries?
+                    if translated_po_entries > 0:
+                        if not (os.path.exists(directory)):
+                            Logger.logger.info("Creating directory " + directory)
+                            os.makedirs(directory)
+                        xml_content = self.__create_languagedata_xml_file(full_filename)
+                        target = open(xml_filename, "w", encoding="utf8")
+                        target.write(xml_content)
+                        target.close()
+                    total_po_entries = len([e for e in po if  not e.obsolete])
+                    total = total + total_po_entries
+
+        print("Statistics (untranslated/fuzzy/translated/total): %d/%d/%d/%d" % (untranslated, fuzzy, translated, total))
+
+
+    def __create_languagedata_xml_file(self, po_file):
+        languagedata = etree.Element('LanguageData')
+        languagedata.addprevious(etree.Comment(' This file autogenerated with RimTranslate.py v%s ' % version))
+        languagedata.addprevious(etree.Comment(' https://github.com/winterheart/RimTranslate/ '))
+        languagedata.addprevious(etree.Comment(' Don\'t edit this file manually, edit PO file and regenerate this file! '))
+        xml = etree.ElementTree(languagedata)
+        po = polib.pofile(po_file)
+        for po_entry in po:
+            if (po_entry.msgstr != "") and ('fuzzy' not in po_entry.flags):
+                entry = etree.SubElement(languagedata, po_entry.msgctxt)
+                entry.text = str(po_entry.msgstr)
+        # Hack - silly lxml cannot write native unicode strings
+        xml_file = etree.tostring(xml, pretty_print=True, xml_declaration=True, encoding='utf-8').decode('utf-8')
+        return xml_file

src/builders/source_dir_builder.py

@@ -0,0 +1,72 @@
+import os
+import polib
+
+from ..helpers.compendium_helpers import build_compendium, merge_compendium
+from ..helpers.pot_helpers import create_pot_file
+from ..logger import Logger
+from ..parser import Parser
+
+class SourceDirBuilder:
+    def __init__(self):
+        self.compendium = None
+        if Parser.args.compendium:
+            self.compendium = build_compendium(Parser.args.compendium, Parser.args.source_dir)
+
+    def build_source_dir(self):
+        Logger.logger.info('Beginning to generate PO-files')
+        self.build_source_dir_defs()
+        self.build_source_dir_keyed()
+
+    def build_source_dir_defs(self):
+        self.build_source_dir_files(
+            'DefInjected',
+            os.path.join(Parser.args.source_dir, 'Defs', '')
+        )
+
+    def build_source_dir_keyed(self):
+        self.build_source_dir_files(
+            'Keyed',
+            os.path.join(Parser.args.source_dir, 'Languages/English/Keyed', '')
+        )
+
+    def build_source_dir_files(self, category_name, source_dir):
+        Logger.logger.info('Generating PO-files from %s' % category_name)
+
+        if not os.path.isdir(source_dir):
+            Logger.logger.error('%s is not a directory' % source_dir)
+            quit()
+
+        for root, dirs, files in os.walk(source_dir):
+            for file in files:
+                if file.endswith('.xml'):
+                    full_filename = os.path.join(root, file)
+                    Logger.logger.info("Processing " + full_filename)
+                    file_dir = full_filename.split(source_dir, 1)[1]
+
+                    if category_name == 'DefInjected':
+                        # Replace Defs to Def (https://github.com/winterheart/RimTranslate/issues/1)
+                        file_dir = file_dir.replace("Defs", "Def")
+
+                    pot = create_pot_file(category_name, full_filename, Parser.args.source_dir, Parser.args.compendium)
+                    pofilename = os.path.join(Parser.args.po_dir, category_name, file_dir)
+                    pofilename += '.po'
+
+                    if os.path.exists(pofilename):
+                        Logger.logger.info("Updating PO file " + pofilename)
+                        po = polib.pofile(pofilename)
+                        po.merge(pot)
+                    else:
+                        # Is there some useful info?
+                        if len(pot) > 0:
+                            directory = os.path.dirname(pofilename)
+                            if not (os.path.exists(directory)):
+                                Logger.logger.info("Creating directory " + directory)
+                                os.makedirs(directory)
+                            Logger.logger.info("Creating PO file " + pofilename)
+                        po = pot
+
+                    if Parser.args.compendium:
+                        po = merge_compendium(self.compendium, po)
+
+                    if len(po):
+                        po.save(pofilename)

src/helpers/compendium_helpers.py

@@ -0,0 +1,35 @@
+import os
+import polib
+
+from .pot_helpers import create_pot_file
+from ..logger import Logger
+
+def build_compendium(compendium_path, source_dir_path):
+    Logger.logger.info('Creating compendium from already exist DefInj XML files')
+
+    if not os.path.isdir(compendium_path):
+        Logger.logger.error('%s is not directory or does not exists!' % compendium_path)
+        quit()
+
+    compendium = polib.POFile()
+
+    for root, dirs, files in os.walk(compendium_path):
+        for file in files:
+            if file.endswith('.xml'):
+                full_filename = os.path.join(root, file)
+                Logger.logger.debug('Processing %s for compendium' % full_filename)
+                compendium += create_pot_file('Keyed', full_filename, source_dir_path, compendium_path, True)
+
+    return compendium
+
+def merge_compendium(compendium, po):
+    # Fill current translation entries with translation memory from compendium
+    for entry in po:
+        if entry.msgstr == '':
+            check_msg = compendium.find(entry.msgctxt, by='msgctxt', include_obsolete_entries=False)
+            if check_msg and check_msg.msgstr:
+                entry.msgstr = check_msg.msgstr
+                if 'fuzzy' not in entry.flags:
+                    entry.flags.append('fuzzy')
+
+    return po

src/helpers/pot_helpers.py

@@ -0,0 +1,151 @@
+import datetime
+from lxml import etree
+import polib
+import re
+
+from config.config import pot_metadata
+from ..logger import Logger
+
+pot_file_metadata = {
+    'Project-Id-Version': '1.0',
+    'Report-Msgid-Bugs-To': pot_metadata['report_msgid_bugs_to'],
+    'POT-Creation-Date': str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M")),
+    'PO-Revision-Date': str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M")),
+    'Last-Translator': '%s <%s>' % (pot_metadata['last_translator']['name'], pot_metadata['last_translator']['email']),
+    'Language-Team': '%s <%s>' % (pot_metadata['language_team']['name'], pot_metadata['language_team']['email']),
+    'MIME-Version': '1.0',
+    'Content-Type': 'text/plain; charset=utf-8',
+    'Content-Transfer-Encoding': '8bit',
+}
+
+def create_pot_file(category, filename, source_dir, compendium, compendium_mode=False):
+    if category == 'DefInjected':
+        return create_pot_file_from_def(filename, source_dir)
+    elif category == 'Keyed':
+        return create_pot_file_from_keyed(filename, source_dir, compendium, compendium_mode)
+    else:
+        Logger.logger.error('Incorrect pot file category requested: %s' % category)
+        quit()
+
+def create_pot_file_from_keyed(filename, source_dir, compendium, compendium_mode=False):
+    """Create compendium from keyed or already created definj XML files"""
+    parser = etree.XMLParser(remove_comments=True)
+    if compendium:
+        basefile = 'compendium'
+    else:
+        basefile = filename.split(source_dir, 1)[1]
+
+    po_file = polib.POFile()
+    po_file.metadata = pot_file_metadata
+    po_file.metadata_is_fuzzy = 1
+    doc = etree.parse(filename, parser)
+    for languageData in doc.xpath('//LanguageData'):
+        for element in languageData:
+            entry = polib.POEntry(
+                msgctxt=element.tag,
+                msgid=element.text,
+                occurrences=[(basefile, str(element.sourceline))]
+            )
+            if compendium_mode:
+                entry.msgstr = element.text
+            po_file.append(entry)
+
+    return po_file
+
+
+def create_pot_file_from_def(filename, source_dir):
+    """Create POT file (only source strings exists) from given filename"""
+    doc = etree.parse(filename)
+    po_file = polib.POFile()
+    basefile = filename.split(source_dir, 1)[1]
+
+    po_file.metadata = pot_file_metadata
+    po_file.metadata_is_fuzzy = 1
+
+    defNames = [
+        'defName',
+        'DefName',  # Some DefNames with first uppercase letter
+    ]
+
+    labels = [
+        'beginLetter',
+        'beginLetterLabel',
+        'description',
+        'fixedName',
+        'gerund',
+        'gerundLabel',
+        'helpText',
+        'ingestCommandString',
+        'ingestReportString',
+        'inspectLine',
+        'label',
+        'labelShort',
+        'letterLabel',
+        'letterText',
+        'pawnLabel',
+        'pawnsPlural',
+        'rulesStrings',         # hard one
+        'recoveryMessage',
+        'reportString',
+        'skillLabel',
+        'text',
+        'useLabel',
+        'verb',
+    ]
+
+    for defName in defNames:
+        for defName_node in doc.findall(".//" + defName):
+            if defName_node is not None:
+                parent = defName_node.getparent()
+                Logger.logger.debug("Found defName '%s' (%s)" % (defName_node.text, doc.getpath(parent)))
+                for label in labels:
+                    parent = defName_node.getparent()
+                    Logger.logger.debug("Checking label %s" % label)
+                    label_nodes = parent.findall(".//" + label)
+                    for label_node in label_nodes:
+                        Logger.logger.debug("Found Label '%s' (%s)" % (label, doc.getpath(label_node)))
+                        if len(label_node):
+                            Logger.logger.debug("Element has children")
+                            for child_node in label_node:
+                                if child_node.tag is not etree.Comment:
+                                    path_label = doc.getpath(child_node).split(doc.getpath(parent), 1)[1]
+                                    path_label = generate_definj_xml_tag(path_label)
+
+                                    Logger.logger.debug("msgctxt: " + defName_node.text + path_label)
+                                    entry = polib.POEntry(
+                                        msgctxt=defName_node.text + path_label,
+                                        msgid=child_node.text,
+                                        occurrences=[(basefile, str(label_node.sourceline))]
+                                    )
+                                    po_file.append(entry)
+                        else:
+                            # Generate string for parenting
+                            path_label = doc.getpath(label_node).split(doc.getpath(parent), 1)[1]
+                            path_label = generate_definj_xml_tag(path_label)
+
+                            Logger.logger.debug("msgctxt: " + defName_node.text + path_label)
+
+                            if not label_node.text:
+                                Logger.logger.warning(path_label + " has 'None' message!")
+                            else:
+                                entry = polib.POEntry(
+                                    msgctxt=defName_node.text + path_label,
+                                    msgid=label_node.text,
+                                    occurrences=[(basefile, str(label_node.sourceline))]
+                                )
+                                po_file.append(entry)
+    # sort by line in source file
+    po_file.sort(key=lambda x: int(x.occurrences[0][1]))
+
+    return po_file
+
+def generate_definj_xml_tag(string):
+    """Create XML tag for InjectDefs"""
+    string = re.sub(r'/', '.', string)
+    string = re.sub(r'\.li\.', '.0.', string)
+    string = re.sub(r'\.li$', '.0', string)
+    match = re.search(r'\.li\[(\d+)\]', string)
+    if match:
+        string = re.sub(r'\.li\[\d+\]', "." + str(int(match.group(1)) - 1), string)
+
+    return string

src/logger.py

@@ -0,0 +1,31 @@
+import logging
+import os
+
+class Logger:
+    logger = None
+
+    @classmethod
+    def init(cls, log_level_string):
+        os.makedirs('logs/', exist_ok=True)
+        if os.path.exists('logs/RimTranslate.log'):
+            os.remove('logs/RimTranslate.log')
+
+        log_format = '%(levelname)s: %(message)s' 
+        log_level = getattr(logging, str.upper(log_level_string))
+
+        logging.basicConfig(
+            format=log_format,
+            level=log_level,
+            filename='logs/RimTranslate.log'
+        )
+
+        # TODO: set streamhandler to not output to stderr by default
+        #       https://docs.python.org/3/library/logging.handlers.html#logging.StreamHandler
+        console = logging.StreamHandler()
+        console.setLevel(log_level)
+        console.setFormatter(logging.Formatter(log_format))
+
+        cls.logger = logging.getLogger('main')
+        cls.logger.addHandler(console)
+
+        return cls.logger