🔧 Copied self hosted docker apps

self_host/LibreTranslate/scripts/compile_locales.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+import os
+import sys
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from babel.messages.frontend import main as pybabel
+
+if __name__ == "__main__":
+    if len(sys.argv) >= 2 and sys.argv[1] == 'mdtable':
+        from libretranslate.locales import get_available_locales
+        locales = get_available_locales(only_reviewed=False, sort_by_name=True)
+        print("Language | Reviewed | Weblate Link")
+        print("-------- | -------- | ------------")
+
+        for l in locales:
+            link = "https://hosted.weblate.org/translate/libretranslate/app/%s/" % l['code']
+            if l['code'] == 'en':
+                link = "https://hosted.weblate.org/projects/libretranslate/app/"
+            print("{} | {} | {}".format(l['name'], ':heavy_check_mark:' if l['reviewed'] else '', "[Edit](%s)" % link))
+    else:
+        locales_dir = os.path.join("libretranslate", "locales")
+        if not os.path.isdir(locales_dir):
+            os.makedirs(locales_dir)
+
+        print("Compiling locales")
+        sys.argv = ["", "compile", "-f", "-d", locales_dir]
+        pybabel()
+
+
+

self_host/LibreTranslate/scripts/gunicorn_conf.py

@@ -0,0 +1,47 @@
+import re
+import sys
+
+from prometheus_client import multiprocess
+
+
+def child_exit(server, worker):
+    multiprocess.mark_process_dead(worker.pid)
+
+def on_starting(server):
+    # Parse command line arguments
+    proc_name = server.cfg.default_proc_name
+    kwargs = {}
+    if proc_name.startswith("wsgi:app"):
+        str_args = re.sub(r'wsgi:app\s*\(\s*(.*)\s*\)', '\\1', proc_name).strip().split(",")
+        for a in str_args:
+            if "=" in a:
+                k,v = a.split("=")
+                k = k.strip()
+                v = v.strip()
+
+                if v.lower() in ["true", "false"]:
+                    v = v.lower() == "true"
+                    if not v:
+                        continue
+                elif v[0] == '"':
+                    v = v[1:-1]
+                kwargs[k] = v
+
+    from libretranslate.main import get_args
+    sys.argv = ['--wsgi']
+
+    for k in kwargs:
+        ck = k.replace("_", "-")
+        if isinstance(kwargs[k], bool) and kwargs[k]:
+            sys.argv.append("--" + ck)
+        else:
+            sys.argv.append("--" + ck)
+            sys.argv.append(kwargs[k])
+
+    args = get_args()
+
+    from libretranslate import flood, scheduler, secret, storage
+    storage.setup(args.shared_storage)
+    scheduler.setup(args)
+    flood.setup(args)
+    secret.setup(args)

self_host/LibreTranslate/scripts/healthcheck.py

@@ -0,0 +1,15 @@
+import requests
+import os
+
+port = os.environ.get('LT_PORT', '5000')
+response = requests.post(
+    url=f'http://localhost:{port}/translate',
+    headers={'Content-Type': 'application/json'},
+    json={
+         'q': 'Hello World!',
+         'source': 'en',
+         'target': 'en'
+    },
+    timeout=60
+)
+response.raise_for_status() # if server unavailable then requests with raise exception and healthcheck will fail

self_host/LibreTranslate/scripts/install_models.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+import os
+import sys
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+import argparse
+
+from libretranslate.init import check_and_install_models
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--load_only_lang_codes", type=str, default="")
+    parser.add_argument("--update", action='store_true')
+    args = parser.parse_args()
+    lang_codes = args.load_only_lang_codes.split(",")
+    if len(lang_codes) == 0 or lang_codes[0] == '':
+        lang_codes = None
+    if args.update:
+        check_and_install_models(update=True, load_only_lang_codes=lang_codes)
+    else:
+        check_and_install_models(force=True, load_only_lang_codes=lang_codes)

self_host/LibreTranslate/scripts/suggestions-to-jsonl.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+import argparse
+import json
+import sqlite3
+import time
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Program to generate JSONL files from a LibreTranslate's suggestions.db")
+    parser.add_argument(
+        "--db",
+        type=str,
+        nargs=1,
+        help="Path to suggestions.db file",
+        default='db/suggestions.db'
+    )
+    parser.add_argument(
+        "--clear",
+        action='store_true',
+        help="Clear suggestions.db after generation",
+        default=False
+    )
+    args = parser.parse_args()
+
+    output_file = str(int(time.time())) + ".jsonl"
+
+    con = sqlite3.connect(args.db, check_same_thread=False)
+    cur = con.cursor()
+
+    with open(output_file, 'w', encoding="utf-8") as f:
+        for row in cur.execute('SELECT q, s, source, target FROM suggestions WHERE source != "auto" ORDER BY source'):
+            q, s, source, target = row
+            obj = {
+                'q': q,
+                's': s,
+                'source': source,
+                'target': target
+            }
+            json.dump(obj, f, ensure_ascii=False)
+            f.write('\n')
+
+    print("Wrote %s" % output_file)
+
+    if args.clear:
+        cur.execute("DELETE FROM suggestions")
+        con.commit()
+        print("Cleared " + args.db)

self_host/LibreTranslate/scripts/update_locales.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python
+import os
+import sys
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+import json
+import re
+
+import polib
+from babel.messages.frontend import main as pybabel
+from flask_swagger import swagger
+from translatehtml import translate_html
+
+from libretranslate.app import create_app, get_version
+from libretranslate.language import improve_translation_formatting, load_languages
+from libretranslate.locales import get_available_locale_codes, swag_eval
+from libretranslate.main import get_args
+
+# Update strings
+if __name__ == "__main__":
+    print("Loading languages")
+    languages = load_languages()
+    en_lang = next((l for l in languages if l.code == 'en'), None)
+    if en_lang is None:
+        print("Error: English model not found. You need it to run this script.")
+        exit(1)
+
+    locales_dir = os.path.join("libretranslate", "locales")
+    if not os.path.isdir(locales_dir):
+        os.makedirs(locales_dir)
+
+    # Dump language list so it gets picked up by pybabel
+    langs_file = os.path.join(locales_dir, ".langs.py")
+    lang_names = {}
+    with open(langs_file, 'w') as f:
+        for l in languages:
+            f.write("_(%s)\n" % json.dumps(l.name))
+            lang_names[l.name] = True
+
+        # Additional names that might be missing
+        for l in ["Serbian", "Ukrainian", "Vietnamese"]:
+            if not l in lang_names:
+                f.write("_(%s)\n" % json.dumps(l))
+                lang_names[l] = True
+    print("Wrote %s" % langs_file)
+
+    # Dump swagger strings
+    args = get_args()
+    app = create_app(args)
+    swag = swagger(app)
+
+    swag_strings = []
+    def add_swag_string(s):
+        if not s in swag_strings:
+            swag_strings.append(s)
+    swag_eval(swag, add_swag_string)
+
+    swag_file = os.path.join(locales_dir, ".swag.py")
+    with open(swag_file, 'w') as f:
+        for ss in swag_strings:
+            f.write("_(%s)\n" % json.dumps(ss))
+    print("Wrote %s" % swag_file)
+
+    messagespot = os.path.join(locales_dir, "messages.pot")
+    print("Updating %s" % messagespot)
+    sys.argv = ["", "extract", "-F", "babel.cfg", "-k", "_e _h",
+                "--copyright-holder", "LibreTranslate Authors",
+                "--project", "LibreTranslate",
+                "--version", get_version(),
+                "-o", messagespot, "libretranslate"]
+    pybabel()
+
+    lang_map = {
+        'zt': 'zh_Hant'
+    }
+    lang_codes = [lang_map.get(l.code, l.code) for l in languages if l.code != "en"]
+    all_folders = [d for d in os.listdir(locales_dir) if os.path.isdir(os.path.join(locales_dir, d))]
+    review_map = {}
+
+    # Init/update
+    for l in all_folders:
+        cmd = "init"
+        if os.path.isdir(os.path.join(locales_dir, l, "LC_MESSAGES")):
+            cmd = "update"
+
+        sys.argv = ["", cmd, "-i", messagespot, "-d", locales_dir, "-l", l] + (["--no-fuzzy-matching"] if cmd == "update" else [])
+        pybabel()
+
+        meta_file = os.path.join(locales_dir, l, "meta.json")
+        reviewed = False
+        if not os.path.isfile(meta_file):
+            with open(meta_file, 'w') as f:
+                f.write(json.dumps({
+                    'name': l if l not in lang_codes else next(lang.name for lang in languages if lang_map.get(lang.code, lang.code) == l),
+                    'reviewed': False
+                }, indent=4))
+                print("Wrote %s" % meta_file)
+        else:
+            with open(meta_file) as f:
+                reviewed = json.loads(f.read()).get('reviewed', False)
+
+        review_map[l] = reviewed
+
+    # Automatically translate strings with libretranslate
+    # when a language model is available and a string is empty
+
+    locales = get_available_locale_codes(only_reviewed=False)
+    for locale in locales:
+        if locale == 'en':
+            continue
+        if review_map.get(locale):
+            # Don't automatically translate reviewed languages
+            continue
+
+        tgt_lang = next((l for l in languages if l.code == locale), None)
+
+        if tgt_lang is None:
+            # We cannot translate
+            continue
+
+        translator = en_lang.get_translation(tgt_lang)
+
+        messages_file = os.path.join(locales_dir, locale, "LC_MESSAGES", 'messages.po')
+        if os.path.isfile(messages_file):
+            print("Translating '%s'" % locale)
+            pofile = polib.pofile(messages_file)
+            c = 0
+
+            for entry in pofile.untranslated_entries():
+                text = entry.msgid
+
+                # Extract placeholders
+                placeholders = re.findall(r'%\(?[^\)]*\)?s', text)
+
+                for p in range(0, len(placeholders)):
+                    text = text.replace(placeholders[p], "<x>%s</x>" % p)
+
+                if len(placeholders) > 0:
+                    translated = str(translate_html(translator, text))
+                else:
+                    translated = improve_translation_formatting(text, translator.translate(text))
+
+                # Restore placeholders
+                for p in range(0, len(placeholders)):
+                    tag = "<x>%s</x>" % p
+                    if tag in translated:
+                        translated = translated.replace(tag, placeholders[p])
+                    else:
+                        # Meh, append
+                        translated += " " + placeholders[p]
+
+                print(entry.msgid, " --> ", translated)
+                entry.msgstr = translated
+                c += 1
+
+            if c > 0:
+                pofile.save(messages_file)
+                print("Saved %s" % messages_file)
+
+
+