Add translation importing to post_release_check_update

app/tasks/importtasks.py

@@ -18,6 +18,7 @@
 import json
 import os
 import shutil
+import sys
 from json import JSONDecodeError
 from zipfile import ZipFile
 
@@ -26,16 +27,16 @@
 from git import GitCommandError
 from git_archive_all import GitArchiver
 from kombu import uuid
-from sqlalchemy import and_
+from sqlalchemy import and_, insert
 
 from app.models import AuditSeverity, db, NotificationType, PackageRelease, MetaPackage, Dependency, PackageType, \
 	MinetestRelease, Package, PackageState, PackageScreenshot, PackageUpdateTrigger, PackageUpdateConfig, \
-	PackageGameSupport
+	PackageGameSupport, PackageTranslation, Language
 from app.tasks import celery, TaskError
 from app.utils import random_string, post_bot_message, add_system_notification, add_system_audit_log, \
 	get_games_from_list, add_audit_log
 from app.utils.git import clone_repo, get_latest_tag, get_latest_commit, get_temp_dir
-from .minetestcheck import build_tree, MinetestCheckError, ContentType
+from .minetestcheck import build_tree, MinetestCheckError, ContentType, PackageTreeNode
 from .webhooktasks import post_discord_webhook
 from app import app
 from app.logic.LogicError import LogicError
@@ -96,7 +97,7 @@ def update_all_game_support():
 
 def post_release_check_update(self, release: PackageRelease, path):
 	try:
-		tree = build_tree(path, expected_type=ContentType[release.package.type.name],
+		tree: PackageTreeNode = build_tree(path, expected_type=ContentType[release.package.type.name],
 				author=release.package.author.username, name=release.package.name)
 
 		if tree.name is not None and release.package.name != tree.name and tree.type == ContentType.MOD:
@@ -162,6 +163,33 @@ def get_meta_packages(names):
 		for meta in get_meta_packages(optional_depends):
 			db.session.add(Dependency(package, meta=meta, optional=True))
 
+		# Read translations
+		allowed_languages = set(db.session.query(Language.id).all())
+		print(f"Allowed languages {allowed_languages}", file=sys.stderr)
+
+		textdomain = tree.get("textdomain", tree.name)
+		raw_translations = tree.get_translations(textdomain)
+		conn = db.session.connection()
+		for raw_translation in raw_translations:
+			if raw_translation.language not in allowed_languages:
+				print(f"Ignoring language {raw_translation.language}", file=sys.stderr)
+
+			to_update = {
+				"title": raw_translation.entries.get(tree.get("title", package.title)),
+				"short_desc": raw_translation.entries.get(tree.get("description", package.short_desc)),
+			}
+			values = {
+				"package_id": package.id,
+				"language_id": raw_translation.language
+			}
+
+			stmt = insert(PackageTranslation).values(**values)
+			stmt = stmt.on_conflict_do_update(
+				index_elements=[PackageTranslation.package_id, PackageTranslation.language_id],
+				set_=to_update
+			)
+			conn.execute(stmt)
+
 		# Update min/max
 		if tree.meta.get("min_minetest_version"):
 			release.min_rel = MinetestRelease.get(tree.meta["min_minetest_version"], None)

app/tasks/minetestcheck/tree.py

@@ -17,10 +17,12 @@
 
 import os
 import re
+import glob
 from typing import Optional
 
 from . import MinetestCheckError, ContentType
 from .config import parse_conf
+from .translation import Translation, parse_tr
 
 basenamePattern = re.compile("^([a-z0-9_]+)$")
 licensePattern = re.compile("^(licen[sc]e|copying)(.[^/\n]+)?$", re.IGNORECASE)
@@ -31,7 +33,7 @@
 }
 
 
-def get_base_dir(path):
+def get_base_dir(path) -> str:
 	if not os.path.isdir(path):
 		raise IOError("Expected dir")
 
@@ -42,7 +44,7 @@ def get_base_dir(path):
 		return path
 
 
-def detect_type(path):
+def detect_type(path) -> ContentType:
 	if os.path.isfile(path + "/game.conf"):
 		return ContentType.GAME
 	elif os.path.isfile(path + "/init.lua"):
@@ -58,7 +60,7 @@ def detect_type(path):
 		return ContentType.UNKNOWN
 
 
-def get_csv_line(line):
+def get_csv_line(line) -> list[str]:
 	if line is None:
 		return []
 
@@ -79,33 +81,43 @@ def check_name_list(key: str, value: list[str], relative: str, allow_star: bool
 
 
 class PackageTreeNode:
-	def __init__(self, base_dir, relative, author=None, repo=None, name=None):
-		self.baseDir  = base_dir
+	baseDir: str
+	relative: str
+	author: Optional[str]
+	name: Optional[str]
+	repo: Optional[str]
+	meta: dict
+	children: list
+	type: ContentType
+
+	def __init__(self, base_dir: str, relative: str,
+			author: Optional[str] = None, repo: Optional[str] = None, name: Optional[str] = None):
+		self.baseDir = base_dir
 		self.relative = relative
-		self.author   = author
-		self.name	 = name
-		self.repo	 = repo
-		self.meta	 = None
+		self.author = author
+		self.name = name
+		self.repo = repo
+		self.meta = {}
 		self.children = []
 
 		# Detect type
 		self.type = detect_type(base_dir)
-		self.read_meta()
+		self._read_meta()
 
 		if self.type == ContentType.GAME:
 			if not os.path.isdir(os.path.join(base_dir, "mods")):
 				raise MinetestCheckError("Game at {} does not have a mods/ folder".format(self.relative))
-			self.add_children_from_mod_dir("mods")
+			self._add_children_from_mod_dir("mods")
 		elif self.type == ContentType.MOD:
 			if self.name and not basenamePattern.match(self.name):
 				raise MinetestCheckError(f"Invalid base name for mod {self.name} at {self.relative}, names must only contain a-z0-9_.")
 
 			if self.name and self.name in DISALLOWED_NAMES:
 				raise MinetestCheckError(f"Forbidden mod name '{self.name}' used at {self.relative}")
 
-			self.check_dir_casing(["textures", "media", "sounds", "models", "locale"])
+			self._check_dir_casing(["textures", "media", "sounds", "models", "locale"])
 		elif self.type == ContentType.MODPACK:
-			self.add_children_from_mod_dir(None)
+			self._add_children_from_mod_dir(None)
 
 	def find_license_file(self) -> Optional[str]:
 		for name in os.listdir(self.baseDir):
@@ -115,7 +127,7 @@ def find_license_file(self) -> Optional[str]:
 
 		return None
 
-	def check_dir_casing(self, dirs):
+	def _check_dir_casing(self, dirs):
 		for dir in next(os.walk(self.baseDir))[1]:
 			lowercase = dir.lower()
 			if lowercase != dir and lowercase in dirs:
@@ -139,7 +151,7 @@ def get_meta_file_name(self):
 		else:
 			return None
 
-	def read_meta(self):
+	def _read_meta(self):
 		result = {}
 
 		# Read .conf file
@@ -229,7 +241,7 @@ def read_meta(self):
 
 		self.meta = result
 
-	def add_children_from_mod_dir(self, subdir):
+	def _add_children_from_mod_dir(self, subdir):
 		dir = self.baseDir
 		relative = self.relative
 		if subdir:
@@ -282,9 +294,15 @@ def fold(self, attr, key=None, retval=None, type_=None):
 
 		return retval
 
-	def get(self, key):
-		return self.meta.get(key)
+	def get(self, key: str, default=None):
+		return self.meta.get(key, default)
 
 	def validate(self):
 		for child in self.children:
 			child.validate()
+
+	def get_translations(self, textdomain: str) -> list[Translation]:
+		ret = []
+		for name in glob.glob(self.baseDir + f"/**/locale/{textdomain}.*.tr"):
+			ret.append(parse_tr(name))
+		return ret