try building translations

build-scripts/gulp/fetch-nightly-translations.js

@@ -0,0 +1,171 @@
+// Task to download the latest Lokalise translations from the nightly workflow artifacts
+
+import { createOAuthDeviceAuth } from "@octokit/auth-oauth-device";
+import { retry } from "@octokit/plugin-retry";
+import { Octokit } from "@octokit/rest";
+import { deleteAsync } from "del";
+import { mkdir, readFile, writeFile } from "fs/promises";
+import gulp from "gulp";
+import jszip from "jszip";
+import path from "path";
+import process from "process";
+import { extract } from "tar";
+
+const MAX_AGE = 24; // hours
+const OWNER = "home-assistant";
+const REPO = "frontend";
+const WORKFLOW_NAME = "nightly.yaml";
+const ARTIFACT_NAME = "translations";
+const CLIENT_ID = "Iv1.3914e28cb27834d1";
+const EXTRACT_DIR = "translations";
+const TOKEN_FILE = path.posix.join(EXTRACT_DIR, "token.json");
+const ARTIFACT_FILE = path.posix.join(EXTRACT_DIR, "artifact.json");
+
+let allowTokenSetup = false;
+gulp.task("allow-setup-fetch-nightly-translations", (done) => {
+  allowTokenSetup = true;
+  done();
+});
+
+gulp.task("fetch-nightly-translations", async function () {
+  // Skip all when environment flag is set (assumes translations are already in place)
+  if (process.env?.SKIP_FETCH_NIGHTLY_TRANSLATIONS) {
+    console.log("Skipping fetch due to environment signal");
+    return;
+  }
+
+  // Read current translations artifact info if it exists,
+  // and stop if they are not old enough
+  let currentArtifact;
+  try {
+    currentArtifact = JSON.parse(await readFile(ARTIFACT_FILE, "utf-8"));
+    const currentAge =
+      (Date.now() - Date.parse(currentArtifact.created_at)) / 3600000;
+    if (currentAge < MAX_AGE) {
+      console.log(
+        "Keeping current translations (only %s hours old)",
+        currentAge.toFixed(1)
+      );
+      return;
+    }
+  } catch {
+    currentArtifact = null;
+  }
+
+  // To store file writing promises
+  const createExtractDir = mkdir(EXTRACT_DIR, { recursive: true });
+  const writings = [];
+
+  // Authenticate to GitHub using GitHub action token if it exists,
+  // otherwise look for a saved user token or generate a new one if none
+  let tokenAuth;
+  if (process.env.GITHUB_TOKEN) {
+    tokenAuth = { token: process.env.GITHUB_TOKEN };
+  } else {
+    try {
+      tokenAuth = JSON.parse(await readFile(TOKEN_FILE, "utf-8"));
+    } catch {
+      if (!allowTokenSetup) {
+        console.log("No token found so build will continue with English only");
+        return;
+      }
+      const auth = createOAuthDeviceAuth({
+        clientType: "github-app",
+        clientId: CLIENT_ID,
+        onVerification: (verification) => {
+          console.log(
+            "Task needs to authenticate to GitHub to fetch the translations from nightly workflow\n" +
+              "Please go to %s to authorize this task\n" +
+              "\nEnter user code: %s\n\n" +
+              "This code will expire in %s minutes\n" +
+              "Task will automatically continue after authorization and token will be saved for future use",
+            verification.verification_uri,
+            verification.user_code,
+            (verification.expires_in / 60).toFixed(0)
+          );
+        },
+      });
+      tokenAuth = await auth({ type: "oauth" });
+      writings.push(
+        createExtractDir.then(
+          writeFile(TOKEN_FILE, JSON.stringify(tokenAuth, null, 2))
+        )
+      );
+    }
+  }
+
+  // Authenticate with token and request workflow runs from GitHub
+  console.log("Fetching new translations...");
+  const octokit = new (Octokit.plugin(retry))({
+    userAgent: "Fetch Nightly Translations",
+    auth: tokenAuth.token,
+  });
+
+  const workflowRunsResponse = await octokit.rest.actions.listWorkflowRuns({
+    owner: OWNER,
+    repo: REPO,
+    workflow_id: WORKFLOW_NAME,
+    status: "success",
+    event: "schedule",
+    per_page: 1,
+    exclude_pull_requests: true,
+  });
+  if (workflowRunsResponse.data.total_count === 0) {
+    throw Error("No successful nightly workflow runs found");
+  }
+  const latestNightlyRun = workflowRunsResponse.data.workflow_runs[0];
+
+  // Stop if current is already the latest, otherwise Find the translations artifact
+  if (currentArtifact?.workflow_run.id === latestNightlyRun.id) {
+    console.log("Stopping because current translations are still the latest");
+    return;
+  }
+  const latestArtifact = (
+    await octokit.actions.listWorkflowRunArtifacts({
+      owner: OWNER,
+      repo: REPO,
+      run_id: latestNightlyRun.id,
+    })
+  ).data.artifacts.find((artifact) => artifact.name === ARTIFACT_NAME);
+  if (!latestArtifact) {
+    throw Error("Latest nightly workflow run has no translations artifact");
+  }
+  writings.push(
+    createExtractDir.then(
+      writeFile(ARTIFACT_FILE, JSON.stringify(latestArtifact, null, 2))
+    )
+  );
+
+  // Remove the current translations
+  const deleteCurrent = Promise.all(writings).then(
+    deleteAsync([`${EXTRACT_DIR}/*`, `!${ARTIFACT_FILE}`, `!${TOKEN_FILE}`])
+  );
+
+  // Get the download URL and follow the redirect to download (stored as ArrayBuffer)
+  const downloadResponse = await octokit.actions.downloadArtifact({
+    owner: OWNER,
+    repo: REPO,
+    artifact_id: latestArtifact.id,
+    archive_format: "zip",
+  });
+  if (downloadResponse.status !== 200) {
+    throw Error("Failure downloading translations artifact");
+  }
+
+  // Artifact is a tarball, but GitHub adds it to a zip file
+  console.log("Unpacking downloaded translations...");
+  const zip = await jszip.loadAsync(downloadResponse.data);
+  await deleteCurrent;
+  const extractStream = zip.file(/.*/)[0].nodeStream().pipe(extract());
+  await new Promise((resolve, reject) => {
+    extractStream.on("close", resolve).on("error", reject);
+  });
+});
+
+gulp.task(
+  "setup-and-fetch-nightly-translations",
+  gulp.series(
+    "allow-setup-fetch-nightly-translations",
+    "fetch-nightly-translations"
+  )
+);

build-scripts/gulp/knx.js

@@ -6,6 +6,7 @@ import "./compress.js";
 import "./entry-html.js";
 import "./gen-icons-json.js";
 import "./rspack.js";
+import "./translations.js";
 
 gulp.task(
   "develop-knx",
@@ -15,6 +16,8 @@ gulp.task(
     },
     "clean-knx",
     "gen-icons-json",
+    "build-translations",
+    "build-locale-data",
     "gen-index-knx-dev",
     "rspack-watch-knx",
   ),
@@ -29,6 +32,8 @@ gulp.task(
     "clean-knx",
     "ensure-knx-build-dir",
     "gen-icons-json",
+    "build-translations",
+    "build-locale-data",
     "rspack-prod-knx",
     "gen-index-knx-prod",
     ...// Don't compress running tests

build-scripts/gulp/locale-data.js

@@ -0,0 +1,69 @@
+import { deleteSync } from "del";
+import { mkdir, readFile, writeFile } from "fs/promises";
+import gulp from "gulp";
+import { join, resolve } from "node:path";
+import paths from "../paths.cjs";
+
+const formatjsDir = join(paths.polymer_dir, "node_modules", "@formatjs");
+const outDir = join(paths.upstream_build_dir, "locale-data");
+
+const INTL_POLYFILLS = {
+  "intl-datetimeformat": "DateTimeFormat",
+  "intl-displaynames": "DisplayNames",
+  "intl-listformat": "ListFormat",
+  "intl-numberformat": "NumberFormat",
+  "intl-relativetimeformat": "RelativeTimeFormat",
+};
+
+const convertToJSON = async (
+  pkg,
+  lang,
+  subDir = "locale-data",
+  addFunc = "__addLocaleData",
+  skipMissing = true,
+) => {
+  let localeData;
+  try {
+    // use "pt" for "pt-BR", because "pt-BR" is unsupported by @formatjs
+    const language = lang === "pt-BR" ? "pt" : lang;
+
+    localeData = await readFile(join(formatjsDir, pkg, subDir, `${language}.js`), "utf-8");
+  } catch (e) {
+    // Ignore if language is missing (i.e. not supported by @formatjs)
+    if (e.code === "ENOENT" && skipMissing) {
+      console.warn(`Skipped missing data for language ${lang} from ${pkg}`);
+      return;
+    }
+    throw e;
+  }
+  // Convert to JSON
+  const obj = INTL_POLYFILLS[pkg];
+  const dataRegex = new RegExp(`Intl\\.${obj}\\.${addFunc}\\((?<data>.*)\\)`, "s");
+  localeData = localeData.match(dataRegex)?.groups?.data;
+  if (!localeData) {
+    throw Error(`Failed to extract data for language ${lang} from ${pkg}`);
+  }
+  // Parse to validate JSON, then stringify to minify
+  localeData = JSON.stringify(JSON.parse(localeData));
+  await writeFile(join(outDir, `${pkg}/${lang}.json`), localeData);
+};
+
+gulp.task("clean-locale-data", async () => deleteSync([outDir]));
+
+gulp.task("create-locale-data", async () => {
+  const translationMeta = JSON.parse(
+    await readFile(resolve(paths.translations_src, "translationMetadata.json"), "utf-8"),
+  );
+  const conversions = [];
+  for (const pkg of Object.keys(INTL_POLYFILLS)) {
+    // eslint-disable-next-line no-await-in-loop
+    await mkdir(join(outDir, pkg), { recursive: true });
+    for (const lang of Object.keys(translationMeta)) {
+      conversions.push(convertToJSON(pkg, lang));
+    }
+  }
+  conversions.push(convertToJSON("intl-datetimeformat", "add-all-tz", ".", "__addTZData", false));
+  await Promise.all(conversions);
+});
+
+gulp.task("build-locale-data", gulp.series("clean-locale-data", "create-locale-data"));