Compare commits

...

10 Commits

Author SHA1 Message Date
Teffen Ellis
eb4163a428 Fix issues surrounding race conditions, stale files. 2025-11-14 23:05:13 +01:00
Teffen Ellis
1525bff851 Prep locale directory for clean up. 2025-11-14 21:43:36 +01:00
Teffen Ellis
e16136f086 Fix order of commands. 2025-11-14 21:35:46 +01:00
Teffen Ellis
0ed9e3b636 Fix hanging process. 2025-11-14 21:35:27 +01:00
Teffen Ellis
0c707a7d44 Fix missing directory constant. 2025-11-14 20:18:22 +01:00
Teffen Ellis
83699cdc69 Add color errors. 2025-11-14 20:18:14 +01:00
Teffen Ellis
8aa5814d91 Fix stale promise. 2025-11-14 20:08:43 +01:00
Teffen Ellis
c9c13665f6 Fix issue where ESBuild context persists after shutdown. 2025-11-14 19:27:16 +01:00
Teffen Ellis
25f1be496f Clean up default language behavior. 2025-11-14 18:51:57 +01:00
Teffen Ellis
37f64ca773 Flesh out clean up. 2025-11-14 18:09:53 +01:00
16 changed files with 308 additions and 18969 deletions

6
web/.gitignore vendored
View File

@@ -2,6 +2,12 @@
# Created by https://www.gitignore.io/api/node
# Edit at https://www.gitignore.io/?templates=node
#region Locale
src/locales/*.ts
xliff/en.xlf
xliff/pseudo-LOCALE.xlf
### Node ###
# Logs
logs

View File

@@ -1,12 +1,15 @@
/**
* @file MDX plugin for ESBuild.
*
* @import { Plugin, PluginBuild, BuildContext } from "esbuild"
* @import { Plugin, PluginBuild, BuildContext, BuildOptions } from "esbuild"
* @import { BaseLogger } from "pino"
*/
import { readFile } from "node:fs/promises";
import { createRequire } from "node:module";
import { dirname, join } from "node:path";
import { dirname, join, relative } from "node:path";
import { ConsoleLogger } from "#logger/node";
import { resolvePackage } from "@goauthentik/core/paths/node";
@@ -16,12 +19,23 @@ const CSSNamespace = /** @type {const} */ ({
Bundled: "css-bundled",
});
/**
* @typedef StyleLoaderPluginOptions
*
* @property {boolean} [watch] Whether to watch for file changes.
* @property {BaseLogger} [logger]
*/
/**
* Selectively apply the ESBuild `css` loader.
*
* @param {StyleLoaderPluginOptions} [options]
* @returns {Plugin}
*/
export function styleLoaderPlugin() {
export function styleLoaderPlugin({
watch = false,
logger = ConsoleLogger.child({ name: "style-loader-plugin" }),
} = {}) {
const patternflyPath = resolvePackage("@patternfly/patternfly", import.meta);
const require = createRequire(import.meta.url);
@@ -56,7 +70,8 @@ export function styleLoaderPlugin() {
const disposables = new Map();
build.onDispose(async () => {
for (const ctx of disposables.values()) {
for (const [filePath, ctx] of disposables) {
logger.debug(`Disposing CSS build context for ${filePath}`);
await ctx.dispose();
}
});
@@ -114,33 +129,50 @@ export function styleLoaderPlugin() {
const cssContent = await readFile(args.path, "utf8");
let context = disposables.get(args.path);
if (!context) {
context = await build.esbuild.context({
stdin: {
contents: cssContent,
resolveDir: dirname(args.path),
loader: "css",
},
metafile: true,
bundle: true,
write: false,
minify: build.initialOptions.minify || false,
logLevel: "silent",
loader: { ".woff": "empty", ".woff2": "empty" },
plugins: [
{
name: "font-resolver",
setup(fontBuild) {
fontBuild.onResolve(...fontResolverArgs);
},
/**
* @type {BuildOptions}
*/
const buildOptions = {
stdin: {
contents: cssContent,
resolveDir: dirname(args.path),
loader: "css",
},
metafile: true,
bundle: true,
write: false,
minify: build.initialOptions.minify || false,
logLevel: "silent",
loader: { ".woff": "empty", ".woff2": "empty" },
plugins: [
{
name: "font-resolver",
setup(fontBuild) {
fontBuild.onResolve(...fontResolverArgs);
},
],
});
},
],
};
disposables.set(args.path, context);
if (!watch) {
const result = await build.esbuild.build(buildOptions);
const bundledCSS = result.outputFiles?.[0].text;
return {
contents: bundledCSS,
loader: "text",
};
}
await context.cancel();
if (!context) {
const relativePath = relative(absWorkingDir, args.path);
logger.debug(`Watching ${relativePath}`);
context = await build.esbuild.context(buildOptions);
disposables.set(args.path, context);
} else {
await context.cancel();
}
// Resolve the CSS content by bundling it with ESBuild.
const result = await context.rebuild();

View File

@@ -4,7 +4,6 @@
"targetLocales": [
"cs_CZ",
"de",
"en",
"es",
"fr",
"it",

View File

@@ -5,12 +5,11 @@
"private": true,
"scripts": {
"build": "wireit",
"build-locales": "wireit",
"build-locales:build": "wireit",
"build-proxy": "wireit",
"build:locales": "wireit",
"build:sfe": "npm run build -w @goauthentik/web-sfe",
"esbuild:watch": "node scripts/build-web.mjs --watch",
"extract-locales": "wireit",
"bundler:watch": "node scripts/build-web.mjs --watch",
"extract-locales": "lit-localize extract",
"format": "wireit",
"lint": "eslint --fix .",
"lint-check": "eslint --max-warnings 0 .",
@@ -27,7 +26,7 @@
"test": "vitest",
"test:e2e": "playwright test",
"tsc": "wireit",
"watch": "run-s build-locales esbuild:watch"
"watch": "run-s build:locales bundler:watch"
},
"type": "module",
"exports": {
@@ -228,7 +227,7 @@
"./dist/styles/**"
],
"dependencies": [
"build-locales",
"build:locales",
"build:sfe"
],
"env": {
@@ -241,16 +240,13 @@
"build-proxy": {
"command": "node scripts/build-web.mjs --proxy",
"dependencies": [
"build-locales"
"build:locales"
]
},
"build-locales:build": {
"command": "lit-localize build"
},
"build-locales:repair": {
"locales:repair": {
"command": "prettier --write ./src/locale-codes.ts"
},
"build-locales": {
"build:locales": {
"command": "node scripts/build-locales.mjs",
"files": [
"./xliff/*.xlf"
@@ -260,9 +256,6 @@
"./src/locale-codes.ts"
]
},
"extract-locales": {
"command": "lit-localize extract"
},
"lint:components": {
"command": "lit-analyzer src"
},
@@ -272,7 +265,7 @@
"NODE_OPTIONS": "--max_old_space_size=8192"
},
"dependencies": [
"build-locales"
"build:locales"
]
},
"lint:lockfile": {
@@ -304,7 +297,7 @@
"NODE_OPTIONS": "--max_old_space_size=8192"
},
"dependencies": [
"build-locales"
"build:locales"
]
}
},

View File

@@ -9,8 +9,6 @@ import { fileURLToPath } from "node:url";
import { DistDirectoryName } from "#paths";
import { resolvePackage } from "@goauthentik/core/paths/node";
const relativeDirname = dirname(fileURLToPath(import.meta.url));
//#region Base paths
@@ -48,8 +46,6 @@ export const DistDirectory = /** @type {`${WebPackageIdentifier}/${DistDirectory
* Matches the type defined in the ESBuild context.
*/
const patternflyPath = resolvePackage("@patternfly/patternfly", import.meta);
/**
* Entry points available for building.
*

View File

@@ -11,93 +11,176 @@
* long spew of "this string is not translated" and replacing it with a
* summary of how many strings are missing with respect to the source locale.
*
* @import { ConfigFile } from "@lit/localize-tools/lib/types/config.js"
* @import { Stats } from "node:fs";
* @import { RuntimeOutputConfig } from "@lit/localize-tools/lib/types/modes.js"
*/
import { spawnSync } from "node:child_process";
import { readFileSync, statSync } from "node:fs";
import path from "node:path";
import * as fs from "node:fs/promises";
import path, { resolve } from "node:path";
import { generatePseudoLocaleModule } from "./pseudolocalize.mjs";
// import localizeRules from "../lit-localize.json" with { type: "json" };
import { ConsoleLogger } from "#logger/node";
import { PackageRoot } from "#paths/node";
/**
* @type {ConfigFile}
*/
const localizeRules = JSON.parse(
readFileSync(path.join(PackageRoot, "lit-localize.json"), "utf-8"),
import { readConfigFileAndWriteSchema } from "@lit/localize-tools/lib/config.js";
import { RuntimeLitLocalizer } from "@lit/localize-tools/lib/modes/runtime.js";
const logger = ConsoleLogger.child({ name: "Locales" });
const localizeRules = readConfigFileAndWriteSchema(path.join(PackageRoot, "lit-localize.json"));
if (localizeRules.interchange.format !== "xliff") {
logger.error("Unsupported interchange type, expected 'xliff'");
process.exit(1);
}
const XLIFFPath = resolve(PackageRoot, localizeRules.interchange.xliffDir);
const EmittedLocalesDirectory = resolve(
PackageRoot,
/** @type {string} */ (localizeRules.output.outputDir),
);
/**
*
* @param {string} loc
* @returns {boolean}
*/
function generatedFileIsUpToDateWithXliffSource(loc) {
const xliff = path.join("./xliff", `${loc}.xlf`);
const gened = path.join("./src/locales", `${loc}.ts`);
async function cleanEmittedLocales() {
logger.info("♻️ Cleaning previously emitted locales...");
logger.info(`♻️ ${EmittedLocalesDirectory}`);
// Returns false if: the expected XLF file doesn't exist, The expected
// generated file doesn't exist, or the XLF file is newer (has a higher date)
// than the generated file. The missing XLF file is important enough it
// generates a unique error message and halts the build.
await fs.rm(EmittedLocalesDirectory, {
recursive: true,
force: true,
});
await fs.mkdir(EmittedLocalesDirectory, {
recursive: true,
});
logger.info(`♻️ Done!`);
}
/**
* Returns false if: the expected XLF file doesn't exist, The expected
* generated file doesn't exist, or the XLF file is newer (has a higher date)
* than the generated file. The missing XLF file is important enough it
* generates a unique error message and halts the build.
*
* @param {string} localeCode
* @returns {Promise<boolean>}
*/
async function checkIfEmittedFileCurrent(localeCode) {
const xliffPath = path.join(XLIFFPath, `${localeCode}.xlf`);
const emittedPath = path.join(EmittedLocalesDirectory, `${localeCode}.ts`);
/**
* @type {Stats}
*/
let xlfStat;
let xliffStat;
try {
xlfStat = statSync(xliff);
xliffStat = await fs.stat(xliffPath);
} catch (_error) {
console.error(`lit-localize expected '${loc}.xlf', but XLF file is not present`);
logger.error(`XLIFF source file missing for locale '${localeCode}': ${xliffPath}`);
process.exit(1);
}
/**
* @type {Stats}
*/
let genedStat;
let emittedStat;
// If the generated file doesn't exist, of course it's not up to date.
try {
genedStat = statSync(gened);
emittedStat = await fs.stat(emittedPath);
} catch (_error) {
return false;
}
// if the generated file is the same age or newer (date is greater) than the xliff file, it's
// Possible if the script was interrupted between clearing and generating.
if (emittedStat.size === 0) {
return false;
}
// If the emitted file is the same age or newer (date is greater) than the xliff file, it's
// presumed to have been generated by that file and is up-to-date.
return genedStat.mtimeMs >= xlfStat.mtimeMs;
return emittedStat.mtimeMs >= xliffStat.mtimeMs;
}
// For all the expected files, find out if any aren't up-to-date.
const upToDate = localizeRules.targetLocales.reduce(
(acc, loc) => acc && generatedFileIsUpToDateWithXliffSource(loc),
true,
);
/**
* Checks if all the locale source files are up-to-date with their XLIFF sources.
* @returns {Promise<boolean>}
*/
async function checkIfLocalesAreCurrent() {
logger.info("Reading locale configuration...");
if (!upToDate) {
const status = spawnSync("npm", ["run", "build-locales:build"], { encoding: "utf8" });
const targetLocales = localizeRules.targetLocales.filter((localeCode) => {
return localeCode !== "pseudo-LOCALE";
});
// Count all the missing message warnings
const counts = status.stderr.split("\n").reduce((acc, line) => {
const match = /^([\w-]+) message/.exec(line);
if (!match) {
return acc;
}
acc.set(match[1], (acc.get(match[1]) || 0) + 1);
return acc;
}, new Map());
logger.info(`Checking ${targetLocales.length} source files...`);
const locales = Array.from(counts.keys());
locales.sort();
let outOfDateCount = 0;
const report = locales
.map((locale) => `Locale '${locale}' has ${counts.get(locale)} missing translations`)
.join("\n");
await Promise.all(
targetLocales.map(async (localeCode) => {
const current = await checkIfEmittedFileCurrent(localeCode);
console.log(`Translation tables rebuilt.\n${report}\n`);
if (!current) {
logger.info(`Locale '${localeCode}' is out-of-date.`);
outOfDateCount++;
}
}),
);
return outOfDateCount === 0;
}
console.log("Locale ./src is up-to-date");
export async function generateLocaleModules() {
logger.info("Updating pseudo-locale...");
await generatePseudoLocaleModule();
logger.info("Generating locale modules...");
const localizer = new RuntimeLitLocalizer({
...localizeRules,
output: /** @type {RuntimeOutputConfig} */ (localizeRules.output),
});
await localizer.build();
logger.info("Complete.");
}
async function delegateCommand() {
const command = process.argv[2];
switch (command) {
case "--clean":
return cleanEmittedLocales();
case "--check":
return checkIfLocalesAreCurrent();
case "--force":
return generateLocaleModules();
}
const upToDate = await checkIfLocalesAreCurrent();
if (upToDate) {
logger.info("Locale is up-to-date!");
return;
}
logger.info("Locale ./src is out-of-date, rebuilding...");
return generateLocaleModules();
}
await delegateCommand()
.then(() => {
process.exit(0);
})
.catch((error) => {
logger.error(`Error during locale build: ${error}`);
process.exit(1);
});

View File

@@ -19,7 +19,7 @@ import { ConsoleLogger } from "#logger/node";
import { DistDirectory, EntryPoint, PackageRoot } from "#paths/node";
import { NodeEnvironment } from "@goauthentik/core/environment/node";
import { MonoRepoRoot, resolvePackage } from "@goauthentik/core/paths/node";
import { MonoRepoRoot } from "@goauthentik/core/paths/node";
import { BuildIdentifier } from "@goauthentik/core/version/node";
import { deepmerge } from "deepmerge-ts";
@@ -37,8 +37,6 @@ const publicBundledDefinitions = Object.fromEntries(
);
logger.info(publicBundledDefinitions, "Bundle definitions");
const patternflyPath = resolvePackage("@patternfly/patternfly", import.meta);
/**
* @type {Readonly<BuildOptions>}
*/
@@ -53,6 +51,7 @@ const BASE_ESBUILD_OPTIONS = {
minify: NodeEnvironment === "production",
legalComments: "external",
splitting: true,
color: !process.env.NO_COLOR,
treeShaking: true,
tsconfig: path.resolve(PackageRoot, "tsconfig.build.json"),
loader: {
@@ -80,7 +79,6 @@ const BASE_ESBUILD_OPTIONS = {
},
],
}),
styleLoaderPlugin(),
mdxPlugin({
root: MonoRepoRoot,
}),
@@ -140,9 +138,11 @@ function doHelp() {
process.exit(0);
}
/**
*
* @returns {Promise<() => Promise<void>>} dispose
*/
async function doWatch() {
const { promise, resolve, reject } = Promise.withResolvers();
logger.info(`🤖 Watching entry points:\n\t${Object.keys(EntryPoint).join("\n\t")}`);
const entryPoints = Object.values(EntryPoint);
@@ -158,7 +158,7 @@ async function doWatch() {
const buildOptions = createESBuildOptions({
entryPoints,
plugins: developmentPlugins,
plugins: [...developmentPlugins, styleLoaderPlugin({ logger, watch: true })],
});
const buildContext = await esbuild.context(buildOptions);
@@ -176,34 +176,23 @@ async function doWatch() {
logger.info(`🔓 ${httpURL.href}`);
logger.info(`🔒 ${httpsURL.href}`);
let disposing = false;
const delegateShutdown = () => {
return () => {
logger.flush();
console.log("");
console.info("");
console.info("🛑 Stopping file watcher...");
// We prevent multiple attempts to dispose the context
// because ESBuild will repeatedly restart its internal clean-up logic.
// However, sending a second SIGINT will still exit the process immediately.
if (disposing) return;
disposing = true;
return buildContext.dispose().then(resolve).catch(reject);
return buildContext.dispose();
};
process.on("SIGINT", delegateShutdown);
return promise;
}
async function doBuild() {
logger.info(`🤖 Watching entry points:\n\t${Object.keys(EntryPoint).join("\n\t")}`);
logger.info(`🤖 Building entry points:\n\t${Object.keys(EntryPoint).join("\n\t")}`);
const entryPoints = Object.values(EntryPoint);
const buildOptions = createESBuildOptions({
entryPoints,
plugins: [styleLoaderPlugin({ logger })],
});
await esbuild.build(buildOptions);
@@ -245,10 +234,43 @@ await cleanDistDirectory()
// ---
.then(() =>
delegateCommand()
.then(() => {
process.exit(0);
.then((dispose) => {
if (!dispose) {
process.exit(0);
}
/**
* @type {Promise<void>}
*/
const signalListener = new Promise((resolve) => {
// We prevent multiple attempts to dispose the context
// because ESBuild will repeatedly restart its internal clean-up logic.
// However, sending a second SIGINT will still exit the process immediately.
let signalCount = 0;
process.on("SIGINT", () => {
if (signalCount > 3) {
// Something is taking too long and the user wants to exit now.
console.log("🛑 Forcing exit...");
process.exit(0);
}
});
process.once("SIGINT", () => {
signalCount++;
dispose().finally(() => {
console.log("✅ Done!");
resolve();
});
});
logger.info("🚪 Press Ctrl+C to exit.");
});
return signalListener;
})
.catch(() => {
process.exit(1);
}),
.then(() => process.exit(0))
.catch(() => process.exit(1)),
);

View File

@@ -11,9 +11,12 @@
import { readFileSync } from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { PackageRoot } from "#paths/node";
import { isMain } from "@goauthentik/core/scripting/node";
import pseudolocale from "pseudolocale";
import { makeFormatter } from "@lit/localize-tools/lib/formatters/index.js";
@@ -22,6 +25,7 @@ import { TransformLitLocalizer } from "@lit/localize-tools/lib/modes/transform.j
const pseudoLocale = /** @type {Locale} */ ("pseudo-LOCALE");
const targetLocales = [pseudoLocale];
const __dirname = fileURLToPath(new URL(".", import.meta.url));
/**
* @type {ConfigFile}
@@ -59,10 +63,16 @@ const pseudoMessagify = (message) => ({
),
});
const localizer = new TransformLitLocalizer(config);
const { messages } = localizer.extractSourceMessages();
const translations = messages.map(pseudoMessagify);
const sorted = sortProgramMessages([...messages]);
const formatter = makeFormatter(config);
export async function generatePseudoLocaleModule() {
const localizer = new TransformLitLocalizer(config);
const { messages } = localizer.extractSourceMessages();
const translations = messages.map(pseudoMessagify);
const sorted = sortProgramMessages([...messages]);
const formatter = makeFormatter(config);
formatter.writeOutput(sorted, new Map([[pseudoLocale, translations]]));
await formatter.writeOutput(sorted, new Map([[pseudoLocale, translations]]));
}
if (isMain(import.meta)) {
generatePseudoLocaleModule();
}

View File

@@ -1,15 +1,13 @@
import { AkLocale, LocaleRow } from "./types.js";
import * as _enLocale from "#locales/en";
import type { LocaleModule } from "@lit/localize";
import { msg } from "@lit/localize";
export const DEFAULT_FALLBACK = "en";
const enLocale: LocaleModule = _enLocale;
export { enLocale };
export const enLocale: LocaleModule = {
templates: {},
};
// NOTE: This table cannot be made any shorter, despite all the repetition of syntax. Bundlers look
// for the `import` #a *string target* for doing alias substitution, so putting
@@ -43,7 +41,7 @@ const debug: LocaleRow = [
// prettier-ignore
const LOCALE_TABLE: LocaleRow[] = [
["de", /^de([_-]|$)/i, () => msg("German"), () => import("#locales/de")],
["en", /^en([_-]|$)/i, () => msg("English"), () => import("#locales/en")],
["en", /^en([_-]|$)/i, () => msg("English"), () => Promise.resolve(enLocale)],
["es", /^es([_-]|$)/i, () => msg("Spanish"), () => import("#locales/es")],
["fr", /^fr([_-]|$)/i, () => msg("French"), () => import("#locales/fr")],
["it", /^it([_-]|$)/i, () => msg("Italian"), () => import("#locales/it")],

View File

@@ -17,17 +17,17 @@ export const LOCALES = RAW_LOCALES.map((locale) =>
locale.code === "en" ? { ...locale, locale: async () => enLocale } : locale,
);
export function getBestMatchLocale(locale: string): AkLocale | undefined {
return LOCALES.find((l) => l.match.test(locale));
export function getBestMatchLocale(locale: string): AkLocale | null {
return LOCALES.find((l) => l.match.test(locale)) || null;
}
// This looks weird, but it's sensible: we have several candidates, and we want to find the first
// one that has a supported locale. Then, from *that*, we have to extract that first supported
// locale.
export function findSupportedLocale(candidates: string[]) {
export function findSupportedLocale(candidates: string[]): AkLocale | null {
const candidate = candidates.find((candidate: string) => getBestMatchLocale(candidate));
return candidate ? getBestMatchLocale(candidate) : undefined;
return candidate ? getBestMatchLocale(candidate) : null;
}
export function localeCodeFromUrl(param = "locale") {

View File

@@ -1,6 +1,11 @@
import type { LocaleModule } from "@lit/localize";
export type LocaleRow = [string, RegExp, () => string, () => Promise<LocaleModule>];
export type LocaleRow = [
code: string,
pattern: RegExp,
label: () => string,
loader: () => Promise<LocaleModule>,
];
export type AkLocale = {
code: string;

View File

@@ -13,7 +13,6 @@ export const sourceLocale = `en`;
export const targetLocales = [
`cs_CZ`,
`de`,
`en`,
`es`,
`fr`,
`it`,
@@ -37,7 +36,6 @@ export const allLocales = [
`cs_CZ`,
`de`,
`en`,
`en`,
`es`,
`fr`,
`it`,

View File

@@ -1 +0,0 @@
*.ts

18
web/types/locale.d.ts vendored Normal file
View File

@@ -0,0 +1,18 @@
/**
* This module is used to satisfy imports from `#locales/*` which have either
* not yet been generated, or are missing.
*
* ```sh
* npm run build:locales
* ```
*/
declare module "#locales/*" {
/**
* If you see this, try running `npm run build:locales` to generate locale files.
*/
type MissingLocale = symbol & { readonly __brand?: never };
const missingLocale: MissingLocale;
export const templates: MissingLocale;
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff