Translation handling (#142)

.github/workflows/QA.yaml

@@ -37,6 +37,10 @@ jobs:
         working-directory: scraper
         run: inv check-pyright
 
+      - name: Check UI translation keys
+        working-directory: scraper
+        run: inv check-ui-keys
+
   check-zimui-qa:
     runs-on: ubuntu-24.04
 

scraper/src/fcc2zim/check_ui_keys.py

@@ -0,0 +1,56 @@
+"""Checks that the ZIM UI only uses translation keys that are listed in ui_keys.yaml."""
+
+import re
+import sys
+from typing import cast
+
+import yaml
+
+from fcc2zim.constants import ROOT_DIR
+
+UI_KEYS_PATH = ROOT_DIR / "ui_keys.yaml"
+ZIMUI_SRC = ROOT_DIR.parent.parent.parent / "zimui" / "src"
+
+T_CALL_RE = re.compile(r"main\.t\('([^']+)'")
+
+
+def flatten_keys(
+    data: dict[str, object], prefix: tuple[str, ...] = ()
+) -> set[tuple[str, ...]]:
+    result: set[tuple[str, ...]] = set()
+    for key, value in data.items():
+        path = (*prefix, key)
+        if value is None:
+            result.add(path)
+        elif isinstance(value, dict):
+            dict_value = cast(dict[str, object], value)
+            if set(dict_value.keys()) == {"placeholders"}:
+                result.add(path)
+            else:
+                result.update(flatten_keys(dict_value, path))
+    return result
+
+
+def main() -> None:
+    spec: dict[str, object] = yaml.safe_load(UI_KEYS_PATH.read_text())
+    allowed = flatten_keys(spec)
+
+    errors: list[str] = []
+    for file in list(ZIMUI_SRC.rglob("*.vue")) + list(ZIMUI_SRC.rglob("*.ts")):
+        content = file.read_text()
+
+        for match in T_CALL_RE.finditer(content):
+            key_path = tuple(match.group(1).split("."))
+            if key_path not in allowed:
+                errors.append(f"{file.name}: unknown key {match.group(1)}")
+
+    if errors:
+        for err in errors:
+            print(err, file=sys.stderr)  # noqa: T201
+        sys.exit(1)
+
+    print("All UI translation keys are in ui_keys.yaml")  # noqa: T201
+
+
+if __name__ == "__main__":
+    main()

scraper/src/fcc2zim/locale_validation.py

@@ -0,0 +1,114 @@
+import json
+import re
+from pathlib import Path
+from typing import cast
+
+import yaml
+
+from fcc2zim.constants import ROOT_DIR
+from fcc2zim.context import Context
+
+logger = Context.logger
+PLACEHOLDER_RE = re.compile(r"\{\{(\w+)\}\}")
+
+
+def validate_locales(locales_path: Path, included_superblocks: dict[str, list[str]]):
+    """Checks locale files against ui_keys.yaml and fails it if anything seems off."""
+
+    spec = yaml.safe_load((ROOT_DIR / "ui_keys.yaml").read_text())
+    translations = json.loads(
+        (locales_path / "translations.json").read_text(encoding="utf-8")
+    )
+    intro = json.loads((locales_path / "intro.json").read_text(encoding="utf-8"))
+    errors: list[str] = []
+
+    # checks static keys from translations.json and intro.json
+    for file_key, data in [("translations", translations), ("intro", intro)]:
+        if file_key not in spec:
+            continue
+        for section, keys in spec[file_key].items():
+            if section not in data:
+                errors.append(f"{file_key}.json: missing section '{section}'")
+                continue
+            for key, opts in keys.items():
+                if key not in data[section]:
+                    errors.append(f"{file_key}.json: missing key '{section}.{key}'")
+                    continue
+
+                value = data[section][key]
+                if not isinstance(value, str):
+                    continue
+
+                actual: set[str] = set(PLACEHOLDER_RE.findall(value))
+                expected: set[str] = (
+                    set(opts.get("placeholders", [])) if opts else set()
+                )
+
+                for name in expected - actual:
+                    errors.append(
+                        f"{file_key}.json: '{section}.{key}'"
+                        f" missing placeholder {{{{{name}}}}}"
+                    )
+                for name in actual - expected:
+                    errors.append(
+                        f"{file_key}.json: '{section}.{key}'"
+                        f" has unexpected placeholder {{{{{name}}}}}"
+                    )
+
+    # checks intro.json has entries for every included superblock/course
+    for superblock, courses in included_superblocks.items():
+        if superblock not in intro:
+            errors.append(f"intro.json: missing superblock '{superblock}'")
+            continue
+
+        sb = intro[superblock]
+        for field in ("title", "intro", "blocks"):
+            if field not in sb:
+                errors.append(f"intro.json: superblock '{superblock}' has no '{field}'")
+
+        # checks superblock intro has no unexpected placeholders
+        if "title" in sb and isinstance(sb["title"], str):
+            for ph in PLACEHOLDER_RE.findall(sb["title"]):
+                errors.append(
+                    f"intro.json: '{superblock}.title'"
+                    f" has unexpected placeholder {{{{{ph}}}}}"
+                )
+        if "intro" in sb and isinstance(sb["intro"], list):
+            for para in sb["intro"]:
+                if isinstance(para, str):
+                    for ph in PLACEHOLDER_RE.findall(para):
+                        errors.append(
+                            f"intro.json: '{superblock}.intro'"
+                            f" has unexpected placeholder {{{{{ph}}}}}"
+                        )
+
+        blocks = sb.get("blocks", {})
+        for course in courses:
+            if course not in blocks:
+                errors.append(
+                    f"intro.json: can't find block '{course}' in '{superblock}'"
+                )
+                continue
+            for field in ("title", "intro"):
+                if field not in blocks[course]:
+                    errors.append(f"intro.json: block '{course}' has no '{field}'")
+                    continue
+                val = blocks[course][field]
+                if isinstance(val, str):
+                    texts: list[str] = [val]
+                elif isinstance(val, list):
+                    texts = cast(list[str], val)
+                else:
+                    texts = []
+                for text in texts:
+                    for ph in PLACEHOLDER_RE.findall(text):
+                        errors.append(
+                            f"intro.json: '{superblock}.blocks.{course}.{field}'"
+                            f" has unexpected placeholder {{{{{ph}}}}}"
+                        )
+
+    if errors:
+        for err in errors:
+            logger.error(err)
+        raise ValueError(f"Locale validation failed with {len(errors)} error(s)")
+    logger.info("Locale files validated")

scraper/src/fcc2zim/prebuild.py

@@ -4,6 +4,7 @@
 
 from fcc2zim.challenge import Challenge
 from fcc2zim.context import Context
+from fcc2zim.locale_validation import validate_locales
 
 logger = Context.logger
 
@@ -101,6 +102,8 @@ def prebuild_command(
         if "blocks" in superblock_content:
             superblocks[superblock_name] = superblock_content["blocks"]
 
+    included_superblocks: dict[str, list[str]] = {}
+
     # eg. ['basic-javascript', 'debugging']
     for course in course_list:
         logger.debug(f"Prebuilding {course}")
@@ -127,6 +130,10 @@ def prebuild_command(
             )
         superblock = matching_superblocks[0]
 
+        if superblock not in included_superblocks:
+            included_superblocks[superblock] = []
+        included_superblocks[superblock].append(course)
+
         challenge_list: list[Challenge] = []
         for file in get_challenges_for_lang(challenges, course, fcc_lang):
             challenge = Challenge(superblock, file)
@@ -144,5 +151,6 @@ def prebuild_command(
 
     # Copy all the locales for this language
     write_locales_to_path(locales, curriculum_dist)
+    validate_locales(curriculum_dist / "locales", included_superblocks)
     logger.info(f"Prebuilt curriculum into {curriculum_dist}")
     logger.info("Scraper: prebuild phase finished")

scraper/src/fcc2zim/ui_keys.yaml

@@ -0,0 +1,25 @@
+# Keys from locale files that the ZIM UI uses.
+# Dynamic content isn't listed here since those keys depend on which courses and superblocks are included.
+
+translations:
+  buttons:
+    check-code:
+    reset-lesson:
+    run:
+    go-to-next:
+  learn:
+    heading:
+    reset:
+    reset-warn:
+      placeholders:
+        - title
+    reset-warn-2:
+
+motivation:
+  compliment:
+
+intro:
+  misc-text:
+    courses:
+    expand:
+    collapse:

scraper/tasks.py

@@ -76,6 +76,12 @@ def check_pyright(ctx: Context, args: str = ""):
     ctx.run(f"pyright {args}", pty=use_pty)
 
 
+@task
+def check_ui_keys(ctx: Context):
+    """checks that the UI code only uses translation keys that are from ui_keys.yaml"""
+    ctx.run("python -m fcc2zim.check_ui_keys", pty=use_pty)
+
+
 @task(optional=["args"], help={"args": "check tools (pyright) additional arguments"})
 def checkall(ctx: Context, args: str = ""):
     """check static types"""

zimui/src/components/CourseBlocks.vue

@@ -37,8 +37,8 @@ const completedChallengesCount = computed(() => {
 <template>
   <button @click="toggleOpen">
     <img :src="block_arrow" class="icon" :class="{ open: isOpen }" aria-hidden="true" />
-    <div v-if="isOpen">{{ main.localesIntroMiscText?.collapse }}</div>
-    <div v-else>{{ main.localesIntroMiscText?.expand }}</div>
+    <div v-if="isOpen">{{ main.t('intro.misc-text.collapse') }}</div>
+    <div v-else>{{ main.t('intro.misc-text.expand') }}</div>
     <div class="completed">
       <img
         :src="completedChallengesCount == totalChallengesCount ? checkbox_checked : checkbox_empty"

zimui/src/components/CurriculumsOverview.vue

@@ -10,17 +10,17 @@ import right_arrow from '../assets/right_arrow.svg'
 const main = useMainStore()
 
 const randomQuote = computed(() => {
-  return main.localesMotivation?.motivationalQuotes.random()
+  return main.getRandomQuote()
 })
 </script>
 
 <template>
   <div
     v-if="
-      main.curriculums && main.localesIntro && main.localesMotivation && main.localesTranslations
+      main.curriculums && main.isIntroReady && main.isMotivationReady && main.isTranslationsReady
     "
   >
-    <h1>{{ main.localesTranslations.learn['heading'] }}</h1>
+    <h1>{{ main.t('translations.learn.heading') }}</h1>
     <blockquote>
       <q>{{ randomQuote?.quote }}</q>
       <footer>
@@ -32,7 +32,7 @@ const randomQuote = computed(() => {
         <RouterLink :to="`/${curriculum}`">
           <div class="course">
             <SuperblockIcon :superblock="curriculum" class="icon" />
-            <span>{{ main.localesIntro[curriculum].title }}</span>
+            <span>{{ main.tIntro(`${curriculum}.title`) }}</span>
             <img :src="right_arrow" aria-hidden="true" />
           </div>
         </RouterLink>

zimui/src/components/SuperblockOverview.vue

@@ -17,32 +17,35 @@ const activeCourse = toRef(props, 'activeCourse')
 </script>
 
 <template>
-  <div class="my-2" v-if="main.localesIntro && main.localesIntroMiscText && main.curriculums">
-    <h1>{{ main.localesIntro[superblock].title }}</h1>
+  <div class="my-2" v-if="main.isIntroReady && main.isIntroMiscTextReady && main.curriculums">
+    <h1>{{ main.tIntro(`${superblock}.title`) }}</h1>
     <SuperblockIcon :superblock="superblock" class="icon" />
     <!-- eslint-disable-next-line vue/no-v-html-->
     <p
-      v-for="(p, jdx) in main.localesIntro[superblock].intro"
+      v-for="(p, jdx) in main.tIntro(`${superblock}.intro`) as string[]"
       :key="jdx"
       class="my-2 super-block-intro"
       v-html="p"
     ></p>
-    <h2>{{ main.localesIntroMiscText.courses }}</h2>
+    <h2>{{ main.t('intro.misc-text.courses') }}</h2>
     <div
       class="block-parent"
       :id="`${superblock}-${course}`"
       v-for="(course, cdx) in Object.keys(main.curriculums[superblock])"
       :key="course"
     >
       <div class="block">
-        <h3 class="block-header">{{ main.localesIntro[superblock].blocks[course].title }}</h3>
+        <h3 class="block-header">
+          {{ main.tIntro(`${superblock}.blocks.${course}.title`) }}
+        </h3>
         <div class="block-description">
           <p
-            v-for="(p, jdx) in main.localesIntro[superblock].blocks[course].intro"
+            v-for="(p, jdx) in main.tIntro(
+              `${superblock}.blocks.${course}.intro`
+            ) as string[]"
             :key="jdx"
             v-html="p"
           ></p>
-          {{ main.localesIntro[course] }}
         </div>
         <CourseBlocks
           :superblock="superblock"

zimui/src/components/challenge/ChallengeBreadcrumbs.vue

@@ -16,11 +16,11 @@ const course = computed(() => singlePathParam(route.params.course))
     <nav>
       <ol>
         <li class="breadcrumb-left">
-          <RouterLink to="/">{{ main.localesIntro?.[superblock].title }} </RouterLink>
+          <RouterLink to="/">{{ main.tIntro(`${superblock}.title`) }} </RouterLink>
         </li>
         <li class="breadcrumb-right">
           <RouterLink :to="`/${superblock}/${course}`">
-            {{ main.localesIntro?.[superblock].blocks[course].title }}
+            {{ main.tIntro(`${superblock}.blocks.${course}.title`) }}
           </RouterLink>
         </li>
       </ol>

zimui/src/components/challenge/ChallengeDesktop.vue

@@ -24,21 +24,18 @@ function checkCode() {
   <Splitpanes>
     <Pane class="left">
       <ChallengeInstructions />
-      <div class="buttons" v-if="main.localesTranslations">
+      <div class="buttons" v-if="main.isTranslationsReady">
         <!--Cheat button (dev-only)-->
         <button v-if="main.cheatMode" @click="main.cheatSolution()">Set solution</button>
 
         <!--Run the tests button-->
-        <button
-          :class="{ 'tests-failed-flash': main.testsFlash }"
-          @click="checkCode"
-        >
-          {{ main.localesTranslations.buttons['check-code'] }}
+        <button :class="{ 'tests-failed-flash': main.testsFlash }" @click="checkCode">
+          {{ main.t('translations.buttons.check-code') }}
         </button>
 
         <!--Reset code-->
         <button @click="main.challengeResetDialogActive = true">
-          {{ main.localesTranslations.buttons['reset-lesson'] }}
+          {{ main.t('translations.buttons.reset-lesson') }}
         </button>
 
         <ChallengeDialogs />