From 27784617464e55cd8d5ec6e024c8c9aa3de2c656 Mon Sep 17 00:00:00 2001 From: Eddie Knight Date: Thu, 23 Jul 2026 14:39:02 -0500 Subject: [PATCH] feat: vendor doc-gen tooling and build from the spec's openapi.yaml release asset The site previously cloned gemaraproj/gemara and ran the markdown tooling (openapi2md, lexicon2md, termlinker, parse-nav.sh) out of that checkout's cmd/ directory. Those tools only exist to render this site, so they now live here under tools/ as their own Go module. The build input is now the spec's OpenAPI projection instead of a source checkout: - make gendocs downloads openapi.yaml from the spec release named by GEMARA_REF (default: latest). GEMARA_DIR=/path/to/gemara still generates it locally via the spec's cue2openapi for unreleased schema work, and GEMARA_OPENAPI=/path/to/file uses a pre-generated copy. - deploy.yml downloads the release asset for the resolved ref, falling back to clone-and-generate for refs that predate the asset (and for branch refs like main). The spec repo publishes the openapi.yaml asset starting with its next release; the fallback keeps builds working against older tags. Signed-off-by: Eddie Knight --- .github/workflows/deploy.yml | 32 +- Makefile | 76 ++- README.md | 21 +- tools/go.mod | 13 + tools/go.sum | 12 + tools/internal/cmd/lexicon2md.go | 92 +++ tools/internal/cmd/openapi2md.go | 618 +++++++++++++++++++ tools/internal/cmd/openapi_types.go | 22 + tools/internal/cmd/root.go | 29 + tools/internal/cmd/termlinker.go | 881 +++++++++++++++++++++++++++ tools/main.go | 11 + tools/scripts/parse-nav.sh | 66 ++ tools/scripts/schema-display-name.sh | 18 + 13 files changed, 1844 insertions(+), 47 deletions(-) create mode 100644 tools/go.mod create mode 100644 tools/go.sum create mode 100644 tools/internal/cmd/lexicon2md.go create mode 100644 tools/internal/cmd/openapi2md.go create mode 100644 tools/internal/cmd/openapi_types.go create mode 100644 tools/internal/cmd/root.go create mode 100644 tools/internal/cmd/termlinker.go create mode 100644 tools/main.go create mode 100755 tools/scripts/parse-nav.sh create mode 100644 tools/scripts/schema-display-name.sh diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index fa71c5a..46a0ebe 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -63,14 +63,6 @@ jobs: echo "Building against gemara spec ref: $ref" echo "ref=$ref" >> "$GITHUB_OUTPUT" - - name: Checkout Gemara spec - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - repository: gemaraproj/gemara - ref: ${{ steps.spec.outputs.ref }} - path: .gemara-spec - persist-credentials: false - - name: Setup Pages if: github.event_name != 'pull_request' id: pages @@ -87,8 +79,30 @@ jobs: with: go-version: '1.25' + - name: Fetch spec OpenAPI + env: + GH_TOKEN: ${{ github.token }} + REF: ${{ steps.spec.outputs.ref }} + run: | + mkdir -p generated + # Releases publish openapi.yaml as an asset; fall back to + # generating it from a spec checkout for refs that predate the + # asset (or when building against a branch like main). + if [ "$REF" != "main" ] && gh release download "$REF" \ + --repo gemaraproj/gemara \ + --pattern openapi.yaml \ + --output generated/openapi.yaml 2>/dev/null; then + echo "Downloaded openapi.yaml from release $REF" + else + echo "::warning::No openapi.yaml release asset for $REF; generating from a spec checkout" + git clone --depth 1 --branch "$REF" https://github.com/gemaraproj/gemara .gemara-spec + (cd .gemara-spec/cmd && go run . cue2openapi \ + --schema .. \ + --output "$GITHUB_WORKSPACE/generated/openapi.yaml") + fi + - name: Generate documentation - run: make gendocs GEMARA_DIR=.gemara-spec + run: make gendocs - name: Build with Jekyll uses: actions/jekyll-build-pages@44a6e6beabd48582f863aeeb6cb2151cc1716697 # v1.0.13 diff --git a/Makefile b/Makefile index 6521ec9..dff107c 100644 --- a/Makefile +++ b/Makefile @@ -2,27 +2,28 @@ # # The site content lives at the repository root. Schema reference pages, # the definitions table, and term cross-links are GENERATED from the Gemara -# specification repo (github.com/gemaraproj/gemara), which provides both the -# CUE schemas and the `gemara-docs` CLI under cmd/. +# specification's OpenAPI projection (openapi.yaml), which the spec repo +# (github.com/gemaraproj/gemara) publishes as a release asset. The markdown +# tooling that renders it lives in this repo under tools/. # -# GEMARA_DIR points at a checkout of the spec repo. By default it is a -# shallow clone under .gemara-spec/ at GEMARA_REF; set GEMARA_DIR=../gemara -# to build against a local sibling checkout instead. +# openapi.yaml acquisition, in order of precedence: +# GEMARA_OPENAPI=/path/to/openapi.yaml use a pre-generated file +# GEMARA_DIR=../gemara generate from a local spec checkout +# (runs its cue2openapi command) +# GEMARA_REF=v1.2.3 (default: latest) download the release asset GEMARA_REPO ?= https://github.com/gemaraproj/gemara -GEMARA_REF ?= main -GEMARA_DIR ?= .gemara-spec +GEMARA_REF ?= latest -SPEC_ABS := $(abspath $(GEMARA_DIR)) SITE_ABS := $(abspath .) +TOOLS_DIR := tools GENERATED_DIR := generated OPENAPI_YAML := $(GENERATED_DIR)/openapi.yaml -MANIFEST_JSON := $(GENERATED_DIR)/schema-manifest.json SPEC_MD_DIR := $(GENERATED_DIR)/spec SCHEMA_DIR := schema SCHEMA_NAV := schema-nav.yml -.PHONY: all fetch-spec genopenapi genmd gendocs serve build test-links cleanup cleanup-links check-jekyll deps +.PHONY: all fetch-openapi genmd gendocs serve build test-links cleanup cleanup-links check-jekyll deps all: gendocs test-links cleanup @@ -36,27 +37,40 @@ check-jekyll: exit 1; \ fi -fetch-spec: - @if [ ! -d "$(GEMARA_DIR)" ]; then \ - echo " > Cloning Gemara spec ($(GEMARA_REF)) into $(GEMARA_DIR)..."; \ - git clone --depth 1 --branch "$(GEMARA_REF)" "$(GEMARA_REPO)" "$(GEMARA_DIR)"; \ +# File target: if generated/openapi.yaml already exists (e.g. CI downloaded +# or generated it beforehand), acquisition is skipped entirely. +$(OPENAPI_YAML): + @mkdir -p $(GENERATED_DIR) + @if [ -n "$(GEMARA_OPENAPI)" ]; then \ + echo " > Using local OpenAPI file $(GEMARA_OPENAPI) ..."; \ + cp "$(GEMARA_OPENAPI)" "$(OPENAPI_YAML)"; \ + elif [ -n "$(GEMARA_DIR)" ]; then \ + echo " > Generating OpenAPI from local spec checkout $(GEMARA_DIR) ..."; \ + cd "$(abspath $(GEMARA_DIR))/cmd" && go run . cue2openapi \ + --schema .. \ + --output $(SITE_ABS)/$(OPENAPI_YAML); \ else \ - echo " > Using existing spec checkout at $(GEMARA_DIR)"; \ + if [ "$(GEMARA_REF)" = "latest" ]; then \ + url="$(GEMARA_REPO)/releases/latest/download/openapi.yaml"; \ + else \ + url="$(GEMARA_REPO)/releases/download/$(GEMARA_REF)/openapi.yaml"; \ + fi; \ + echo " > Downloading $$url ..."; \ + curl --fail --silent --show-error --location "$$url" --output "$(OPENAPI_YAML)" || { \ + rm -f "$(OPENAPI_YAML)"; \ + echo "ERROR: could not download openapi.yaml for spec ref '$(GEMARA_REF)'."; \ + echo "Releases before the asset existed can be built from a checkout instead:"; \ + echo " make gendocs GEMARA_DIR=/path/to/gemara"; \ + exit 1; \ + }; \ fi -genopenapi: fetch-spec - @echo " > Converting CUE schema to OpenAPI ..." - @mkdir -p $(GENERATED_DIR) - @cd $(SPEC_ABS)/cmd && go run . cue2openapi \ - --schema $(SPEC_ABS) \ - --output $(SITE_ABS)/$(OPENAPI_YAML) \ - --manifest $(SITE_ABS)/$(MANIFEST_JSON) - @echo " > OpenAPI schema generation complete!" +fetch-openapi: $(OPENAPI_YAML) -genmd: genopenapi +genmd: fetch-openapi @echo " > Generating markdown from OpenAPI ..." @mkdir -p $(SPEC_MD_DIR) - @cd $(SPEC_ABS)/cmd && go run . openapi2md \ + @cd $(TOOLS_DIR) && go run . openapi2md \ --input $(SITE_ABS)/$(OPENAPI_YAML) \ --output $(SITE_ABS)/$(SPEC_MD_DIR) \ --nav $(SITE_ABS)/$(SCHEMA_NAV) @@ -65,7 +79,7 @@ genmd: genopenapi gendocs: genmd @echo " > Copying schema pages to $(SCHEMA_DIR)/ for website ..." @mkdir -p $(SCHEMA_DIR) - @sh "$(SPEC_ABS)/cmd/scripts/parse-nav.sh" "$(SCHEMA_NAV)" list-pages | while IFS='|' read -r filename title; do \ + @sh "$(TOOLS_DIR)/scripts/parse-nav.sh" "$(SCHEMA_NAV)" list-pages | while IFS='|' read -r filename title; do \ if [ -f "$(SPEC_MD_DIR)/$$filename.md" ]; then \ { \ echo "---"; \ @@ -80,7 +94,7 @@ gendocs: genmd @echo " > Updating schema list in $(SCHEMA_DIR)/index.md ..." @if [ -f "$(SCHEMA_DIR)/index.md" ]; then \ schema_list_file="$(SCHEMA_DIR)/index.md.schema_list.tmp"; \ - sh "$(SPEC_ABS)/cmd/scripts/parse-nav.sh" "$(SCHEMA_NAV)" list-pages | while IFS='|' read -r filename title; do \ + sh "$(TOOLS_DIR)/scripts/parse-nav.sh" "$(SCHEMA_NAV)" list-pages | while IFS='|' read -r filename title; do \ [ -f "$(SCHEMA_DIR)/$$filename.md" ] && echo "- [$$title]($$filename.html)"; \ done > "$$schema_list_file"; \ awk -v list_file="$$schema_list_file" ' \ @@ -108,11 +122,11 @@ gendocs: genmd @if [ -f "model/02-definitions.md.template" ]; then \ cp "model/02-definitions.md.template" "model/02-definitions.md"; \ fi - @cd $(SPEC_ABS)/cmd && go run . lexicon2md \ + @cd $(TOOLS_DIR) && go run . lexicon2md \ --lexicon $(SITE_ABS)/lexicon.yaml \ --output $(SITE_ABS)/model/02-definitions.md @echo " > Linking defined terms across documentation ..." - @cd $(SPEC_ABS)/cmd && go run . termlinker \ + @cd $(TOOLS_DIR) && go run . termlinker \ --lexicon $(SITE_ABS)/lexicon.yaml \ --docs $(SITE_ABS) @echo " > Documentation generation complete!" @@ -137,7 +151,7 @@ test-links: cleanup-links: @echo " > Removing termlinker-generated links from documentation ..." - @cd $(SPEC_ABS)/cmd && go run . termlinker \ + @cd $(TOOLS_DIR) && go run . termlinker \ --lexicon $(SITE_ABS)/lexicon.yaml \ --docs $(SITE_ABS) \ --cleanup @@ -145,7 +159,7 @@ cleanup-links: cleanup: cleanup-links @echo " > Removing generated documentation files and links..." - @sh "$(SPEC_ABS)/cmd/scripts/parse-nav.sh" "$(SCHEMA_NAV)" list-pages | while IFS='|' read -r filename title; do \ + @sh "$(TOOLS_DIR)/scripts/parse-nav.sh" "$(SCHEMA_NAV)" list-pages | while IFS='|' read -r filename title; do \ rm -f "$(SCHEMA_DIR)/$$filename.md"; \ done @rm -f model/02-definitions.md diff --git a/README.md b/README.md index 0460007..8bb7e50 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ Keep the front matter. Edit everything below it. You need **Ruby 3.2 or newer** and **Go 1.25 or newer**. -Go is needed because the schema pages are generated by a tool in the spec repo. +Go is needed because the schema pages are generated by the tooling in `tools/`. ```bash make deps # install Ruby dependencies (run once) @@ -113,9 +113,9 @@ That means `make serve` may leave link markup in your working copy. ## How the build works ``` -gemaraproj/gemara ──► CUE schemas + the gemara-docs CLI +gemaraproj/gemara release ──► openapi.yaml (release asset) │ - │ make gendocs (clones the spec into .gemara-spec/) + │ make gendocs (downloads the asset, renders it with tools/) ▼ generated/ ──► schema/*.md and model/02-definitions.md │ @@ -124,14 +124,21 @@ gemaraproj/gemara ──► CUE schemas + the gemara-docs CLI _site/ ──► GitHub Pages ──► gemara.openssf.org ``` -By default the build clones the spec repo into `.gemara-spec/`. +By default the build downloads `openapi.yaml` from the **latest spec release** +(`GEMARA_REF=v1.2.3` pins a specific one). The markdown renderers +(`openapi2md`, `lexicon2md`, `termlinker`) live in this repo under `tools/`. -To build against a local checkout of the spec instead: +To build against a local checkout of the spec instead (e.g. for unreleased +schema changes, or releases that predate the asset): ```bash make serve GEMARA_DIR=../gemara ``` +This runs the spec repo's `cue2openapi` command against that checkout to +produce `generated/openapi.yaml`. A pre-generated file also works: +`make serve GEMARA_OPENAPI=/path/to/openapi.yaml`. + ### Make targets | Command | What it does | @@ -186,8 +193,8 @@ Maintainers are listed in `_data/maintainers.yml`. Run `make deps` first. **Schema pages are empty or missing** -The spec checkout may be stale. Delete it and try again: -`rm -rf .gemara-spec && make gendocs` +The downloaded OpenAPI file may be stale. Delete it and try again: +`rm -rf generated && make gendocs` **Weird link markup all over my diff** That's the term linker. Run `make cleanup`. diff --git a/tools/go.mod b/tools/go.mod new file mode 100644 index 0000000..fb483cf --- /dev/null +++ b/tools/go.mod @@ -0,0 +1,13 @@ +module github.com/gemaraproj/website/tools + +go 1.25.0 + +require ( + github.com/goccy/go-yaml v1.19.2 + github.com/spf13/cobra v1.10.1 +) + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.9 // indirect +) diff --git a/tools/go.sum b/tools/go.sum new file mode 100644 index 0000000..6a6d0ef --- /dev/null +++ b/tools/go.sum @@ -0,0 +1,12 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/internal/cmd/lexicon2md.go b/tools/internal/cmd/lexicon2md.go new file mode 100644 index 0000000..dffc616 --- /dev/null +++ b/tools/internal/cmd/lexicon2md.go @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "bytes" + "fmt" + "os" + "strings" + "text/template" + + "github.com/goccy/go-yaml" + "github.com/spf13/cobra" +) + +type TemplateData struct { + Table string +} + +var lexicon2MDCmd = &cobra.Command{ + Use: "lexicon2md", + Short: "Generate definitions table from lexicon YAML", + Long: `Generate a definitions table in Markdown format from a lexicon YAML file. +The lexicon file should contain an array of terms with their definitions and references. +The output will be written to a Markdown file using a template.`, + RunE: runLexicon2MD, +} + +var lexicon2MDFlags struct { + lexiconFile string + outputFile string +} + +func newLexicon2MDCmd() *cobra.Command { + lexicon2MDCmd.Flags().StringVarP(&lexicon2MDFlags.lexiconFile, "lexicon", "l", "lexicon.yaml", "Input lexicon YAML file") + lexicon2MDCmd.Flags().StringVarP(&lexicon2MDFlags.outputFile, "output", "o", "model/02-definitions.md", "Output markdown file") + return lexicon2MDCmd +} + +func runLexicon2MD(cmd *cobra.Command, args []string) error { + data, err := os.ReadFile(lexicon2MDFlags.lexiconFile) + if err != nil { + return fmt.Errorf("Error reading lexicon file: %v", err) + } + + var lexicon Lexicon + if err := yaml.Unmarshal(data, &lexicon); err != nil { + return fmt.Errorf("Error parsing lexicon YAML: %v", err) + } + + var tableRows strings.Builder + for _, term := range lexicon.Terms { + slug := termToSlug(term.Title) + + termName := fmt.Sprintf("**%s**", slug, term.Title) + + refs := make([]string, 0, len(term.References)) + for _, r := range term.References { + refs = append(refs, r.Citation) + } + appliesTo := strings.Join(refs, "
") + + definition := strings.TrimSpace(strings.ReplaceAll(term.Definition, "|", "\\|")) + + tableRows.WriteString(fmt.Sprintf("| %s | %s | %s |\n", termName, definition, appliesTo)) + } + + templateContent, err := os.ReadFile(lexicon2MDFlags.outputFile) + if err != nil { + return fmt.Errorf("Error reading output file: %v", err) + } + + tmpl, err := template.New("definitions").Parse(string(templateContent)) + if err != nil { + return fmt.Errorf("Error parsing template: %v", err) + } + + var output bytes.Buffer + templateData := TemplateData{ + Table: strings.TrimSpace(tableRows.String()), + } + if err := tmpl.Execute(&output, templateData); err != nil { + return fmt.Errorf("Error executing template: %v", err) + } + + if err := os.WriteFile(lexicon2MDFlags.outputFile, output.Bytes(), 0644); err != nil { + return fmt.Errorf("Error writing output file: %v", err) + } + + fmt.Printf("Successfully generated definitions table in %s\n", lexicon2MDFlags.outputFile) + return nil +} diff --git a/tools/internal/cmd/openapi2md.go b/tools/internal/cmd/openapi2md.go new file mode 100644 index 0000000..09ac5c5 --- /dev/null +++ b/tools/internal/cmd/openapi2md.go @@ -0,0 +1,618 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "unicode" + + "github.com/goccy/go-yaml" + "github.com/spf13/cobra" +) + +type Schema struct { + Type string `yaml:"type"` + Description string `yaml:"description"` + Properties map[string]interface{} `yaml:"properties"` + Required []string `yaml:"required"` + Pattern string `yaml:"pattern"` + Format string `yaml:"format"` + Items interface{} `yaml:"items"` + Ref string `yaml:"$ref"` + XStatus string `yaml:"x-status"` +} + +type NavPage struct { + Title string `yaml:"title"` + Filename string `yaml:"filename"` + Schemas []string `yaml:"schemas"` +} + +type NavConfig struct { + Pages []NavPage `yaml:"pages"` +} + +var openAPI2MDCmd = &cobra.Command{ + Use: "openapi2md", + Short: "Convert OpenAPI YAML to Markdown documentation", + Long: `Convert OpenAPI 3.0.3 YAML specifications to Markdown documentation. +Supports three modes: + - Navigation-based: Uses a nav.yml file to organize schemas into pages + - Manifest-based: Uses a manifest.json to map CUE files to schemas + - Roots-based: Uses a comma-separated list of root schema names`, + RunE: runOpenAPI2MD, +} + +var openAPI2MDFlags struct { + inputFile string + outputDir string + manifestPath string + navPath string + rootsFlag string +} + +func newOpenAPI2MDCmd() *cobra.Command { + openAPI2MDCmd.Flags().StringVarP(&openAPI2MDFlags.inputFile, "input", "i", "openapi.yaml", "Input OpenAPI YAML file") + openAPI2MDCmd.Flags().StringVarP(&openAPI2MDFlags.outputDir, "output", "o", "spec", "Output directory for markdown files") + openAPI2MDCmd.Flags().StringVarP(&openAPI2MDFlags.manifestPath, "manifest", "m", "", "Path to schema-manifest.json for per-file mode") + openAPI2MDCmd.Flags().StringVarP(&openAPI2MDFlags.navPath, "nav", "n", "", "Path to schema-nav.yml for nav-based mode") + openAPI2MDCmd.Flags().StringVarP(&openAPI2MDFlags.rootsFlag, "roots", "r", "", "Comma-separated list of root schema names (used when -manifest and -nav are not set)") + return openAPI2MDCmd +} + +func runOpenAPI2MD(cmd *cobra.Command, args []string) error { + if openAPI2MDFlags.navPath != "" { + if err := convertFromNav(openAPI2MDFlags.inputFile, openAPI2MDFlags.outputDir, openAPI2MDFlags.navPath); err != nil { + return err + } + } else if openAPI2MDFlags.manifestPath != "" { + if err := convertPerFile(openAPI2MDFlags.inputFile, openAPI2MDFlags.outputDir, openAPI2MDFlags.manifestPath); err != nil { + return err + } + } else { + roots := splitRoots(openAPI2MDFlags.rootsFlag) + if len(roots) == 0 { + return fmt.Errorf("Error: -roots is required when -manifest and -nav are not set") + } + if err := convertOpenAPIToMarkdown(openAPI2MDFlags.inputFile, openAPI2MDFlags.outputDir, roots); err != nil { + return err + } + } + + fmt.Printf("Markdown documentation generated successfully in %s/\n", openAPI2MDFlags.outputDir) + return nil +} + +func splitRoots(s string) []string { + if s == "" { + return nil + } + var out []string + for _, part := range strings.Split(s, ",") { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + return out +} + +func loadManifest(path string) (map[string][]string, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read manifest: %w", err) + } + var m map[string][]string + if err := json.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("parse manifest: %w", err) + } + return m, nil +} + +func loadNavFile(path string) (*NavConfig, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read nav file: %w", err) + } + var nav NavConfig + if err := yaml.Unmarshal(data, &nav); err != nil { + return nil, fmt.Errorf("parse nav file: %w", err) + } + return &nav, nil +} + +func slugify(s string) string { + var result strings.Builder + for _, r := range s { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + result.WriteRune(unicode.ToLower(r)) + } else if r == ' ' || r == '-' { + result.WriteRune('-') + } + } + return result.String() +} + +func convertFromNav(inputFile, outputDir, navPath string) error { + // Load OpenAPI spec + data, err := os.ReadFile(inputFile) + if err != nil { + return fmt.Errorf("failed to read OpenAPI file: %w", err) + } + var spec OpenAPISpec + if err := yaml.Unmarshal(data, &spec); err != nil { + return fmt.Errorf("failed to parse OpenAPI YAML: %w", err) + } + + // Load nav file + nav, err := loadNavFile(navPath) + if err != nil { + return err + } + + if err := os.MkdirAll(outputDir, 0755); err != nil { + return fmt.Errorf("failed to create output directory: %w", err) + } + + // Build schema-to-filename map for generating links + schemaToFile := make(map[string]string) + for _, page := range nav.Pages { + filename := page.Filename + if filename == "" { + filename = slugify(page.Title) + } + for _, schemaName := range page.Schemas { + schemaToFile[schemaName] = filename + } + } + + // For each page in nav + for _, page := range nav.Pages { + var buf strings.Builder + + // For each schema name listed in the page's schemas array + for _, schemaName := range page.Schemas { + // Look up schema in spec.Components.Schemas + schemaData, ok := spec.Components.Schemas[schemaName] + if !ok { + return fmt.Errorf("schema %q not found in OpenAPI spec (referenced in page %q)", schemaName, page.Title) + } + + // Parse schema data into Schema struct + schemaBytes, _ := yaml.Marshal(schemaData) + var schema Schema + if err := yaml.Unmarshal(schemaBytes, &schema); err != nil { + return fmt.Errorf("failed to parse schema %q: %w", schemaName, err) + } + + // Use isAlias() to determine schema type + if isAlias(schema) { + buf.WriteString(generateAliasBlock(schemaName, schema, false)) + } else { + buf.WriteString(generateRootSection(schemaName, schema, spec, schemaToFile)) + } + } + + // Determine output filename + filename := page.Filename + if filename == "" { + filename = slugify(page.Title) + } + + // Write page buffer to {filename}.md + outPath := filepath.Join(outputDir, filename+".md") + if err := os.WriteFile(outPath, []byte(buf.String()), 0644); err != nil { + return fmt.Errorf("write %s: %w", outPath, err) + } + } + + return nil +} + +func convertPerFile(inputFile, outputDir, manifestPath string) error { + manifest, err := loadManifest(manifestPath) + if err != nil { + return err + } + + data, err := os.ReadFile(inputFile) + if err != nil { + return fmt.Errorf("failed to read OpenAPI file: %w", err) + } + var spec OpenAPISpec + if err := yaml.Unmarshal(data, &spec); err != nil { + return fmt.Errorf("failed to parse OpenAPI YAML: %w", err) + } + + if err := os.MkdirAll(outputDir, 0755); err != nil { + return fmt.Errorf("failed to create output directory: %w", err) + } + + fileOrder := make([]string, 0, len(manifest)) + for k := range manifest { + fileOrder = append(fileOrder, k) + } + sort.Strings(fileOrder) + + // Empty map since we don't have nav file info in this mode + schemaToFile := make(map[string]string) + + for _, cueFile := range fileOrder { + schemaNames := manifest[cueFile] + if len(schemaNames) == 0 { + continue + } + base := strings.TrimSuffix(cueFile, ".cue") + + var buf strings.Builder + + for _, name := range schemaNames { + schemaData, ok := spec.Components.Schemas[name] + if !ok { + continue + } + schemaBytes, _ := yaml.Marshal(schemaData) + var schema Schema + if err := yaml.Unmarshal(schemaBytes, &schema); err != nil { + continue + } + if isAlias(schema) { + buf.WriteString(generateAliasBlock(name, schema, false)) + } else { + buf.WriteString(generateRootSection(name, schema, spec, schemaToFile)) + } + } + + outPath := filepath.Join(outputDir, base+".md") + if err := os.WriteFile(outPath, []byte(buf.String()), 0644); err != nil { + return fmt.Errorf("write %s: %w", outPath, err) + } + } + + return nil +} + +func generateAliasBlock(name string, schema Schema, subheading bool) string { + var buf strings.Builder + level := "##" + if subheading { + level = "###" + } + buf.WriteString(fmt.Sprintf("%s `%s`\n\n", level, name)) + if schema.Description != "" { + buf.WriteString(schema.Description + "\n\n") + } + buf.WriteString(fmt.Sprintf("- **Type**: `%s`\n", schema.Type)) + if schema.Format != "" { + buf.WriteString(fmt.Sprintf("- **Format**: `%s`\n", schema.Format)) + } + if schema.Pattern != "" { + buf.WriteString(fmt.Sprintf("- **Value**: `%s`\n", schema.Pattern)) + } + buf.WriteString("\n---\n\n") + return buf.String() +} + +func convertOpenAPIToMarkdown(inputFile, outputDir string, roots []string) error { + data, err := os.ReadFile(inputFile) + if err != nil { + return fmt.Errorf("failed to read OpenAPI file: %w", err) + } + + var spec OpenAPISpec + if err := yaml.Unmarshal(data, &spec); err != nil { + return fmt.Errorf("failed to parse OpenAPI YAML: %w", err) + } + + if err := os.MkdirAll(outputDir, 0755); err != nil { + return fmt.Errorf("failed to create output directory: %w", err) + } + + rootSet := make(map[string]bool) + for _, r := range roots { + rootSet[r] = true + } + + // Resolve root schemas and fail if any are missing + rootSchemas := make(map[string]Schema) + for _, name := range roots { + data, exists := spec.Components.Schemas[name] + if !exists { + return fmt.Errorf("root schema %q not found in OpenAPI spec", name) + } + var s Schema + bytes, _ := yaml.Marshal(data) + if err := yaml.Unmarshal(bytes, &s); err != nil { + return fmt.Errorf("failed to parse root schema %q: %w", name, err) + } + rootSchemas[name] = s + } + + // Collect aliases (exclude all roots) + var aliasTypes []string + for schemaName, schemaData := range spec.Components.Schemas { + if rootSet[schemaName] { + continue + } + schemaBytes, _ := yaml.Marshal(schemaData) + var schema Schema + if err := yaml.Unmarshal(schemaBytes, &schema); err != nil { + continue + } + if isAlias(schema) { + aliasTypes = append(aliasTypes, schemaName) + } + } + sort.Strings(aliasTypes) + + title := spec.Info.Title + if title == "" { + title = "Schema" + } + version := spec.Info.Version + if version == "" { + version = "unknown" + } + + var buf strings.Builder + // Empty map since we don't have nav file info in this mode + schemaToFile := make(map[string]string) + + // H1 and optional intro + buf.WriteString(fmt.Sprintf("# %s _(%s)_\n\n", title, version)) + if spec.Info.Description != "" { + buf.WriteString(spec.Info.Description + "\n\n") + } + + // Table of Contents + buf.WriteString("**Table of Contents**\n\n") + buf.WriteString("* \n") + buf.WriteString("{:toc}\n\n") + buf.WriteString("---\n\n") + + // One major section per root + for _, name := range roots { + schema := rootSchemas[name] + buf.WriteString(generateRootSection(name, schema, spec, schemaToFile)) + } + + // Aliases section + if len(aliasTypes) > 0 { + buf.WriteString("\n## Aliases\n\n") + buf.WriteString("The following aliases are used throughout the schema for consistency.\n\n") + + for _, name := range aliasTypes { + schemaBytes, _ := yaml.Marshal(spec.Components.Schemas[name]) + var schema Schema + if err := yaml.Unmarshal(schemaBytes, &schema); err != nil { + continue + } + buf.WriteString(generateAliasBlock(name, schema, true)) + } + } + + outputPath := filepath.Join(outputDir, "schema.md") + if err := os.WriteFile(outputPath, []byte(buf.String()), 0644); err != nil { + return fmt.Errorf("failed to write %s: %w", outputPath, err) + } + + return nil +} + +func isAlias(schema Schema) bool { + // Aliases are anything that is NOT an object with properties + // This includes: string types (with or without patterns), boolean, and simple object types + return schema.Properties == nil +} + +func resolveSchemaRef(ref string, spec OpenAPISpec) (*Schema, error) { + if !strings.HasPrefix(ref, "#/components/schemas/") { + return nil, fmt.Errorf("invalid ref format: %s", ref) + } + + schemaName := strings.TrimPrefix(ref, "#/components/schemas/") + schemaData, exists := spec.Components.Schemas[schemaName] + if !exists { + return nil, fmt.Errorf("schema not found: %s", schemaName) + } + + schemaBytes, _ := yaml.Marshal(schemaData) + var schema Schema + if err := yaml.Unmarshal(schemaBytes, &schema); err != nil { + return nil, fmt.Errorf("failed to parse schema %s: %v", schemaName, err) + } + + return &schema, nil +} + +// getSchemaStatus extracts the gemara-status extension value from a schema. +func getSchemaStatus(schema Schema) string { + if schema.XStatus != "" { + return schema.XStatus + } + return "" +} + +// formatStatusBadge returns a markdown badge for the status. +func formatStatusBadge(status string) string { + switch status { + case "experimental": + return "Experimental" + case "stable": + return "Stable" + case "deprecated": + return "Deprecated" + default: + return "" + } +} + +// formatFieldInline formats a field's information and returns (fieldLine, description) +// fieldLine format: `field` **type** _Required_ or `field` **type** +// description is returned separately +func formatFieldInline(fieldName string, fieldSchema Schema, spec OpenAPISpec, prefix string, isRequired bool, schemaToFile map[string]string) (string, string) { + // Field name with full path + fieldPath := fieldName + if prefix != "" { + fieldPath = prefix + "." + fieldName + } + + // Type + typeStr := formatFieldType(fieldSchema, spec, schemaToFile) + + // Build field line: `field` **type** _Required_ or `field` **type** + var fieldLineParts []string + fieldLineParts = append(fieldLineParts, fmt.Sprintf("`%s`", fieldPath)) + if typeStr != "" { + fieldLineParts = append(fieldLineParts, fmt.Sprintf("**%s**", typeStr)) + } + if isRequired { + fieldLineParts = append(fieldLineParts, "_Required_") + } + fieldLine := strings.Join(fieldLineParts, " ") + + // Description + description := fieldSchema.Description + if fieldSchema.Ref != "" { + refSchema, err := resolveSchemaRef(fieldSchema.Ref, spec) + if err == nil { + if description == "" { + description = refSchema.Description + } + } + } + + return fieldLine, description +} + +// formatFieldType returns the type string for a field with markdown links for custom types +func formatFieldType(fieldSchema Schema, spec OpenAPISpec, schemaToFile map[string]string) string { + if fieldSchema.Ref != "" { + refType := strings.TrimPrefix(fieldSchema.Ref, "#/components/schemas/") + // Check if this is a custom type that should be linked + if filename, exists := schemaToFile[refType]; exists { + // Create markdown link: [TypeName](filename#typename) - no .md extension for Jekyll + anchor := strings.ToLower(refType) + return fmt.Sprintf("[%s](%s#%s)", refType, filename, anchor) + } + // Return just the type name if not found in schema map + return refType + } + + if fieldSchema.Type != "" { + typeStr := fieldSchema.Type + + // Handle array items - format as array[Type] + if fieldSchema.Type == "array" && fieldSchema.Items != nil { + itemsBytes, _ := yaml.Marshal(fieldSchema.Items) + var itemsSchema Schema + if err := yaml.Unmarshal(itemsBytes, &itemsSchema); err == nil { + var itemType string + var itemTypeLink string + if itemsSchema.Ref != "" { + refType := strings.TrimPrefix(itemsSchema.Ref, "#/components/schemas/") + // Check if this is a custom type that should be linked + if filename, exists := schemaToFile[refType]; exists { + anchor := strings.ToLower(refType) + itemTypeLink = fmt.Sprintf("[%s](%s#%s)", refType, filename, anchor) + } else { + itemTypeLink = refType + } + itemType = itemTypeLink + } else if itemsSchema.Type != "" { + itemType = itemsSchema.Type + } + if itemType != "" { + typeStr = fmt.Sprintf("array[%s]", itemType) + } + } + } + + return typeStr + } + + return "" +} + +// formatFieldWithNested formats a field inline (nested expansion disabled). +func formatFieldWithNested(fieldName string, fieldSchema Schema, spec OpenAPISpec, isRequired bool, schemaToFile map[string]string) string { + var buf strings.Builder + fieldLine, description := formatFieldInline(fieldName, fieldSchema, spec, "", isRequired, schemaToFile) + buf.WriteString(fieldLine + "\n\n") + if description != "" { + buf.WriteString(description + "\n") + } + return buf.String() +} + +func generateRootSection(rootName string, schema Schema, spec OpenAPISpec, schemaToFile map[string]string) string { + var buf strings.Builder + + buf.WriteString(fmt.Sprintf("## `%s`\n\n", rootName)) + if status := getSchemaStatus(schema); status != "" { + buf.WriteString(formatStatusBadge(status) + "\n\n") + } + if schema.Description != "" { + buf.WriteString(schema.Description + "\n\n") + } + + if schema.Properties != nil { + propNames := make([]string, 0, len(schema.Properties)) + for propName := range schema.Properties { + propNames = append(propNames, propName) + } + sort.Strings(propNames) + + // Output all fields in order (required first, then optional) + // Sort by required status, then by name + type fieldInfo struct { + name string + schema Schema + required bool + } + var fields []fieldInfo + + for _, propName := range propNames { + isRequired := false + for _, req := range schema.Required { + if req == propName { + isRequired = true + break + } + } + + propData := schema.Properties[propName] + propBytes, _ := yaml.Marshal(propData) + var prop Schema + if err := yaml.Unmarshal(propBytes, &prop); err != nil { + continue + } + fields = append(fields, fieldInfo{ + name: propName, + schema: prop, + required: isRequired, + }) + } + + // Sort: required first, then by name + sort.Slice(fields, func(i, j int) bool { + if fields[i].required != fields[j].required { + return fields[i].required // required fields come first + } + return fields[i].name < fields[j].name + }) + + // Output all fields + for _, field := range fields { + buf.WriteString(formatFieldWithNested(field.name, field.schema, spec, field.required, schemaToFile)) + buf.WriteString("\n") + } + } + + return buf.String() +} diff --git a/tools/internal/cmd/openapi_types.go b/tools/internal/cmd/openapi_types.go new file mode 100644 index 0000000..945b206 --- /dev/null +++ b/tools/internal/cmd/openapi_types.go @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +// These types mirror the OpenAPI document emitted by the spec repo's +// `gemara-docs cue2openapi` command (github.com/gemaraproj/gemara, cmd/). + +type OpenAPISpec struct { + OpenAPI string `yaml:"openapi" json:"openapi"` + Info OpenAPIInfo `yaml:"info" json:"info"` + Components OpenAPIComponents `yaml:"components" json:"components"` +} + +type OpenAPIInfo struct { + Title string `yaml:"title" json:"title"` + Version string `yaml:"version" json:"version"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` +} + +type OpenAPIComponents struct { + Schemas map[string]interface{} `yaml:"schemas" json:"schemas"` +} diff --git a/tools/internal/cmd/root.go b/tools/internal/cmd/root.go new file mode 100644 index 0000000..8f32a7a --- /dev/null +++ b/tools/internal/cmd/root.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +var rootCmd = &cobra.Command{ + Use: "website-docs", + Short: "Doc-generation tooling for the Gemara website", +} + +// Execute adds all child commands to the root command and sets flags appropriately. +func Execute() { + if err := rootCmd.Execute(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func init() { + rootCmd.AddCommand(newOpenAPI2MDCmd()) + rootCmd.AddCommand(newLexicon2MDCmd()) + rootCmd.AddCommand(newTermLinkerCmd()) +} diff --git a/tools/internal/cmd/termlinker.go b/tools/internal/cmd/termlinker.go new file mode 100644 index 0000000..bc60034 --- /dev/null +++ b/tools/internal/cmd/termlinker.go @@ -0,0 +1,881 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "unicode" + + "github.com/goccy/go-yaml" + "github.com/spf13/cobra" +) + +type Lexicon struct { + Terms []Term `yaml:"terms"` +} + +type LexiconReference struct { + Citation string `yaml:"citation"` +} + +type Term struct { + ID string `yaml:"id"` + Title string `yaml:"title"` + Definition string `yaml:"definition"` + References []LexiconReference `yaml:"references"` +} + +type TermInfo struct { + OriginalTerm string + LowerTerm string + Slug string + Regex *regexp.Regexp +} + +var termLinkerCmd = &cobra.Command{ + Use: "termlinker", + Short: "Link defined terms across documentation", + Long: `Link defined terms from the lexicon across all markdown files in the documentation. +This command finds occurrences of terms defined in the lexicon and creates markdown +links to the definitions page. Use the --cleanup flag to remove previously generated links.`, + RunE: runTermLinker, +} + +var termLinkerFlags struct { + lexiconFile string + docsDir string + cleanup bool +} + +func newTermLinkerCmd() *cobra.Command { + termLinkerCmd.Flags().StringVarP(&termLinkerFlags.lexiconFile, "lexicon", "l", "lexicon.yaml", "Input lexicon YAML file") + termLinkerCmd.Flags().StringVarP(&termLinkerFlags.docsDir, "docs", "d", ".", "Documentation directory to process") + termLinkerCmd.Flags().BoolVarP(&termLinkerFlags.cleanup, "cleanup", "c", false, "Remove termlinker-generated links instead of adding them") + return termLinkerCmd +} + +func runTermLinker(cmd *cobra.Command, args []string) error { + // Load terms from lexicon + terms, err := loadTerms(termLinkerFlags.lexiconFile) + if err != nil { + return fmt.Errorf("Error loading terms: %v", err) + } + + // Build term info with regex patterns (sorted by length, longest first) + termInfos := buildTermInfos(terms) + + // Find all markdown files + mdFiles, err := findMarkdownFiles(termLinkerFlags.docsDir) + if err != nil { + return fmt.Errorf("Error finding markdown files: %v", err) + } + + // Process each file + processedCount := 0 + for _, file := range mdFiles { + // Skip the definitions page itself + if strings.HasSuffix(file, "model/02-definitions.md") { + continue + } + + if termLinkerFlags.cleanup { + if err := cleanupFile(file, termInfos, termLinkerFlags.docsDir); err != nil { + fmt.Fprintf(os.Stderr, "Error cleaning up %s: %v\n", file, err) + continue + } + } else { + if err := processFile(file, termInfos, termLinkerFlags.docsDir); err != nil { + fmt.Fprintf(os.Stderr, "Error processing %s: %v\n", file, err) + continue + } + } + processedCount++ + } + + if termLinkerFlags.cleanup { + fmt.Printf("Successfully cleaned up %d markdown files\n", processedCount) + } else { + fmt.Printf("Successfully processed %d markdown files\n", processedCount) + } + return nil +} + +func loadTerms(lexiconFile string) ([]Term, error) { + data, err := os.ReadFile(lexiconFile) + if err != nil { + return nil, fmt.Errorf("read lexicon file: %w", err) + } + + var lexicon Lexicon + if err := yaml.Unmarshal(data, &lexicon); err != nil { + return nil, fmt.Errorf("parse lexicon YAML: %w", err) + } + + return lexicon.Terms, nil +} + +func buildTermInfos(terms []Term) []TermInfo { + termInfos := make([]TermInfo, 0, len(terms)) + + for _, term := range terms { + lowerTerm := strings.ToLower(term.Title) + slug := termToSlug(term.Title) + + // Create regex for whole-word, case-insensitive matching + // Escape special regex characters in the term + escapedTerm := regexp.QuoteMeta(term.Title) + // Use word boundaries for whole-word matching + pattern := `(?i)\b` + escapedTerm + `\b` + regex, err := regexp.Compile(pattern) + if err != nil { + // Skip terms that can't be compiled (shouldn't happen) + continue + } + + termInfos = append(termInfos, TermInfo{ + OriginalTerm: term.Title, + LowerTerm: lowerTerm, + Slug: slug, + Regex: regex, + }) + } + + // Sort by length (longest first) to avoid partial matches + sort.Slice(termInfos, func(i, j int) bool { + return len(termInfos[i].OriginalTerm) > len(termInfos[j].OriginalTerm) + }) + + return termInfos +} + +func termToSlug(term string) string { + // Convert to lowercase and replace spaces with hyphens + slug := strings.ToLower(term) + slug = strings.ReplaceAll(slug, " ", "-") + // Remove any other non-alphanumeric characters except hyphens + var result strings.Builder + for _, r := range slug { + if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '-' { + result.WriteRune(r) + } + } + return result.String() +} + +func findMarkdownFiles(docsDir string) ([]string, error) { + var files []string + err := filepath.Walk(docsDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(path, ".md") { + files = append(files, path) + } + return nil + }) + return files, err +} + +func processFile(filePath string, termInfos []TermInfo, docsDir string) error { + content, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("read file: %w", err) + } + + // Calculate relative path to definitions page + relPath := calculateRelativePath(filePath, docsDir) + + // Process the content + processed := processContent(string(content), termInfos, relPath) + + // Write back + if err := os.WriteFile(filePath, []byte(processed), 0644); err != nil { + return fmt.Errorf("write file: %w", err) + } + + return nil +} + +func cleanupFile(filePath string, termInfos []TermInfo, docsDir string) error { + content, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("read file: %w", err) + } + + // Calculate relative path to definitions page (for matching) + relPath := calculateRelativePath(filePath, docsDir) + + // Process the content to remove links + processed := cleanupContent(string(content), termInfos, relPath) + + // Write back + if err := os.WriteFile(filePath, []byte(processed), 0644); err != nil { + return fmt.Errorf("write file: %w", err) + } + + return nil +} + +func calculateRelativePath(filePath, docsDir string) string { + // Get the directory of the current file + fileDir := filepath.Dir(filePath) + + // Calculate relative path from file directory to definitions page + defPagePath := filepath.Join(docsDir, "model/02-definitions.html") + relPath, err := filepath.Rel(fileDir, defPagePath) + if err != nil { + // Fallback to absolute path + return "/model/02-definitions.html" + } + + // Normalize path separators for URLs + return filepath.ToSlash(relPath) +} + +func processContent(content string, termInfos []TermInfo, defPath string) string { + lines := strings.Split(content, "\n") + var result strings.Builder + + state := &contentState{} + + for i, line := range lines { + originalLine := line + state.update(line) + skip := state.skipLine(line) + + if skip { + result.WriteString(originalLine) + if i < len(lines)-1 { + result.WriteString("\n") + } + continue + } + + // Process the line + processedLine := processLine(line, termInfos, defPath) + result.WriteString(processedLine) + if i < len(lines)-1 { + result.WriteString("\n") + } + } + + return result.String() +} + +type contentState struct { + inCodeBlock bool + inFrontMatter bool + inHTMLBlock bool + htmlTagStack int + inJekyllInclude bool +} + +func (s *contentState) update(line string) { + trimmed := strings.TrimSpace(line) + + // Track front matter - (fenced with ---) + if strings.HasPrefix(trimmed, "---") { + s.inFrontMatter = !s.inFrontMatter + } + + // Track code blocks (fenced with ```) + if strings.HasPrefix(trimmed, "```") { + s.inCodeBlock = !s.inCodeBlock + } + + // Track Jekyll includes - starts with {% include and ends with %} + if strings.Contains(trimmed, "{%") && strings.Contains(trimmed, "include") { + s.inJekyllInclude = true + } + if s.inJekyllInclude && strings.Contains(trimmed, "%}") { + s.inJekyllInclude = false + } + + // Track HTML blocks - check for opening and closing HTML tags + // This handles multi-line HTML blocks like
...
+ htmlTagPattern := regexp.MustCompile(`<[^>]+>`) + htmlTags := htmlTagPattern.FindAllString(line, -1) + for _, tag := range htmlTags { + tagLower := strings.ToLower(strings.TrimSpace(tag)) + // Check for opening tags (not self-closing and not closing tags) + if !strings.HasPrefix(tagLower, "") { + s.htmlTagStack++ + s.inHTMLBlock = true + } + // Check for closing tags + if strings.HasPrefix(tagLower, " 0 { + return true + } + + // Skip headers (lines starting with #) + if strings.HasPrefix(trimmed, "#") { + return true + } + + return false +} + +func processLine(line string, termInfos []TermInfo, defPath string) string { + // Find all existing markdown links, inline code, and HTML tags to skip + linkPattern := regexp.MustCompile(`\[([^\]]+)\]\([^\)]+\)`) + inlineCodePattern := regexp.MustCompile("`[^`]+`") + // Match HTML tags: , , , , etc. + // This regex matches opening tags, closing tags, and self-closing tags + htmlTagPattern := regexp.MustCompile(`<[^>]+>`) + + // First, clean up any existing nested links (e.g., [Term [Subterm](...)](...)) + // Replace them with just the outer link, using the proper slug for the compound term + // This pattern matches nested links including malformed hash fragments + // Pattern 1: [[Word](url#word) RestOfTerm](url#[word](url#word)-restofterm) + splitTermPattern := regexp.MustCompile(`\[\[([^\]]+)\]\(([^\)]+)\)\s+([^\]]+)\]\(([^\)]+)\)`) + result := splitTermPattern.ReplaceAllStringFunc(line, func(match string) string { + submatches := splitTermPattern.FindStringSubmatch(match) + if len(submatches) >= 5 { + firstWord := strings.TrimSpace(submatches[1]) + _ = submatches[2] // First word URL (unused) + restOfTerm := strings.TrimSpace(submatches[3]) + _ = submatches[4] // Outer URL with malformed hash (unused) + + // Combine to form the full compound term + fullTerm := firstWord + " " + restOfTerm + + // Check if this matches a known compound term + for _, termInfo := range termInfos { + if strings.EqualFold(fullTerm, termInfo.OriginalTerm) { + // This is a known compound term - use its proper slug + return fmt.Sprintf("[%s](%s#%s)", fullTerm, defPath, termInfo.Slug) + } + } + // If not a known term, create a clean slug + slug := termToSlug(fullTerm) + return fmt.Sprintf("[%s](%s#%s)", fullTerm, defPath, slug) + } + return match + }) + + // Pattern 2: [Term [Subterm](...)](...) - general nested link pattern + // This pattern needs to handle nested parentheses in the hash fragment + nestedLinkPattern := regexp.MustCompile(`\[([^\[]*)\[([^\]]+)\]\(([^\)]+)\)\]\(([^\)]+)\)`) + result = nestedLinkPattern.ReplaceAllStringFunc(result, func(match string) string { + // Extract the outer term and inner term from the nested link + submatches := nestedLinkPattern.FindStringSubmatch(match) + if len(submatches) >= 5 { + outerTerm := strings.TrimSpace(submatches[1] + submatches[2]) // Combine prefix and inner term + _ = submatches[3] // The inner link URL (unused) + _ = submatches[4] // The outer link URL (may contain malformed hash) + + // Check if this matches a known compound term + for _, termInfo := range termInfos { + if strings.EqualFold(outerTerm, termInfo.OriginalTerm) { + // This is a known compound term - use its proper slug + return fmt.Sprintf("[%s](%s#%s)", outerTerm, defPath, termInfo.Slug) + } + } + // If not a known term, try to extract a clean slug from the outer link + // Remove the nested link structure and create a clean link + slug := termToSlug(outerTerm) + return fmt.Sprintf("[%s](%s#%s)", outerTerm, defPath, slug) + } + return match // Return unchanged if we can't parse it + }) + + // Pattern 3: Clean up extremely long hash fragments that contain way too much text + // This handles cases where the hash includes entire definitions or sentences + // Pattern: ](url#extremely-long-hash-that-contains-too-much-text) or ](url#extremely-long-hash-that-contains-too-much-text)) + longHashPattern := regexp.MustCompile(`(\]\([^\)]+#)([a-z0-9-]{50,})(\))\)?`) + result = longHashPattern.ReplaceAllStringFunc(result, func(match string) string { + submatches := longHashPattern.FindStringSubmatch(match) + if len(submatches) >= 4 { + linkStart := submatches[1] // ](url# + longHash := submatches[2] // The extremely long hash + _ = submatches[3] // ) or )) (unused - we always return single ) + + // Try to extract a reasonable slug from the long hash + // Look for common compound term patterns at the end + for _, termInfo := range termInfos { + termSlug := termInfo.Slug + // Check if the hash ends with this term's slug + if strings.HasSuffix(longHash, termSlug) { + // Extract the part before the term slug to see if it's a prefix + prefix := strings.TrimSuffix(longHash, termSlug) + // If the prefix ends with the first word of the term, use the full term slug + firstWord := strings.ToLower(strings.Fields(termInfo.OriginalTerm)[0]) + if strings.HasSuffix(prefix, firstWord) || prefix == "" { + // Return with just one closing parenthesis (remove any extra) + return linkStart + termSlug + ")" + } + } + } + // If we can't find a match, try to extract just the last reasonable part + // Split by common delimiters and take the last meaningful segment + parts := strings.Split(longHash, "-") + if len(parts) >= 2 { + // Take the last 2-3 parts as a potential compound term slug + potentialSlug := strings.Join(parts[len(parts)-2:], "-") + // Check if this matches a known term + for _, termInfo := range termInfos { + if termInfo.Slug == potentialSlug { + // Return with just one closing parenthesis (remove any extra) + return linkStart + potentialSlug + ")" + } + } + } + } + return match + }) + + // Clean up trailing patterns left from malformed hash fragments + // Pattern: ](#term-slug)-last-part) should become just ](#term-slug) + // This handles cases like [Control Catalog](#control-catalog)-catalog) -> [Control Catalog](#control-catalog) + // The trailing part matches the last segment of the slug (after the last hyphen) + trailingSlugPattern := regexp.MustCompile(`(\]\([^\)]+#([a-z0-9-]+)\))-([a-z0-9-]+)\)`) + result = trailingSlugPattern.ReplaceAllStringFunc(result, func(match string) string { + submatches := trailingSlugPattern.FindStringSubmatch(match) + if len(submatches) >= 4 { + fullSlug := submatches[2] + trailingPart := submatches[3] + // Extract the last segment of the slug (after the last hyphen) + lastHyphenIndex := strings.LastIndex(fullSlug, "-") + if lastHyphenIndex >= 0 { + lastSegment := fullSlug[lastHyphenIndex+1:] + // If the trailing part matches the last segment, remove it + if lastSegment == trailingPart { + return submatches[1] + ")" + } + } else if fullSlug == trailingPart { + // If there's no hyphen and they match exactly, remove it + return submatches[1] + ")" + } + } + return match + }) + + // Clean up any extra closing parentheses that might be left after fixing nested links + // This handles cases where the original nested link had malformed hash fragments + // Pattern: markdown link followed by extra ) and then **, whitespace, or end of string + // BUT: Don't match if the link is inside parentheses (e.g., ([ACR](link)) - that's intentional) + extraParenPattern := regexp.MustCompile(`(\[([^\]]+)\]\([^\)]+\))\)(\s*\*\*|\s|[,.;:!?)]|$)`) + extraParenMatches := extraParenPattern.FindAllStringIndex(result, -1) + // Process from end to start to preserve indices + for i := len(extraParenMatches) - 1; i >= 0; i-- { + match := extraParenMatches[i] + start, end := match[0], match[1] + // Check if there's an opening parenthesis immediately before the link + // If so, this is an intentional acronym pattern like ([ACR](link)), don't remove the paren + if start > 0 && result[start-1] == '(' { + continue // Skip this match, it's intentional + } + // Also preserve the trailing ) if there is an earlier unmatched ( + // in the prose before the link — e.g., "(e.g., access [control](url))." — + // where the closing ) is balancing prose, not link malformation. + // URL parens are balanced pairs and don't shift the count. + if strings.Count(result[:start], "(") > strings.Count(result[:start], ")") { + continue + } + // Otherwise, remove the extra closing parenthesis + matchStr := result[start:end] + submatches := extraParenPattern.FindStringSubmatch(matchStr) + if len(submatches) >= 4 { + // Replace with link + trailing content, without extra ) + replacement := submatches[1] + submatches[3] + result = result[:start] + replacement + result[end:] + } + } + + // Also clean up cases where there are double closing parentheses: ](...))) + // This handles cases like [term](#slug)) or *[term](#slug))* + // Match ](url)) and replace with ](url) - be careful to only match link endings + // Pattern: ]( followed by non-) chars, then )), but make sure it's a link ending + linkDoubleParenPattern := regexp.MustCompile(`(\]\([^\)]+\))\)\)([^*\w]|$)`) + result = linkDoubleParenPattern.ReplaceAllString(result, `$1)$2`) + + // Clean up any double opening brackets that might have been created + // Pattern: **[[text](...)]** should be **[text](...)]** + doubleBracketPattern := regexp.MustCompile(`\*\*\[\[([^\]]+)\]\(([^\)]+)\)\]\*\*`) + result = doubleBracketPattern.ReplaceAllString(result, `**[$1]($2)**`) + + // Process each term (longest first) + // Terms are already sorted by length in buildTermInfos() to ensure compound terms + // like "Preventive Enforcement" are processed before shorter terms like "Enforcement" + // This prevents shorter terms from matching within longer compound terms + for _, termInfo := range termInfos { + // Rebuild skip ranges after each replacement + linkRanges := linkPattern.FindAllStringIndex(result, -1) + codeRanges := inlineCodePattern.FindAllStringIndex(result, -1) + htmlRanges := htmlTagPattern.FindAllStringIndex(result, -1) + + // Find HTML tag pairs (opening and closing tags) to skip ALL content between them + htmlContentRanges := findHTMLContentRanges(result, htmlRanges) + + // Combine skip ranges + var skipRanges []rangeInfo + for _, r := range linkRanges { + skipRanges = append(skipRanges, rangeInfo{start: r[0], end: r[1]}) + } + for _, r := range codeRanges { + skipRanges = append(skipRanges, rangeInfo{start: r[0], end: r[1]}) + } + // Add HTML tags themselves + for _, r := range htmlRanges { + skipRanges = append(skipRanges, rangeInfo{start: r[0], end: r[1]}) + } + // Add ALL content between HTML tag pairs - this completely skips linking terms inside HTML + skipRanges = append(skipRanges, htmlContentRanges...) + + // Sort by start position + sort.Slice(skipRanges, func(i, j int) bool { + return skipRanges[i].start < skipRanges[j].start + }) + + matches := termInfo.Regex.FindAllStringIndex(result, -1) + if len(matches) == 0 { + continue + } + + // Process matches from end to start to preserve indices + for i := len(matches) - 1; i >= 0; i-- { + match := matches[i] + start, end := match[0], match[1] + + // Check if this match is in a skip range (includes existing links, code, HTML) + shouldSkip := false + for _, skipRange := range skipRanges { + // Skip if match overlaps with a skip range at all + // This prevents linking terms that are inside existing links, code blocks, or HTML + if start < skipRange.end && end > skipRange.start { + shouldSkip = true + break + } + } + + if shouldSkip { + continue + } + + // Additional safety check: if the matched text itself contains a markdown link pattern + // This catches edge cases where a link might be embedded in the matched text + matchedText := result[start:end] + linkInMatchPattern := regexp.MustCompile(`\[[^\]]+\]\([^\)]+\)`) + if linkInMatchPattern.MatchString(matchedText) { + // The matched text contains a markdown link, skip it + continue + } + + // Check if match overlaps with any link's text portion (the part between [ and ]) + // This prevents linking shorter terms that are part of longer terms that were already linked + for _, linkRange := range linkRanges { + linkStart, linkEnd := linkRange[0], linkRange[1] + // Find the link text portion (between [ and ]) + linkTextStart := linkStart + 1 // Skip opening [ + linkTextEnd := linkTextStart + for linkTextEnd < linkEnd && result[linkTextEnd] != ']' { + linkTextEnd++ + } + // Check if our match overlaps with the link text portion + // We need to check if the match is completely or partially inside the link text + if start >= linkTextStart && start < linkTextEnd { + // Match starts inside link text - skip to avoid nested links + shouldSkip = true + break + } + if end > linkTextStart && end <= linkTextEnd { + // Match ends inside link text - skip to avoid nested links + shouldSkip = true + break + } + if start <= linkTextStart && end >= linkTextEnd { + // Match completely encompasses link text - skip to avoid nested links + shouldSkip = true + break + } + } + + if shouldSkip { + continue + } + + // Create the link + link := fmt.Sprintf("[%s](%s#%s)", matchedText, defPath, termInfo.Slug) + + // Replace in result + result = result[:start] + link + result[end:] + } + } + + return result +} + +// rangeInfo represents a range of text to skip +type rangeInfo struct { + start, end int +} + +// findHTMLContentRanges finds ranges of content between opening and closing HTML tags +func findHTMLContentRanges(text string, tagRanges [][]int) []rangeInfo { + var contentRanges []rangeInfo + + if len(tagRanges) == 0 { + return contentRanges + } + + // Extract tag information + type tagInfo struct { + start, end int + tagName string + isClosing bool + isSelfClose bool + } + + var tags []tagInfo + // Improved regex: matches tag name at the start (after , , , , etc. + tagNamePattern := regexp.MustCompile(`") || (strings.HasSuffix(tagText, " />") && !isClosing) + + tags = append(tags, tagInfo{ + start: r[0], + end: r[1], + tagName: tagName, + isClosing: isClosing, + isSelfClose: isSelfClose, + }) + } + + // Match opening and closing tags using a stack to handle nesting + type tagStackItem struct { + tagName string + start int + } + var stack []tagStackItem + + for _, tag := range tags { + if tag.isSelfClose { + // Self-closing tags don't have content + continue + } + + if !tag.isClosing { + // Opening tag - push to stack + stack = append(stack, tagStackItem{ + tagName: tag.tagName, + start: tag.end, // Content starts after the opening tag + }) + } else { + // Closing tag - find matching opening tag (search from top of stack) + for i := len(stack) - 1; i >= 0; i-- { + if stack[i].tagName == tag.tagName { + // Found matching opening tag + contentStart := stack[i].start + contentEnd := tag.start // Content ends before the closing tag + + if contentEnd > contentStart { + contentRanges = append(contentRanges, rangeInfo{ + start: contentStart, + end: contentEnd, + }) + } + + // Remove this tag and all tags after it (they were nested inside) + stack = stack[:i] + break + } + } + } + } + + // Handle any unclosed tags (opening tags without matching closing tags) + // This can happen with malformed HTML or tags that span multiple lines + for _, item := range stack { + // For unclosed tags, skip content from the tag end to the end of the text + contentRanges = append(contentRanges, rangeInfo{ + start: item.start, + end: len(text), + }) + } + + return contentRanges +} + +func cleanupContent(content string, termInfos []TermInfo, defPath string) string { + lines := strings.Split(content, "\n") + var result strings.Builder + + inCodeBlock := false + inFrontMatter := false + frontMatterCount := 0 + + for i, line := range lines { + originalLine := line + + // Track front matter + if strings.HasPrefix(strings.TrimSpace(line), "---") { + if !inFrontMatter { + inFrontMatter = true + frontMatterCount = 1 + } else { + frontMatterCount++ + if frontMatterCount == 2 { + inFrontMatter = false + } + } + } + + // Skip processing in front matter + if inFrontMatter { + result.WriteString(originalLine) + if i < len(lines)-1 { + result.WriteString("\n") + } + continue + } + + // Track code blocks (fenced with ```) + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "```") { + inCodeBlock = !inCodeBlock + } + + // Skip code blocks + if inCodeBlock { + result.WriteString(originalLine) + if i < len(lines)-1 { + result.WriteString("\n") + } + continue + } + + // Process the line to remove termlinker-generated links + processedLine := cleanupLine(line, termInfos, defPath) + result.WriteString(processedLine) + if i < len(lines)-1 { + result.WriteString("\n") + } + } + + return result.String() +} + +func cleanupLine(line string, termInfos []TermInfo, defPath string) string { + // Create a map of term text to term info for quick lookup + termMap := make(map[string]TermInfo) + for _, termInfo := range termInfos { + termMap[strings.ToLower(termInfo.OriginalTerm)] = termInfo + } + + // Manually parse markdown links to handle parentheses in link text correctly + // This is more robust than regex for handling nested parentheses + var result strings.Builder + i := 0 + for i < len(line) { + // Look for the start of a markdown link: [ + if line[i] == '[' { + // Find the matching closing bracket ] + bracketStart := i + bracketEnd := -1 + for j := i + 1; j < len(line); j++ { + if line[j] == ']' { + bracketEnd = j + break + } + } + + // If we found a closing bracket, check if it's followed by ( + if bracketEnd != -1 && bracketEnd+1 < len(line) && line[bracketEnd+1] == '(' { + // Extract link text (everything between [ and ]) + linkText := line[bracketStart+1 : bracketEnd] + + // Find the matching closing parenthesis for the URL + parenStart := bracketEnd + 1 + parenEnd := -1 + parenDepth := 1 + for j := parenStart + 1; j < len(line); j++ { + if line[j] == '(' { + parenDepth++ + } else if line[j] == ')' { + parenDepth-- + if parenDepth == 0 { + parenEnd = j + break + } + } + } + + // If we found a matching closing parenthesis, extract the URL + if parenEnd != -1 { + linkURL := line[parenStart+1 : parenEnd] + + // Check if this link points to the definitions page + if strings.Contains(linkURL, "02-definitions") { + // Extract the slug from the URL (part after #) + hashIndex := strings.Index(linkURL, "#") + if hashIndex != -1 { + slug := linkURL[hashIndex+1:] + + // Check if the link text matches a term (case-insensitive) + lowerLinkText := strings.ToLower(linkText) + termInfo, found := termMap[lowerLinkText] + if found && termInfo.Slug == slug { + // This is a termlinker-generated link, remove it and return just the text + result.WriteString(linkText) + i = parenEnd + 1 + continue + } + } + } + } + } + } + + // Not a termlinker link, or parsing failed, keep the character + result.WriteByte(line[i]) + i++ + } + + return result.String() +} diff --git a/tools/main.go b/tools/main.go new file mode 100644 index 0000000..bfa4c87 --- /dev/null +++ b/tools/main.go @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "github.com/gemaraproj/website/tools/internal/cmd" +) + +func main() { + cmd.Execute() +} diff --git a/tools/scripts/parse-nav.sh b/tools/scripts/parse-nav.sh new file mode 100755 index 0000000..abe2908 --- /dev/null +++ b/tools/scripts/parse-nav.sh @@ -0,0 +1,66 @@ +#!/bin/sh +# SPDX-License-Identifier: Apache-2.0 +# Parses docs/schema-nav.yml and outputs page information +# Usage: parse-nav.sh +# Commands: list-pages, get-title , get-filename + +NAV_FILE="$1" +COMMAND="$2" +ARG="$3" + +case "$COMMAND" in + list-pages) + # Output: filename|title (one per line) + awk '/^ - title:/ { + title = $0; + gsub(/.*title: "/, "", title); + gsub(/".*/, "", title); + getline; + if ($0 ~ /filename:/) { + filename = $0; + gsub(/.*filename: /, "", filename); + gsub(/[ "]/, "", filename); + } else { + filename = ""; + } + if (filename == "") { + filename = tolower(title); + gsub(/ /, "-", filename); + } + print filename "|" title + }' "$NAV_FILE" + ;; + get-title) + # Get title for a given filename + awk -v filename="$ARG" ' + BEGIN { found = 0 } + /^ - title:/ { + title = $0; + gsub(/.*title: "/, "", title); + gsub(/".*/, "", title); + getline; + if ($0 ~ /filename:/) { + file = $0; + gsub(/.*filename: /, "", file); + gsub(/[ "]/, "", file); + } else { + file = ""; + } + if (file == "") { + file = tolower(title); + gsub(/ /, "-", file); + } + if (file == filename) { + print title; + found = 1; + exit + } + } + END { if (!found) exit 1 } + ' "$NAV_FILE" + ;; + *) + echo "Unknown command: $COMMAND" >&2 + exit 1 + ;; +esac diff --git a/tools/scripts/schema-display-name.sh b/tools/scripts/schema-display-name.sh new file mode 100644 index 0000000..f98fba6 --- /dev/null +++ b/tools/scripts/schema-display-name.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# SPDX-License-Identifier: Apache-2.0 +# Outputs a display title for a schema file base name (e.g. controlcatalog -> "Control Catalog"). +case "$1" in +base) echo "Base" ;; +metadata) echo "Metadata" ;; +mapping-inline) echo "Mapping Primitives" ;; +guidancecatalog) echo "Guidance Catalog" ;; +vectorcatalog) echo "Vector Catalog" ;; +controlcatalog) echo "Control Catalog" ;; +threatcatalog) echo "Threat Catalog" ;; +riskcatalog) echo "Risk Catalog" ;; +policy) echo "Policy" ;; +mappingdocument) echo "Mapping Document" ;; +evaluationlog) echo "Evaluation Log" ;; +enforcementlog) echo "Enforcement Log" ;; +*) echo "$1" ;; +esac