Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 23 additions & 9 deletions .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,14 +63,6 @@ jobs:
echo "Building against gemara spec ref: $ref"
echo "ref=$ref" >> "$GITHUB_OUTPUT"

- name: Checkout Gemara spec
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
repository: gemaraproj/gemara
ref: ${{ steps.spec.outputs.ref }}
path: .gemara-spec
persist-credentials: false

- name: Setup Pages
if: github.event_name != 'pull_request'
id: pages
Expand All @@ -87,8 +79,30 @@ jobs:
with:
go-version: '1.25'

- name: Fetch spec OpenAPI
env:
GH_TOKEN: ${{ github.token }}
REF: ${{ steps.spec.outputs.ref }}
run: |
mkdir -p generated
# Releases publish openapi.yaml as an asset; fall back to
# generating it from a spec checkout for refs that predate the
# asset (or when building against a branch like main).
if [ "$REF" != "main" ] && gh release download "$REF" \
--repo gemaraproj/gemara \
--pattern openapi.yaml \
--output generated/openapi.yaml 2>/dev/null; then
echo "Downloaded openapi.yaml from release $REF"
else
echo "::warning::No openapi.yaml release asset for $REF; generating from a spec checkout"
git clone --depth 1 --branch "$REF" https://github.com/gemaraproj/gemara .gemara-spec

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit

Suggested change
git clone --depth 1 --branch "$REF" https://github.com/gemaraproj/gemara .gemara-spec
timeout 120 git clone --depth 1 --branch "$REF" https://github.com/gemaraproj/gemara .gemara-spec

(cd .gemara-spec/cmd && go run . cue2openapi \
--schema .. \
--output "$GITHUB_WORKSPACE/generated/openapi.yaml")
fi

- name: Generate documentation
run: make gendocs GEMARA_DIR=.gemara-spec
run: make gendocs

- name: Build with Jekyll
uses: actions/jekyll-build-pages@44a6e6beabd48582f863aeeb6cb2151cc1716697 # v1.0.13
Expand Down
76 changes: 45 additions & 31 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,28 @@
#
# The site content lives at the repository root. Schema reference pages,
# the definitions table, and term cross-links are GENERATED from the Gemara
# specification repo (github.com/gemaraproj/gemara), which provides both the
# CUE schemas and the `gemara-docs` CLI under cmd/.
# specification's OpenAPI projection (openapi.yaml), which the spec repo
# (github.com/gemaraproj/gemara) publishes as a release asset. The markdown
# tooling that renders it lives in this repo under tools/.
#
# GEMARA_DIR points at a checkout of the spec repo. By default it is a
# shallow clone under .gemara-spec/ at GEMARA_REF; set GEMARA_DIR=../gemara
# to build against a local sibling checkout instead.
# openapi.yaml acquisition, in order of precedence:
# GEMARA_OPENAPI=/path/to/openapi.yaml use a pre-generated file
# GEMARA_DIR=../gemara generate from a local spec checkout
# (runs its cue2openapi command)
# GEMARA_REF=v1.2.3 (default: latest) download the release asset

GEMARA_REPO ?= https://github.com/gemaraproj/gemara
GEMARA_REF ?= main
GEMARA_DIR ?= .gemara-spec
GEMARA_REF ?= latest

SPEC_ABS := $(abspath $(GEMARA_DIR))
SITE_ABS := $(abspath .)
TOOLS_DIR := tools
GENERATED_DIR := generated
OPENAPI_YAML := $(GENERATED_DIR)/openapi.yaml
MANIFEST_JSON := $(GENERATED_DIR)/schema-manifest.json
SPEC_MD_DIR := $(GENERATED_DIR)/spec
SCHEMA_DIR := schema
SCHEMA_NAV := schema-nav.yml

.PHONY: all fetch-spec genopenapi genmd gendocs serve build test-links cleanup cleanup-links check-jekyll deps
.PHONY: all fetch-openapi genmd gendocs serve build test-links cleanup cleanup-links check-jekyll deps

all: gendocs test-links cleanup

Expand All @@ -36,27 +37,40 @@ check-jekyll:
exit 1; \
fi

fetch-spec:
@if [ ! -d "$(GEMARA_DIR)" ]; then \
echo " > Cloning Gemara spec ($(GEMARA_REF)) into $(GEMARA_DIR)..."; \
git clone --depth 1 --branch "$(GEMARA_REF)" "$(GEMARA_REPO)" "$(GEMARA_DIR)"; \
# File target: if generated/openapi.yaml already exists (e.g. CI downloaded
# or generated it beforehand), acquisition is skipped entirely.
$(OPENAPI_YAML):
@mkdir -p $(GENERATED_DIR)
@if [ -n "$(GEMARA_OPENAPI)" ]; then \
echo " > Using local OpenAPI file $(GEMARA_OPENAPI) ..."; \
cp "$(GEMARA_OPENAPI)" "$(OPENAPI_YAML)"; \
elif [ -n "$(GEMARA_DIR)" ]; then \
echo " > Generating OpenAPI from local spec checkout $(GEMARA_DIR) ..."; \
cd "$(abspath $(GEMARA_DIR))/cmd" && go run . cue2openapi \
--schema .. \
--output $(SITE_ABS)/$(OPENAPI_YAML); \
else \
echo " > Using existing spec checkout at $(GEMARA_DIR)"; \
if [ "$(GEMARA_REF)" = "latest" ]; then \
url="$(GEMARA_REPO)/releases/latest/download/openapi.yaml"; \
else \
url="$(GEMARA_REPO)/releases/download/$(GEMARA_REF)/openapi.yaml"; \
fi; \
echo " > Downloading $$url ..."; \
curl --fail --silent --show-error --location "$$url" --output "$(OPENAPI_YAML)" || { \
rm -f "$(OPENAPI_YAML)"; \
echo "ERROR: could not download openapi.yaml for spec ref '$(GEMARA_REF)'."; \
echo "Releases before the asset existed can be built from a checkout instead:"; \
echo " make gendocs GEMARA_DIR=/path/to/gemara"; \
exit 1; \
}; \
fi

genopenapi: fetch-spec
@echo " > Converting CUE schema to OpenAPI ..."
@mkdir -p $(GENERATED_DIR)
@cd $(SPEC_ABS)/cmd && go run . cue2openapi \
--schema $(SPEC_ABS) \
--output $(SITE_ABS)/$(OPENAPI_YAML) \
--manifest $(SITE_ABS)/$(MANIFEST_JSON)
@echo " > OpenAPI schema generation complete!"
fetch-openapi: $(OPENAPI_YAML)

genmd: genopenapi
genmd: fetch-openapi
@echo " > Generating markdown from OpenAPI ..."
@mkdir -p $(SPEC_MD_DIR)
@cd $(SPEC_ABS)/cmd && go run . openapi2md \
@cd $(TOOLS_DIR) && go run . openapi2md \
--input $(SITE_ABS)/$(OPENAPI_YAML) \
--output $(SITE_ABS)/$(SPEC_MD_DIR) \
--nav $(SITE_ABS)/$(SCHEMA_NAV)
Expand All @@ -65,7 +79,7 @@ genmd: genopenapi
gendocs: genmd
@echo " > Copying schema pages to $(SCHEMA_DIR)/ for website ..."
@mkdir -p $(SCHEMA_DIR)
@sh "$(SPEC_ABS)/cmd/scripts/parse-nav.sh" "$(SCHEMA_NAV)" list-pages | while IFS='|' read -r filename title; do \
@sh "$(TOOLS_DIR)/scripts/parse-nav.sh" "$(SCHEMA_NAV)" list-pages | while IFS='|' read -r filename title; do \
if [ -f "$(SPEC_MD_DIR)/$$filename.md" ]; then \
{ \
echo "---"; \
Expand All @@ -80,7 +94,7 @@ gendocs: genmd
@echo " > Updating schema list in $(SCHEMA_DIR)/index.md ..."
@if [ -f "$(SCHEMA_DIR)/index.md" ]; then \
schema_list_file="$(SCHEMA_DIR)/index.md.schema_list.tmp"; \
sh "$(SPEC_ABS)/cmd/scripts/parse-nav.sh" "$(SCHEMA_NAV)" list-pages | while IFS='|' read -r filename title; do \
sh "$(TOOLS_DIR)/scripts/parse-nav.sh" "$(SCHEMA_NAV)" list-pages | while IFS='|' read -r filename title; do \
[ -f "$(SCHEMA_DIR)/$$filename.md" ] && echo "- [$$title]($$filename.html)"; \
done > "$$schema_list_file"; \
awk -v list_file="$$schema_list_file" ' \
Expand Down Expand Up @@ -108,11 +122,11 @@ gendocs: genmd
@if [ -f "model/02-definitions.md.template" ]; then \
cp "model/02-definitions.md.template" "model/02-definitions.md"; \
fi
@cd $(SPEC_ABS)/cmd && go run . lexicon2md \
@cd $(TOOLS_DIR) && go run . lexicon2md \
--lexicon $(SITE_ABS)/lexicon.yaml \
--output $(SITE_ABS)/model/02-definitions.md
@echo " > Linking defined terms across documentation ..."
@cd $(SPEC_ABS)/cmd && go run . termlinker \
@cd $(TOOLS_DIR) && go run . termlinker \
--lexicon $(SITE_ABS)/lexicon.yaml \
--docs $(SITE_ABS)
@echo " > Documentation generation complete!"
Expand All @@ -137,15 +151,15 @@ test-links:

cleanup-links:
@echo " > Removing termlinker-generated links from documentation ..."
@cd $(SPEC_ABS)/cmd && go run . termlinker \
@cd $(TOOLS_DIR) && go run . termlinker \
--lexicon $(SITE_ABS)/lexicon.yaml \
--docs $(SITE_ABS) \
--cleanup
@echo " > Link cleanup complete!"

cleanup: cleanup-links
@echo " > Removing generated documentation files and links..."
@sh "$(SPEC_ABS)/cmd/scripts/parse-nav.sh" "$(SCHEMA_NAV)" list-pages | while IFS='|' read -r filename title; do \
@sh "$(TOOLS_DIR)/scripts/parse-nav.sh" "$(SCHEMA_NAV)" list-pages | while IFS='|' read -r filename title; do \
rm -f "$(SCHEMA_DIR)/$$filename.md"; \
done
@rm -f model/02-definitions.md
Expand Down
21 changes: 14 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ Keep the front matter. Edit everything below it.

You need **Ruby 3.2 or newer** and **Go 1.25 or newer**.

Go is needed because the schema pages are generated by a tool in the spec repo.
Go is needed because the schema pages are generated by the tooling in `tools/`.

```bash
make deps # install Ruby dependencies (run once)
Expand Down Expand Up @@ -113,9 +113,9 @@ That means `make serve` may leave link markup in your working copy.
## How the build works

```
gemaraproj/gemara ──► CUE schemas + the gemara-docs CLI
gemaraproj/gemara release ──► openapi.yaml (release asset)
│ make gendocs (clones the spec into .gemara-spec/)
│ make gendocs (downloads the asset, renders it with tools/)
generated/ ──► schema/*.md and model/02-definitions.md
Expand All @@ -124,14 +124,21 @@ gemaraproj/gemara ──► CUE schemas + the gemara-docs CLI
_site/ ──► GitHub Pages ──► gemara.openssf.org
```

By default the build clones the spec repo into `.gemara-spec/`.
By default the build downloads `openapi.yaml` from the **latest spec release**
(`GEMARA_REF=v1.2.3` pins a specific one). The markdown renderers
(`openapi2md`, `lexicon2md`, `termlinker`) live in this repo under `tools/`.

To build against a local checkout of the spec instead:
To build against a local checkout of the spec instead (e.g. for unreleased
schema changes, or releases that predate the asset):

```bash
make serve GEMARA_DIR=../gemara
```

This runs the spec repo's `cue2openapi` command against that checkout to
produce `generated/openapi.yaml`. A pre-generated file also works:
`make serve GEMARA_OPENAPI=/path/to/openapi.yaml`.

### Make targets

| Command | What it does |
Expand Down Expand Up @@ -186,8 +193,8 @@ Maintainers are listed in `_data/maintainers.yml`.
Run `make deps` first.

**Schema pages are empty or missing**
The spec checkout may be stale. Delete it and try again:
`rm -rf .gemara-spec && make gendocs`
The downloaded OpenAPI file may be stale. Delete it and try again:
`rm -rf generated && make gendocs`

**Weird link markup all over my diff**
That's the term linker. Run `make cleanup`.
Expand Down
13 changes: 13 additions & 0 deletions tools/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
module github.com/gemaraproj/website/tools

go 1.25.0

require (
github.com/goccy/go-yaml v1.19.2
github.com/spf13/cobra v1.10.1
)

require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/spf13/pflag v1.0.9 // indirect
)
12 changes: 12 additions & 0 deletions tools/go.sum
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=
github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
92 changes: 92 additions & 0 deletions tools/internal/cmd/lexicon2md.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
// SPDX-License-Identifier: Apache-2.0

package cmd

import (
"bytes"
"fmt"
"os"
"strings"
"text/template"

"github.com/goccy/go-yaml"
"github.com/spf13/cobra"
)

type TemplateData struct {
Table string
}

var lexicon2MDCmd = &cobra.Command{
Use: "lexicon2md",
Short: "Generate definitions table from lexicon YAML",
Long: `Generate a definitions table in Markdown format from a lexicon YAML file.
The lexicon file should contain an array of terms with their definitions and references.
The output will be written to a Markdown file using a template.`,
RunE: runLexicon2MD,
}

var lexicon2MDFlags struct {
lexiconFile string
outputFile string
}

func newLexicon2MDCmd() *cobra.Command {
lexicon2MDCmd.Flags().StringVarP(&lexicon2MDFlags.lexiconFile, "lexicon", "l", "lexicon.yaml", "Input lexicon YAML file")
lexicon2MDCmd.Flags().StringVarP(&lexicon2MDFlags.outputFile, "output", "o", "model/02-definitions.md", "Output markdown file")
return lexicon2MDCmd
}

func runLexicon2MD(cmd *cobra.Command, args []string) error {
data, err := os.ReadFile(lexicon2MDFlags.lexiconFile)
if err != nil {
return fmt.Errorf("Error reading lexicon file: %v", err)
}

var lexicon Lexicon
if err := yaml.Unmarshal(data, &lexicon); err != nil {
return fmt.Errorf("Error parsing lexicon YAML: %v", err)
}

var tableRows strings.Builder
for _, term := range lexicon.Terms {
slug := termToSlug(term.Title)

termName := fmt.Sprintf("<a id=\"%s\"></a>**%s**", slug, term.Title)

refs := make([]string, 0, len(term.References))
for _, r := range term.References {
refs = append(refs, r.Citation)
}
appliesTo := strings.Join(refs, "<br>")

definition := strings.TrimSpace(strings.ReplaceAll(term.Definition, "|", "\\|"))

tableRows.WriteString(fmt.Sprintf("| %s | %s | %s |\n", termName, definition, appliesTo))
}

templateContent, err := os.ReadFile(lexicon2MDFlags.outputFile)
if err != nil {
return fmt.Errorf("Error reading output file: %v", err)
}

tmpl, err := template.New("definitions").Parse(string(templateContent))
if err != nil {
return fmt.Errorf("Error parsing template: %v", err)
}

var output bytes.Buffer
templateData := TemplateData{
Table: strings.TrimSpace(tableRows.String()),
}
if err := tmpl.Execute(&output, templateData); err != nil {
return fmt.Errorf("Error executing template: %v", err)
}

if err := os.WriteFile(lexicon2MDFlags.outputFile, output.Bytes(), 0644); err != nil {
return fmt.Errorf("Error writing output file: %v", err)
}

fmt.Printf("Successfully generated definitions table in %s\n", lexicon2MDFlags.outputFile)
return nil
}
Loading
Loading