Merge pull request #1164 from bact/use-spdx-3-model-develop-for-publish #141

Summary
Jobs
- build
Run details
- Usage
- Workflow file

Workflow file for this run

.github/workflows/publish_v3.yml at 873afc0

	# Publish SPDX specification to https://spdx.github.io/spdx-spec/
	#
	# This workflow is configured differently for the
	# "main", "develop", and "support" branches, each publishing to a unique URL.
	#
	# For example,
	# the workflow in "main" may publish to https://spdx.github.io/spdx-spec/v3.0.1/,
	# the workflow in "develop" may publish to https://spdx.github.io/spdx-spec/v3.1-dev/,
	# the workflow in "support/3.0" may publish to https://spdx.github.io/spdx-spec/v3.0.x/.
	#
	# The workflow should be configured to have an URL without a version number
	# specified be redirected to an URL published from "main" branch.
	#
	# ## Workflow overview
	#
	# 1) Generate model documents and RDFs from model files in spdx-3-model repo
	# 2) Combine the model documents from (1) with the chapters in spdx-spec repo
	# 3) Generate a website using files from (2)
	# 4) Upload RDFs from (1) and a website from (3) to GitHub Pages
	# 5) Make URL redirections as needed
	#
	# See notes at:
	# https://github.com/spdx/spdx-spec/issues/1155
	# https://github.com/spdx/spdx-spec/pull/1146
	# See branch structure at:
	# https://github.com/spdx/spdx-spec/blob/develop/README.md#branch-structure

	on:
	push:
	branches:
	- develop # This should match with REF_SPEC,
	# to automatically publish from a correct branch
	repository_dispatch:
	types:
	- publish_v3_spec
	workflow_dispatch: {} # Manually trigger from https://github.com/spdx/spdx-spec/actions
	jobs:
	build:
	runs-on: ubuntu-latest
	env:
	REF_SPEC: "develop" # spdx-spec branch: "main" or "develop" or "support/x.y" or tag/commit ID
	REF_MODEL: "develop" # spdx-3-model branch: "main" or "develop" or "support/x.y" or tag/commit ID
	REF_PARSER: "main" # spec-parser branch: "main" or tag/commit ID
	GH_PAGES_BRANCH: "gh-pages" # spdx-spec branch to publish HTML to
	VERSION_DEFAULT: "v3.0.1" # Default version:
	# - A version to be redirected to from the URL without
	# a version number specified
	# - Should be the latest stable version from the "main" branch
	# - VERSION_DEFAULT should be the same across all
	# branches/tags
	# - VERSION_DEFAULT should also match the
	# mike's canonical_version in mkdocs.yml
	VERSION: "v3.1-dev" # Publishing version, to be published from this branch:
	# - VERSION can be different from VERSION_DEFAULT;
	# For example, if VERSION is a draft/release candidate,
	# or if VERSION is a stable version that is behind the
	# default version (e.g. v3.0.2 vs v3.1)
	# - VERSION from "develop" branch should be indicated with
	# a suffix ("-dev", "-draft", etc.).
	# The content of this version will constantly change.
	# - VERSION should match with the version in the copyright
	# text defined in mkdocs.yml
	# e.g. "SPDX v3.x.x Copyright (c) 2010-2024, ..."
	# - A release candidate (with suffix "-RC") may be published
	# from a very short-lived "support" branch.
	# The content of this version should be kept unchanged,
	# so it can be properly referenced during the review period,
	# but the URL of the RC version may subjected to be
	# redirected to the release version later.
	# For example, v3.0-RC1 was redirected to v3.0 and
	# will be redirected to v3.0.1 later.
	VERSION_ALIASES: "v3.1"
	# VERSION_ALIASES are names that will be redirected to VERSION
	# - Can be empty, can be multiple; separated by space
	# - "latest" should be reserved for the latest version
	# - Versions like "v3.0" will be expanded to "v3.0 3.0"
	GIT_USER_NAME: "ci-bot" # Username for gh-pages commit
	GIT_USER_EMAIL: "[email protected]" # E-mail for gh-pages commit
	MKDOCS_MODEL_YML: "model-files.yml" # Contains list of model Markdown files:
	# - relative to PARSER_OUT_BASE_DIR
	MKDOCS_BASE_YML: "mkdocs.yml" # Initial MkDocs configuration; from spdx-spec repo
	MKDOCS_FULL_YML: "__mkdocs-full.yml" # MkDocs configuration combined with model list:
	# - to be generated from MKDOCS_BASE_YML and MKDOCS_MODEL_YML
	REDIRECT_MAP_PATH: "etc/redirect-map.csv" # URL redirect map
	REDIRECT_TEMPLATE_PATH: "etc/redirect-template.html" # URL redirect HTML template
	steps:
	- name: Expand version aliases to include a version without 'v' prefix
	# For example, given
	# - VERSION: "v3.0.1"; and
	# - Original VERSION_ALIASES: "latest v3.0"
	# the expanded VERSION_ALIASES will be "3.0.1 latest v3.0 3.0"
	run: \|
	echo VERSION: $VERSION
	echo Original VERSION_ALIASES: $VERSION_ALIASES
	original_aliases="$VERSION_ALIASES"
	expanded_aliases=""

	if [[ $VERSION =~ ^v[0-9] ]]; then
	expanded_aliases="$expanded_aliases ${VERSION#v}"
	fi

	for version in $original_aliases; do
	expanded_aliases="$expanded_aliases $version"
	if [[ $version =~ ^v[0-9] ]]; then
	expanded_aliases="$expanded_aliases ${version#v}"
	fi
	done

	expanded_aliases=$(echo $expanded_aliases \| sed 's/^ *//g')
	echo "VERSION_ALIASES=$expanded_aliases" >> $GITHUB_ENV
	- name: Check expanded version aliases
	run: \|
	echo Expanded VERSION_ALIASES: $VERSION_ALIASES
	- name: Checkout spdx-spec
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2
	with:
	ref: ${{ env.REF_SPEC }}
	path: spdx-spec
	fetch-depth: 0 # Because we will be pushing the gh-pages branch
	- name: Checkout spdx-3-model
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2
	with:
	repository: spdx/spdx-3-model
	ref: ${{ env.REF_MODEL }}
	path: spdx-3-model
	- name: Checkout spec-parser
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2
	with:
	repository: spdx/spec-parser
	ref: ${{ env.REF_PARSER }}
	path: spec-parser
	- name: Set up specific Python version
	uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 #v5.4.0
	with:
	python-version: "3.12"
	cache: "pip"
	- name: Install pre-requisites for spdx-spec
	run: pip install -r spdx-spec/requirements.txt
	- name: Install pre-requisites for spec-parser
	run: pip install -r spec-parser/requirements.txt
	- name: Build model files
	# Also move the model file list to spdx-spec/ directory,
	# as it should not be part of the publicly accessible website.
	run: \|
	python3 spec-parser/main.py --force --generate-rdf --output-rdf spdx-spec/docs/rdf --generate-mkdocs --output-mkdocs spdx-spec/docs/model --generate-plantuml --output-plantuml spdx-spec/docs/diagram --generate-jsondump --output-jsondump spdx-spec/docs/jsondump spdx-3-model/model
	mv spdx-spec/docs/model/$MKDOCS_MODEL_YML spdx-spec/
	- name: Copy JSON annotations
	# JSON annotations URL will be redirected
	# from https://spdx.org/rdf/<version>/spdx-json-serialize-annotations.ttl
	# to https://spdx.github.io/spdx-spec/v<version>/rdf/jsonld-annotations.ttl
	#
	# To maintain availability of files at the old location at
	# https://spdx.github.io/spdx-spec/v3.0/model/<filename>,
	# another copy will be made into spdx-spec/docs/model/.
	#
	# Note: When release a new version, update the content of annotations.ttl to match the version
	run: \|
	cp spdx-spec/serialization/jsonld/annotations.ttl spdx-spec/docs/rdf/jsonld-annotations.ttl
	cp spdx-spec/serialization/jsonld/annotations.ttl spdx-spec/docs/model/jsonld-annotations.ttl
	- name: Copy JSON-LD context and RDFs
	# JSON-LD context and RDF URLs will be redirected
	# from https://spdx.org/rdf/<version>/spdx-context.jsonld, spdx-model.ttl, etc.
	# to https://spdx.github.io/spdx-spec/v<version>/rdf/spdx-context.jsonld, etc.
	#
	# Make a copy of spdx-model.json-ld as spdx-model.jsonld,
	# because we have spdx-context.jsonld.
	run: \|
	cp spdx-spec/docs/rdf/spdx-model.json-ld spdx-spec/docs/rdf/spdx-model.jsonld
	cp spdx-spec/docs/rdf/spdx-context.jsonld spdx-spec/docs/model/
	cp spdx-spec/docs/rdf/spdx-model.* spdx-spec/docs/model/
	- name: Generate JSON schema
	# JSON schema URL will be redirected
	# from https://spdx.org/schema/<version>/spdx-json-schema.json
	# to https://spdx.github.io/spdx-spec/v<version>/rdf/schema.json
	#
	# Note: When release a new version, update URL in --context-url line to match the version
	run: \|
	shacl2code generate \
	--input spdx-spec/docs/rdf/spdx-model.ttl \
	--input spdx-spec/docs/rdf/jsonld-annotations.ttl \
	--context-url spdx-spec/docs/rdf/spdx-context.jsonld https://spdx.org/rdf/3.0.1/spdx-context.jsonld \
	jsonschema \
	--output spdx-spec/docs/rdf/schema.json
	cp spdx-spec/docs/rdf/schema.json spdx-spec/docs/model/schema.json
	- name: Set Git identity
	working-directory: spdx-spec
	run: git config user.name $GIT_USER_NAME; git config user.email $GIT_USER_EMAIL
	- name: Sync GitHub Pages
	working-directory: spdx-spec
	run: git checkout $GH_PAGES_BRANCH && git pull && git checkout $REF_SPEC
	- name: Build complete MkDocs configuration
	# Combines model file list (MKDOCS_MODEL_YML, generated by spec-parser)
	# with the base MkDocs configuration file (MKDOCS_BASE_YML),
	# to produce the full MkDocs configuration file (MKDOCS_FULL_YML).
	# The script below finds "__MODEL_PLACEHOLDER__" string in
	# MKDOCS_BASE_YML, replaces it with the content from MKDOCS_MODEL_YML.
	# MKDOCS_FULL_YML will be used by mike in the deploy step.
	working-directory: spdx-spec
	run: \|
	echo "Build $MKDOCS_FULL_YML from $MKDOCS_BASE_YML and $MKDOCS_MODEL_YML"
	bin/make-mkdocs-config.sh \
	-b "$MKDOCS_BASE_YML" \
	-m "$MKDOCS_MODEL_YML" \
	-f "$MKDOCS_FULL_YML" \
	-p "__MODEL_PLACEHOLDER__"
	echo "===================="
	echo "Full MkDocs configuration: $MKDOCS_FULL_YML"
	echo "--------------------"
	cat "$MKDOCS_FULL_YML"
	echo "===================="
	- name: Deploy and set aliases
	# mike is used here to manage multiple versions of MkDocs-powered documentation
	# This step does 2 things:
	# 1) delete existing aliases (in VERSION_ALIASES), if exists
	# 2) deploy as VERSION, with aliases
	# If the existing aliases were redirected to other versions,
	# it means this VERSION will "steal" the aliases from those versions.
	working-directory: spdx-spec
	run: \|
	for alias in $VERSION_ALIASES; do
	mike delete --config-file "$MKDOCS_FULL_YML" --branch $GH_PAGES_BRANCH --push --allow-empty "$alias" \|\| true
	done
	mike deploy --update-aliases --config-file "$MKDOCS_FULL_YML" --branch $GH_PAGES_BRANCH --push $VERSION $VERSION_ALIASES
	- name: Set default version
	# Set default version to VERSION_DEFAULT;
	# if not set, the default version will remain the same.
	# Should only be done from the "main" branch.
	if: github.ref == 'refs/heads/main'
	working-directory: spdx-spec
	run: \|
	mike set-default --config-file "$MKDOCS_FULL_YML" --branch $GH_PAGES_BRANCH --push $VERSION_DEFAULT
	- name: Copy JSON annotations, JSON schema, JSON-LD context, and RDFs to alias directories
	# Fallback for backward compatibility with old URLs before v3.0.1
	# This step creates copies of annotations/schema/RDFs to all alias
	# directories, so they can be accessible from all old URLs.
	# For example,
	# - https://spdx.github.io/spdx-spec/v3.0/model/schema.json (old directory structure)
	# - https://spdx.github.io/spdx-spec/v3.0.1/rdf/schema.json (new directory structure)
	# will all be accessible and have the same content.
	# Unlike HTML files, these files have to be a copy,
	# since it cannot use the HTML refresh mechanism.
	working-directory: spdx-spec
	run: \|
	git checkout $GH_PAGES_BRANCH
	dirs="$VERSION_ALIASES"
	for dir in $dirs; do
	mkdir -p "$dir"/rdf
	mkdir -p "$dir"/model
	cp $VERSION/rdf/* "$dir"/rdf
	cp $VERSION/rdf/* "$dir"/model
	mkdir -p "$dir"/diagram
	cp $VERSION/diagram/* "$dir"/diagram
	mkdir -p "$dir"/jsondump
	cp $VERSION/jsondump/* "$dir"/jsondump
	git add "$dir"/rdf/* "$dir"/model/* "$dir"/diagram/* "$dir"/jsondump/*
	done
	git commit -m "Copy schema and RDFs to alias directories: $VERSION_ALIASES"
	git push origin $GH_PAGES_BRANCH
	- name: Make redirections (for renamed model elements and moved annexes)
	# Fallback for backward compatibility with old URLs before v3.0.1
	# More redirections can be added in etc/redirect-map.csv (from,to)
	# See name changes in model at
	# https://github.com/spdx/spdx-3-model/blob/main/CHANGELOG.md
	#
	# This step creates a HTML files to facilitate additional directions.
	# It reads a redirect map from /etc/redirect-map.csv; in the CSV,
	# first value is 'from' (source) and second value is 'to' (target).
	#
	# The 'from' and 'to' values will be inserted into a HTML template at
	# from /etc/redirect-template.html, to create a redirect HTML
	# page (index.html) under a subdirectory with the name of 'from'
	# that will refresh the browser to a URL of 'to'.
	#
	# For example, given:
	#
	# VERSION = "v3.0.1"
	# VERSION_ALIASES = "latest v3.0"
	# from = "model/Core/Properties/imports"
	# to = "model/Core/Properties/import"
	#
	# these HTML files will be created for every aliases:
	#
	# v3.0.1/model/Core/Properties/imports/index.html
	# latest/model/Core/Properties/imports/index.html
	# v3.0/model/Core/Properties/imports/index.html
	#
	# and all of them will redirect to
	#
	# v3.0.1/model/Core/Properties/import/
	working-directory: spdx-spec
	run: \|
	ALL_VERSIONS=$(echo "$VERSION" "$VERSION_ALIASES")
	INDEX_HTML="index.html"
	git checkout $REF_SPEC
	maps=$(cat "$REDIRECT_MAP_PATH")
	template=$(cat "$REDIRECT_TEMPLATE_PATH")
	echo "===================="
	echo "Redirect map: $REDIRECT_MAP_PATH"
	echo "--------------------"
	echo "$maps"
	echo "===================="
	echo "===================="
	echo "Redirect HTML template: $REDIRECT_TEMPLATE_PATH"
	echo "--------------------"
	echo "$template"
	echo "===================="
	git checkout $GH_PAGES_BRANCH
	for alias in $ALL_VERSIONS; do
	echo "$maps" \| while read -r line; do
	from=$(echo "$line" \| cut -d',' -f1)
	to=$(echo "$line" \| cut -d',' -f2)
	slash_count=$(echo "$from" \| tr -cd '/' \| wc -c)
	upper_dirs=".."
	if [ -n "$from" ]; then
	for i in $(seq 0 $slash_count); do
	upper_dirs="$upper_dirs/.."
	done
	fi
	escaped_upper=$(echo "$upper_dirs" \| sed 's/[\/&]/\\&/g')
	escaped_version=$(echo "$VERSION" \| sed 's/[\/&]/\\&/g')
	html=""
	case "$to" in
	http://\|https://)
	echo "Redirect: $alias/$from -> $to"
	escaped_to=$(echo "$to" \| sed 's/[\/&]/\\&/g')
	html=$(echo "$template" \| sed -e "s\|__UPPER__/__VERSION__/__TO__\|$escaped_to\|g")
	;;
	*)
	echo "Redirect: $alias/$from -> $VERSION/$to"
	escaped_to=$(echo "$to" \| sed 's/[\/&]/\\&/g')
	html=$(echo "$template" \| sed -e "s/__UPPER__/$escaped_upper/g" -e "s/__VERSION__/$escaped_version/g" -e "s/__TO__/$escaped_to/g")
	;;
	esac
	mkdir -p "$alias/$from"
	echo "$html" > "$alias/$from/$INDEX_HTML"
	git add "$alias/$from/$INDEX_HTML"
	done
	done
	git commit -m "Add redirections for: $ALL_VERSIONS"
	git push origin $GH_PAGES_BRANCH

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Merge pull request #1164 from bact/use-spdx-3-model-develop-for-publish #141

Workflow file

Merge pull request #1164 from bact/use-spdx-3-model-develop-for-publish #141

Jobs

Run details

Workflow file for this run