Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
be73042
Add identify() with provenance, overrides, tag parsing and a golden c…
mbauman Nov 17, 2026
839cf28
Don't accept placeholder versions from Repology's URL data
mbauman Nov 17, 2026
028d9e0
Make data maps order-independent; address review: short commits, mixe…
mbauman Nov 17, 2026
413e513
Don't treat all-digit date tags as commit hashes
mbauman Nov 17, 2026
535cc30
Allow restarts after a numeric prefix when they parse more components
mbauman Nov 17, 2026
675d868
Don't start a version after a dot that follows a word (RELEASE.2025-...)
mbauman Nov 17, 2026
b6ac95a
Add a curated msgpack-c release pattern
mbauman Nov 17, 2026
5142b8b
Add vercmp using libversion_jll with per-project version schemes
mbauman Nov 17, 2026
8a92672
Ignore build metadata when comparing SemVer versions
mbauman Nov 17, 2026
632a73a
Leave data refreshes to the scheduled workflow; give it permission to…
mbauman Nov 17, 2026
1876bfa
Fold overrides.toml into additional_info.toml; curated URL patterns o…
mbauman Nov 17, 2026
4ea39e1
Accept upstream suffixes JLL versions can't spell in the corpus; prop…
mbauman Nov 17, 2026
c1f70ad
corpus_match: require the whole version in the URL, not a prefix of it
mbauman Nov 17, 2026
5a7c7c0
Revert corpus_match; correct the xorriso corpus entry to its upstream…
mbauman Nov 17, 2026
c761cc3
Leave the xorriso corpus fix to the refresh PR the workflow opens
mbauman Nov 17, 2026
770a099
Correct the xorriso corpus entry: upstream is 1.5.6.pl02 (GeneralMeta…
mbauman Nov 17, 2026
6d2705d
Curate url_patterns for ffmpeg, libvirt and bokeh tarballs claimed by…
mbauman Nov 17, 2026
d92597e
Check the project of every corpus entry; send refreshes that move an …
mbauman Nov 17, 2026
0b6403c
Curated repositories take precedence over Repology's; tag_patterns pi…
mbauman Nov 17, 2026
6bf93c3
Match SDL prerelease tags to their major version's project
mbauman Nov 17, 2026
ae2203f
Curate url_patterns for 16 projects with known corpus misses; correct…
mbauman Nov 17, 2026
62b3d4f
Fix gaps a full GeneralMetadata regeneration exposed: v-prefixed tags…
mbauman Nov 17, 2026
910bb1b
Don't accept known placeholder versions like vim's "git" as plausible
mbauman Nov 17, 2026
52634ee
Cover 32-bit Git for Windows tarballs; tag schemes for opencl-clang, …
mbauman Nov 17, 2026
06269b8
Don't accept commit hashes as plausible versions
mbauman Nov 18, 2026
fede20b
Gate artifact refreshes on the corpus: push Artifacts.toml only if th…
mbauman Nov 18, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 67 additions & 6 deletions .github/workflows/repology.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,15 @@ name: Export from Repology's DB dump

on:
workflow_dispatch:
schedule:
- cron: '0 6 * * 1'

jobs:
repology-export:
runs-on: ubuntu-latest
permissions:
contents: write # to create the data release and push Artifacts.toml
pull-requests: write # to propose refreshes that need review
defaults:
run:
shell: bash # -eo pipefail
Expand Down Expand Up @@ -36,13 +39,28 @@ jobs:
id: update
env:
REPOLOGY_DATA_OUT: ${{ runner.temp }}/repology_data
REPOLOGY_PROJECT_CHANGES: ${{ runner.temp }}/project_changes.md
run: |
julia --project=. scripts/update_repology_info.jl
if [ -s "$REPOLOGY_PROJECT_CHANGES" ]; then
echo "project_changes=true" >> "$GITHUB_OUTPUT"
else
echo "project_changes=false" >> "$GITHUB_OUTPUT"
fi

- name: Test the new data
- name: Check the corpus against the new data
id: corpus
env:
REPOLOGY_DATA: ${{ runner.temp }}/repology_data
REPOLOGY_ALLOW_FIXED: 1
run: |
if julia --project=. -e 'using Pkg; Pkg.test()'; then
echo "passed=true" >> "$GITHUB_OUTPUT"
else
echo "passed=false" >> "$GITHUB_OUTPUT"
fi
julia --project=. scripts/known_failures.jl
# known_failures.toml now accepts the regressions, so anything still failing is a real bug
julia --project=. -e 'using Pkg; Pkg.test()'

- name: Publish the new data as a dated release and bind it in Artifacts.toml (if it changed)
Expand Down Expand Up @@ -77,19 +95,62 @@ jobs:
- name: Commit changes (if any)
id: git_commit
run: |
git add Artifacts.toml
git add Artifacts.toml test/known_failures.toml
if git diff --cached --quiet; then
echo "No changes to commit."
echo "committed=false" >> "$GITHUB_OUTPUT"
else
git diff --cached -U0 test/known_failures.toml > "$RUNNER_TEMP/known_failures.diff"
git commit -m "[automatic] update the repology_data artifact"
echo "committed=true" >> "$GITHUB_OUTPUT"
fi

- name: Push directly to branch or main if exact
if: |
steps.git_commit.outputs.committed == 'true'
- name: Push to the current branch if the corpus passed and no exported URL changed project
if: steps.git_commit.outputs.committed == 'true' && steps.corpus.outputs.passed == 'true' && steps.update.outputs.project_changes != 'true'
run: |
# For PRs, push to the PR's branch; for main, push to main
current_branch="${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
git push origin HEAD:"$current_branch"

- name: Otherwise propose the refresh as a PR
if: steps.git_commit.outputs.committed == 'true' && (steps.corpus.outputs.passed != 'true' || steps.update.outputs.project_changes == 'true')
env:
GH_TOKEN: ${{ github.token }}
run: |
set -u
base="${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
refresh_branch="automatic/repology-data-refresh-${base//\//-}"
git push --force origin HEAD:"$refresh_branch"
known_failures="$RUNNER_TEMP/known_failures.diff"
project_changes="$RUNNER_TEMP/project_changes.md"
body="$RUNNER_TEMP/pr_body.md"
# shellcheck disable=SC2016 # backticks in the PR text are Markdown
{
printf 'This Repology data refresh needs review, so it was not pushed to `%s`.\n' "$base"
if [ -s "$known_failures" ]; then
cat <<'EOF'

It regressed corpus entries, now recorded in `test/known_failures.toml`. Before merging, either fix the regression in `additional_info.toml`, correct the entry in `test/corpus.toml` if the corpus was wrong and Repology is right, or accept it as a known failure.

```diff
EOF
cat "$known_failures"
printf '```\n'
fi
if [ -s "$project_changes" ]; then
printf '\n%s previously-exported URLs now map to a different project. ' "$(($(wc -l < "$project_changes") - 2))"
cat <<'EOF'
Many are Repology renames; where a downstream package has claimed an upstream URL, add a `url_patterns` entry for the upstream project in `additional_info.toml`.

<details><summary>Project changes</summary>

EOF
head -n 302 "$project_changes"
printf '\n</details>\n'
fi
printf '\nFrom %s/%s/actions/runs/%s\n' "$GITHUB_SERVER_URL" "$GITHUB_REPOSITORY" "$GITHUB_RUN_ID"
} > "$body"
if [ -n "$(gh pr list --head "$refresh_branch" --state open --json number --jq '.[].number')" ]; then
gh pr edit "$refresh_branch" --body-file "$body"
else
gh pr create --base "$base" --head "$refresh_branch" --title "[automatic] Repology data refresh for review" --body-file "$body"
fi
2 changes: 2 additions & 0 deletions Project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,15 @@ Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
libversion_jll = "536dcc1b-10b1-5f38-bd5e-495471764a69"

[compat]
Artifacts = "1"
DataFrames = "1.8.0"
DataStructures = "0.19.1"
TOML = "1.0.3"
julia = "1.10"
libversion_jll = "3.0.3"

[extras]
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
Expand Down
195 changes: 193 additions & 2 deletions additional_info.toml
Original file line number Diff line number Diff line change
@@ -1,4 +1,14 @@
# Extra information for packages that are in repology, but haven't used the sources we used (or were otherwise skipped from the dump extract)
#
# Per project, all optional:
# url_patterns = regexes matching source URLs; the first capture group is the version. These take precedence
# over Repology's own URL data.
# repositories = source repository URLs
# tag_patterns = regexes extracting the version from a git tag
# cpes = CPE prefixes
# version_scheme / version_flags = how versions compare; see `Repology.version_scheme`
# sources = hand-verified one-off sources that no pattern covers, e.g. an untagged commit; checked first:
# sources = [{url = "https://github.com/owner/repo.git", commit = "0123abc...", version = "1.2.3"}]

[ampl-asl]
url_patterns = ["^\\Qhttp://netlib.org/ampl/solvers.tgz\\E()$"] # No versioning info available
Expand All @@ -16,10 +26,10 @@ url_patterns = ["^\\Qhttp://fftw.org/~stevenj/fftw-\\E(\\d.*)\\Q.tar.gz\\E$"]
repositories = ["https://github.com/sciapp/gr.git"]

[git]
url_patterns = ["^\\Qhttps://github.com/git-for-windows/git/releases/download/v\\E(\\d.*)\\Q.windows.1/Git-\\E\\1\\Q-64-bit.tar.bz2\\E$"]
url_patterns = ["^\\Qhttps://github.com/git-for-windows/git/releases/download/v\\E(\\d.*)\\Q.windows.1/Git-\\E\\1-(?:32|64)\\Q-bit.tar.bz2\\E$"]

[git-for-windows]
url_patterns = ["^\\Qhttps://github.com/git-for-windows/git/releases/download/v\\E(\\d.*)\\Q/Git-\\E.+\\Q-64-bit.tar.bz2\\E$"]
url_patterns = ["^\\Qhttps://github.com/git-for-windows/git/releases/download/v\\E(\\d.*)\\Q/Git-\\E.+-(?:32|64)\\Q-bit.tar.bz2\\E$"]

[glib]
url_patterns = ["^\\Qhttps://ftp.gnome.org/pub/gnome/sources/glib/\\E[^/]+\\Q/glib-\\E(\\d.*)\\Q.tar.xz\\E$"]
Expand Down Expand Up @@ -52,6 +62,10 @@ url_patterns = ["^\\Qhttps://github.com/llvm/llvm-project/releases/download/llvm
[mbedtls]
repositories = ["https://github.com/ARMmbed/mbedtls.git"]

[msgpack-c]
# Repology splits these release tarballs between msgpack and msgpack-c
url_patterns = ["^\\Qhttps://github.com/msgpack/msgpack-c/releases/download/c-\\E(\\d[^/]*)\\Q/msgpack-c-\\E\\1\\Q.tar.gz\\E$"]

[openspecfun]
repositories = ["https://github.com/JuliaMath/openspecfun.git"]
url_patterns = ["^\\Qhttps://github.com/JuliaMath/openspecfun/archive/v\\E(\\d.*)\\Q.tar.gz\\E$"]
Expand Down Expand Up @@ -104,3 +118,180 @@ url_patterns = ["^\\Qhttps://download.qt.io/official_releases/qt/\\E[^/]+/(.+)\\
repositories = ["https://github.com/JuliaStats/Rmath-julia.git"]
url_patterns = ["^\\Qhttps://github.com/JuliaStats/Rmath-julia/archive/v\\E(\\d.*)\\Q.tar.gz\\E$"]

[imagemagick]
# 7.1.2-1 is a patch release after 7.1.2
version_scheme = "libversion"
url_patterns = ["^\\Qhttps://github.com/ImageMagick/ImageMagick6/archive/\\E(6\\.[^/]*)\\Q.tar.gz\\E$"]
repositories = ["https://github.com/ImageMagick/ImageMagick", "https://github.com/ImageMagick/ImageMagick6"]
tag_patterns = [{regex = "^(\\d+\\.\\d+\\.\\d+-\\d+)$"}]

[openssh]
# 9.2p1 is a patch release after 9.2
version_flags = ["P_IS_PATCH"]

[giflib]
url_patterns = ["^\\Qhttps://sourceforge.net/projects/giflib/files/giflib-\\E[^/]+\\Q/giflib-\\E(\\d[^/]*)\\Q.tar.gz\\E$"]

[ncurses]
# GNU mirrors
url_patterns = ["^https?://[^/]+/(?:pub/)?gnu/ncurses/ncurses-(\\d[^/]*)\\.tar\\.gz$"]

[util-linux]
# Repology attributes some of these tarballs to the util-linux-libuuid subpackage
url_patterns = ["^\\Qhttps://mirrors.edge.kernel.org/pub/linux/utils/util-linux/v\\E[^/]+\\Q/util-linux-\\E(\\d[^/]*)\\Q.tar.xz\\E$"]

# Downstream subpackages built from these upstream tarballs (emacs:nxml-libvirt-schemas, ffmpeg-static-libs-for-sunshine,
# node-bokehjs) can be the only Repology packages still using a given release URL
[ffmpeg]
url_patterns = ["^\\Qhttps://ffmpeg.org/releases/ffmpeg-\\E(\\d[^/]*)\\Q.tar.\\E\\w+$"]

[libvirt]
url_patterns = ["^https?://(?:download\\.libvirt\\.org|libvirt\\.org/sources)/libvirt-(\\d[^/]*)\\Q.tar.\\E\\w+$"]

["python:bokeh"]
url_patterns = ["^\\Qhttps://files.pythonhosted.org/packages/source/b/bokeh/bokeh-\\E(\\d[^/]*)\\Q.tar.gz\\E$"]

[qtgraphs]
url_patterns = ["^\\Qhttps://download.qt.io/official_releases/qt/\\E[^/]+/(.+)\\Q/submodules/qtgraphs-\\E"]

# The CLI has its own Repology project; "bitwarden" is the desktop app
[bitwarden-cli]
url_patterns = ["^\\Qhttps://github.com/bitwarden/clients/releases/download/cli-v\\E(\\d[^/]*)\\Q/bw-\\E"]

# Repology splits PMIx between pmix and openpmix; the CPE is on openpmix
[openpmix]
url_patterns = ["^\\Qhttps://github.com/openpmix/openpmix/releases/download/v\\E(\\d[^/]*)\\Q/pmix-\\E\\1\\Q.tar.\\E\\w+$"]

# Downstream packages (ptxdist's rsync3, AOSC's libtiff-5) can be the only ones still using a given release URL
[rsync]
url_patterns = ["^\\Qhttps://download.samba.org/pub/rsync/src/rsync-\\E(\\d[^/]*)\\Q.tar.gz\\E$"]

[tiff]
url_patterns = ["^\\Qhttps://download.osgeo.org/libtiff/tiff-\\E(\\d[^/]*)\\Q.tar.\\E\\w+$"]

# python:quarto-cli is a PyPI wrapper
[quarto]
url_patterns = ["^\\Qhttps://github.com/quarto-dev/quarto-cli/releases/download/v\\E(\\d[^/]*)\\Q/quarto-\\E\\1-"]

# Repology maps these repositories to a single (or the wrong) project
[sdl2]
repositories = ["https://github.com/libsdl-org/SDL.git"]
tag_patterns = [{regex = "^(?:pre)?release-(2\\..+)$"}]

[sdl3]
repositories = ["https://github.com/libsdl-org/SDL.git"]
tag_patterns = [{regex = "^(?:pre)?release-(3\\..+)$"}]

[openpolicyagent]
repositories = ["https://github.com/open-policy-agent/opa.git"]

["python:apache-tvm-ffi"]
repositories = ["https://github.com/apache/tvm-ffi.git"]

# Release URLs that Repology's data doesn't cover
[qdrant]
url_patterns = ["^\\Qhttps://github.com/qdrant/qdrant/releases/download/v\\E(\\d[^/]*)\\Q/qdrant-\\E"]

[t8code]
url_patterns = ["^\\Qhttps://github.com/DLR-AMR/t8code/releases/download/v\\E(\\d[^/]*)\\Q/t8-\\E\\1\\Q.tar.gz\\E$"]

[wayland]
url_patterns = ["^\\Qhttps://wayland.freedesktop.org/releases/wayland-\\E(\\d[^/]*)\\Q.tar.\\E\\w+$"]

[libxcb]
url_patterns = ["^https?://(?:www\\.)?x\\.org/(?:archive/)?individual/xcb/libxcb-(\\d[^/]*)\\.tar\\.\\w+$"]

[xcb-proto]
url_patterns = ["^https?://(?:www\\.)?x\\.org/(?:archive/)?individual/(?:xcb|proto)/xcb-proto-(\\d[^/]*)\\.tar\\.\\w+$"]

[util-macros]
url_patterns = ["^https?://(?:www\\.)?x\\.org/(?:archive/)?individual/util/util-macros-(\\d[^/]*)\\.tar\\.\\w+$"]

[htop]
url_patterns = ["^https?://hisham\\.hm/htop/releases/[^/]+/htop-(\\d[^/]*)\\.tar\\.\\w+$"]

[opus]
url_patterns = ["^https?://archive\\.mozilla\\.org/pub/opus/opus-(\\d[^/]*)\\.tar\\.\\w+$"]

[eudev]
url_patterns = ["^\\Qhttps://dev.gentoo.org/~blueness/eudev/eudev-\\E(\\d[^/]*)\\Q.tar.gz\\E$"]

[sundials]
url_patterns = ["^https?://computation\\.llnl\\.gov/projects/sundials/download/sundials-(\\d[^/]*)\\.tar\\.gz$"]

[libxslt]
url_patterns = ["^ftp://xmlsoft\\.org/libxslt/libxslt-(\\d[^/]*)\\.tar\\.gz$"]

[grass-gis]
url_patterns = ["^\\Qhttps://grass.osgeo.org/grass\\E\\d+\\Q/source/grass-\\E(\\d[^/]*)\\Q.tar.gz\\E$"]

[ghostscript]
url_patterns = ["^\\Qhttps://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs\\E\\d+\\Q/ghostscript-\\E(\\d[^/]*)\\Q.tar.\\E\\w+$"]

[biome]
url_patterns = ["^\\Qhttps://github.com/biomejs/biome/archive/refs/tags/@biomejs/biome@\\E(\\d[^/]*)\\Q.tar.gz\\E$"]

[cudss]
url_patterns = ["^\\Qhttps://developer.download.nvidia.com/compute/cudss/redist/libcudss/\\E[^/]+\\Q/libcudss-\\E[^/]+?-(\\d+\\.\\d+\\.\\d+\\.\\d+)_cuda"]

# Git repositories and tag schemes that Repology's data doesn't (or no longer) cover
[nccl]
repositories = ["https://github.com/NVIDIA/nccl.git"]
tag_patterns = [{regex = "^v(\\d+\\.\\d+\\.\\d+-\\d+)$"}]

[minio]
tag_patterns = [{regex = "^RELEASE\\.(\\d{4}-\\d\\d-\\d\\dT\\d\\d-\\d\\d-\\d\\dZ)$"}]

["llama.cpp"]
repositories = ["https://github.com/ggerganov/llama.cpp.git", "https://github.com/ggml-org/llama.cpp.git"]
tag_patterns = [{regex = "^(b\\d+)$"}]

[p7zip]
repositories = ["https://github.com/p7zip-project/p7zip"]

[uasm]
repositories = ["https://github.com/Terraspace/UASM"]

[onnx]
repositories = ["https://github.com/onnx/onnx.git"]

[libfabric]
repositories = ["https://github.com/ofiwg/libfabric"]

[zxing-cpp]
repositories = ["https://github.com/zxing-cpp/zxing-cpp.git"]

[liquid-dsp]
repositories = ["https://github.com/jgaeddert/liquid-dsp.git"]

[sdl2-ttf]
repositories = ["https://github.com/libsdl-org/SDL_ttf.git"]
tag_patterns = [{regex = "^(?:pre)?release-(2\\..+)$"}]

[sdl3-ttf]
repositories = ["https://github.com/libsdl-org/SDL_ttf.git"]
tag_patterns = [{regex = "^(?:pre)?release-(3\\..+)$"}]

["rust:regex"]
repositories = ["https://github.com/rust-lang/regex.git"]

[wayland-protocols]
url_patterns = ["^\\Qhttps://wayland.freedesktop.org/releases/wayland-protocols-\\E(\\d[^/]*)\\Q.tar.\\E\\w+$"]

[portaudio]
url_patterns = ["^https?://(?:www\\.)?portaudio\\.com/archives/pa_stable_v\\d+_\\d+\\.tgz()$"] # Repology spells these 190600_20161030, which isn't comparable

[cutensor]
url_patterns = ["^\\Qhttps://developer.download.nvidia.com/compute/cutensor/redist/libcutensor/\\E[^/]+\\Q/libcutensor-\\E[^/]+?-(\\d+\\.\\d+\\.\\d+\\.\\d+)(?:_cuda\\d+)?-archive"]

[crosstool-ng]
url_patterns = ["^https?://crosstool-ng\\.org/download/crosstool-ng/crosstool-ng-(\\d[^/]*)\\.tar\\.\\w+$"]

[opencl-clang]
tag_patterns = [{regex = "^v(\\d+\\.\\d+\\.\\d+-\\d+)$"}]

[spirv-llvm-translator]
tag_patterns = [{regex = "^v(\\d+\\.\\d+\\.\\d+-\\d+)$"}]

[win32-openssh]
tag_patterns = [{regex = "^[vV]?(\\d+\\.\\d+\\.\\d+\\.\\d+p\\d+-[A-Za-z]+)$"}]
Loading
Loading