Add "skipped-versions" index.json field for the ability to skip testing of broken library versions (#792)

Author: jormundur00

docs/CONTRIBUTING.md

@@ -153,6 +153,34 @@ a regexp (Java format) matching the version pattern. For example, for the exampl
 }
 ```
 
+The optional `skipped-versions` key can be used to explicitly exclude specific
+library versions from being considered as supported metadata. A skipped version
+should include both the version and the reason it is excluded. This is useful
+when a version exists in Maven but is known to be broken, incompatible, or
+should be omitted from testing for any other reason. An example of the library
+above using this field:
+
+```json
+{
+  "metadata-version": "1.0.0",
+  "module": "org.example:library",
+  "tested-versions": [
+    "1.0.0",
+    "1.1.0"
+  ],
+  "skipped-versions": [
+    {
+      "version": "1.0.5",
+      "reason": "Known incompatible API change."
+    },
+    {
+      "version": "1.0.7",
+      "reason": "Integrated reflect-config.json does not parse."
+    }
+  ]
+}
+```
+
 You can also list each supported version is listed in `tested-versions`, as that value is used in build tools to match
 metadata to a specific library, but this is more likely to break when new versions are released.
 Every metadata for a specific library version has a `index.json`. For this

metadata/io.opentelemetry/opentelemetry-sdk-metrics/index.json

@@ -23,6 +23,12 @@
       "1.31.0",
       "1.32.0",
       "1.33.0"
+    ],
+    "skipped-versions": [
+      {
+        "version": "1.34.0",
+        "reason": "Dependency io.opentelemetry:opentelemetry-api:1.34.0 provides reflect-config.json which does not parse."
+      }
     ]
   }
 ]

metadata/io.opentelemetry/opentelemetry-sdk-trace/index.json

@@ -23,6 +23,12 @@
       "1.31.0",
       "1.32.0",
       "1.33.0"
+    ],
+    "skipped-versions": [
+      {
+        "version": "1.34.0",
+        "reason": "Dependency io.opentelemetry:opentelemetry-api:1.34.0 provides reflect-config.json which does not parse."
+      }
     ]
   }
 ]

tests/tck-build-logic/src/main/groovy/org/graalvm/internal/tck/harness/tasks/FetchExistingLibrariesWithNewerVersionsTask.groovy

@@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.databind.SerializationFeature
 import groovy.json.JsonOutput
 import org.graalvm.internal.tck.model.MetadataVersionsIndexEntry
+import org.graalvm.internal.tck.model.SkippedVersionEntry
 import org.gradle.api.DefaultTask
 import org.gradle.api.provider.ListProperty
 import org.gradle.api.tasks.Input
@@ -59,6 +60,11 @@ abstract class FetchExistingLibrariesWithNewerVersionsTask extends DefaultTask {
             String libraryName = it
             if (INFRASTRUCTURE_TESTS.stream().noneMatch(testName -> libraryName.startsWith(testName))) {
                 List<String> versions = getNewerVersionsFor(libraryName, getLatestLibraryVersion(libraryName))
+                List<String> skipped = getSkippedVersions(libraryName)
+
+                // filter out skipped versions
+                versions = versions.findAll { !skipped.contains(it) }
+
                 versions.forEach {
                     newerVersions.add(libraryName.concat(":").concat(it))
                 }
@@ -164,4 +170,45 @@ abstract class FetchExistingLibrariesWithNewerVersionsTask extends DefaultTask {
         }
     }
 
+    /**
+     * Returns all versions of a given library that are marked as skipped in the
+     * metadata index.
+     * <p>
+     * For the provided Maven coordinates (in the format {@code <groupId>:<artifactId>}),
+     * this method reads the corresponding {@code index.json} file located under:
+     * {@code metadata/<groupId>/<artifactId>/index.json}
+     * and collects all version entries listed under {@code skipped-versions}.
+     */
+    static List<String> getSkippedVersions(String libraryModule) {
+        try {
+            String[] coordinates = libraryModule.split(":");
+            String group = coordinates[0];
+            String artifact = coordinates[1];
+
+            File coordinatesMetadataIndex = new File("metadata/" + group + "/" + artifact + "/index.json");
+            ObjectMapper objectMapper = new ObjectMapper()
+                    .enable(SerializationFeature.INDENT_OUTPUT)
+                    .setSerializationInclusion(JsonInclude.Include.NON_NULL);
+
+            List<MetadataVersionsIndexEntry> entries = objectMapper.readValue(
+                    coordinatesMetadataIndex,
+                    new TypeReference<List<MetadataVersionsIndexEntry>>() {}
+            );
+
+            List<String> skipped = new ArrayList<>();
+            for (MetadataVersionsIndexEntry entry : entries) {
+                if (entry.skippedVersions() != null) {
+                    skipped.addAll(
+                            entry.skippedVersions().stream()
+                                    .map(SkippedVersionEntry::version)
+                                    .toList()
+                    );
+                }
+            }
+
+            return skipped;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
 }

tests/tck-build-logic/src/main/java/org/graalvm/internal/tck/ScaffoldTask.java

@@ -269,12 +269,7 @@ private void updateCoordinatesMetadataRootJson(Path metadataRoot, Coordinates co
         List<MetadataVersionsIndexEntry> entries = objectMapper.readValue(metadataIndex, new TypeReference<>() {});
 
         // add new entry
-        MetadataVersionsIndexEntry newEntry = new MetadataVersionsIndexEntry(null,
-                null,
-                coordinates.group() + ":" + coordinates.artifact(),
-                null,
-                coordinates.version(),
-                List.of(coordinates.version()));
+        MetadataVersionsIndexEntry newEntry = new MetadataVersionsIndexEntry(null, null, coordinates.group() + ":" + coordinates.artifact(), null, coordinates.version(), List.of(coordinates.version()), null);
 
         entries.add(newEntry);
 
@@ -307,7 +302,7 @@ private void updateCoordinatesMetadataRootJson(Path metadataRoot, Coordinates co
 
     private void setLatest( List<MetadataVersionsIndexEntry> list, int index, Boolean newValue) {
         MetadataVersionsIndexEntry oldEntry = list.remove(index);
-        list.add(new MetadataVersionsIndexEntry(newValue, oldEntry.override(), oldEntry.module(), oldEntry.defaultFor(), oldEntry.metadataVersion(), oldEntry.testedVersions()));
+        list.add(new MetadataVersionsIndexEntry(newValue, oldEntry.override(), oldEntry.module(), oldEntry.defaultFor(), oldEntry.metadataVersion(), oldEntry.testedVersions(), oldEntry.skippedVersions()));
     }
 
     private String getEmptyJsonArray() {

tests/tck-build-logic/src/main/java/org/graalvm/internal/tck/TestedVersionUpdaterTask.java

@@ -162,7 +162,8 @@ private MetadataVersionsIndexEntry handlePreReleases(MetadataVersionsIndexEntry
                         entry.module(),
                         entry.defaultFor(),
                         newVersion,
-                        entry.testedVersions()
+                        entry.testedVersions(),
+                        entry.skippedVersions()
                 );
             }
         }

tests/tck-build-logic/src/main/java/org/graalvm/internal/tck/model/MetadataVersionsIndexEntry.java

@@ -13,5 +13,7 @@ public record MetadataVersionsIndexEntry (
         @JsonProperty("metadata-version")
         String metadataVersion,
         @JsonProperty("tested-versions")
-        List<String> testedVersions
+        List<String> testedVersions,
+        @JsonProperty("skipped-versions")
+        List<SkippedVersionEntry> skippedVersions
 ){}

tests/tck-build-logic/src/main/java/org/graalvm/internal/tck/model/SkippedVersionEntry.java

@@ -0,0 +1,10 @@
+package org.graalvm.internal.tck.model;
+
+/**
+ * Represents a version of a library that should be skipped during testing,
+ * along with the reason why this version is skipped.
+ */
+public record SkippedVersionEntry(
+        String version,
+        String reason
+){}