Support nested version arrays in TOML plugins

tools/internal-schema/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## Unreleased
+
+#### 🚀 Updates
+
+- Added `manifest-array-key` (format v1) and `index_array_key` (format v2) to load versions from a named array in a JSON index object.
+
 ## 0.19.0
 
 #### 🚀 Updates

tools/internal-schema/src/proto.rs

@@ -175,6 +175,31 @@ fn create_version(cap: Captures) -> String {
     version
 }
 
+fn select_index_rows(response: JsonValue, array_key: Option<&str>) -> FnResult<Vec<JsonValue>> {
+    match response {
+        JsonValue::Array(rows) => Ok(rows),
+        JsonValue::Object(mut object) => {
+            let key = array_key.ok_or_else(|| {
+                plugin_err!(
+                    "Version index returned an object; configure manifest-array-key (v1) or index_array_key (v2)."
+                )
+            })?;
+
+            match object.remove(key) {
+                Some(JsonValue::Array(rows)) => Ok(rows),
+                Some(_) => Err(plugin_err!(
+                    "Version index key '{}' must contain an array.",
+                    key
+                )),
+                None => Err(plugin_err!("Version index key '{}' was not found.", key)),
+            }
+        }
+        _ => Err(plugin_err!(
+            "Version index must return an array or an object containing an array."
+        )),
+    }
+}
+
 #[plugin_fn]
 pub fn detect_version_files(
     Json(_): Json<DetectVersionInput>,
@@ -211,9 +236,10 @@ pub fn load_versions(Json(input): Json<LoadVersionsInput>) -> FnResult<Json<Load
     else if let Some(endpoint) = resolve.index_url.as_deref() {
         let pattern = regex::Regex::new(resolve.get_version_pattern())?;
         let version_key = resolve.get_index_version_key();
-        let response: Vec<JsonValue> = fetch_json(endpoint)?;
+        let response: JsonValue = fetch_json(endpoint)?;
+        let rows = select_index_rows(response, resolve.index_array_key.as_deref())?;
 
-        for row in response {
+        for row in rows {
             match row {
                 JsonValue::String(v) => {
                     if let Some(cap) = pattern.captures(&v) {
@@ -253,6 +279,59 @@ pub fn load_versions(Json(input): Json<LoadVersionsInput>) -> FnResult<Json<Load
     Ok(Json(output))
 }
 
+#[cfg(test)]
+mod index_tests {
+    use super::*;
+    use serde_json::json;
+
+    #[test]
+    fn accepts_top_level_arrays_with_or_without_a_configured_key() {
+        let rows = json!(["1.2.3", { "version": "2.0.0" }]);
+
+        let expected = rows.as_array().unwrap().clone();
+
+        assert_eq!(select_index_rows(rows.clone(), None).unwrap(), expected);
+        assert_eq!(select_index_rows(rows, Some("versions")).unwrap(), expected);
+    }
+
+    #[test]
+    fn selects_nested_strings_and_objects() {
+        let rows = json!(["1.2.3", { "version": "2.0.0" }]);
+        let response = json!({ "versions": rows, "latest": "2.0.0" });
+
+        assert_eq!(
+            select_index_rows(response, Some("versions")).unwrap(),
+            rows.as_array().unwrap().clone()
+        );
+    }
+
+    #[test]
+    fn rejects_missing_or_invalid_nested_arrays() {
+        let missing = select_index_rows(json!({ "other": [] }), Some("versions"))
+            .unwrap_err()
+            .0
+            .to_string();
+        assert!(missing.contains("'versions' was not found"), "{missing}");
+
+        let wrong_type = select_index_rows(json!({ "versions": "1.2.3" }), Some("versions"))
+            .unwrap_err()
+            .0
+            .to_string();
+        assert!(
+            wrong_type.contains("'versions' must contain an array"),
+            "{wrong_type}"
+        );
+
+        let no_key = select_index_rows(json!({ "versions": [] }), None)
+            .unwrap_err()
+            .0
+            .to_string();
+        assert!(no_key.contains("configure manifest-array-key"), "{no_key}");
+
+        assert!(select_index_rows(json!("1.2.3"), None).is_err());
+    }
+}
+
 #[plugin_fn]
 pub fn download_prebuilt(
     Json(input): Json<DownloadPrebuiltInput>,

tools/internal-schema/src/schema/mod.rs

@@ -105,6 +105,7 @@ impl Schema {
             Self::V1(inner) => v2::ResolveSchema {
                 version_pattern: Some(inner.resolve.version_pattern.clone()),
                 index_url: inner.resolve.manifest_url.clone(),
+                index_array_key: inner.resolve.manifest_array_key.clone(),
                 index_version_key: Some(inner.resolve.manifest_version_key.clone()),
                 git_url: inner.resolve.git_url.clone(),
                 git_tag_pattern: inner.resolve.git_tag_pattern.clone(),
@@ -198,3 +199,70 @@ pub fn interpolate_tokens(
 
     value
 }
+
+#[cfg(test)]
+mod index_key_tests {
+    use super::*;
+    use serde_json::json;
+
+    #[test]
+    fn maps_v1_manifest_array_key_to_shared_resolver() {
+        let v1 = serde_json::from_value(json!({
+            "name": "test",
+            "resolve": {
+                "manifest-url": "https://example.com/versions.json",
+                "manifest-array-key": "versions"
+            }
+        }))
+        .unwrap();
+        let resolve = Schema::V1(v1).get_resolve(&UnresolvedVersionSpec::parse("1.0.0").unwrap());
+
+        assert_eq!(resolve.index_array_key.as_deref(), Some("versions"));
+        assert_eq!(resolve.get_index_version_key(), "version");
+    }
+
+    #[test]
+    fn v2_overrides_can_change_index_array_key() {
+        let mut resolve: v2::ResolveSchema = serde_json::from_value(json!({
+            "index_url": "https://example.com/stable.json",
+            "index_array_key": "releases"
+        }))
+        .unwrap();
+        let next = serde_json::from_value(json!({
+            "index_url": "https://example.com/nightly.json",
+            "index_array_key": "versions"
+        }))
+        .unwrap();
+
+        resolve.override_with(&next);
+
+        assert_eq!(
+            resolve.index_url.as_deref(),
+            Some("https://example.com/nightly.json")
+        );
+        assert_eq!(resolve.index_array_key.as_deref(), Some("versions"));
+    }
+
+    #[test]
+    fn v2_url_override_without_array_key_keeps_base_key() {
+        let mut resolve: v2::ResolveSchema = serde_json::from_value(json!({
+            "index_url": "https://example.com/stable.json",
+            "index_array_key": "releases",
+            "index_version_key": "tag"
+        }))
+        .unwrap();
+        let next = serde_json::from_value(json!({
+            "index_url": "https://example.com/nightly.json"
+        }))
+        .unwrap();
+
+        resolve.override_with(&next);
+
+        assert_eq!(
+            resolve.index_url.as_deref(),
+            Some("https://example.com/nightly.json")
+        );
+        assert_eq!(resolve.index_array_key.as_deref(), Some("releases"));
+        assert_eq!(resolve.get_index_version_key(), "tag");
+    }
+}

tools/internal-schema/src/schema/v1.rs

@@ -112,6 +112,7 @@ pub struct ResolveSchema {
     pub version_pattern: String,
     // Manifest
     pub manifest_url: Option<String>,
+    pub manifest_array_key: Option<String>,
     pub manifest_version_key: String,
     // Tags
     pub git_url: Option<String>,
@@ -123,6 +124,7 @@ impl Default for ResolveSchema {
         ResolveSchema {
             aliases: HashMap::new(),
             manifest_url: None,
+            manifest_array_key: None,
             manifest_version_key: "version".to_string(),
             git_url: None,
             git_tag_pattern: None,

tools/internal-schema/src/schema/v2.rs

@@ -21,6 +21,7 @@ pub struct ResolveSchema {
     pub version_pattern: Option<String>,
     // Manifest
     pub index_url: Option<String>,
+    pub index_array_key: Option<String>,
     pub index_version_key: Option<String>,
     // Tags
     pub git_url: Option<String>,
@@ -57,6 +58,10 @@ impl ResolveSchema {
             self.index_version_key = Some(value.to_owned());
         }
 
+        if let Some(value) = &other.index_array_key {
+            self.index_array_key = Some(value.to_owned());
+        }
+
         if let Some(value) = &other.version_pattern {
             self.version_pattern = Some(value.to_owned());
         }

tools/internal-schema/tests/versions_test.rs

@@ -1,9 +1,99 @@
 use proto_pdk_test_utils::*;
 use starbase_sandbox::locate_fixture;
+use std::io::{Read, Write};
+use std::net::TcpListener;
+use std::thread;
+use std::time::{Duration, Instant};
+
+fn bind_json() -> (String, TcpListener) {
+    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+    let address = listener.local_addr().unwrap();
+
+    (format!("http://{address}/versions.json"), listener)
+}
+
+fn serve_json(listener: TcpListener, body: &'static str) -> thread::JoinHandle<bool> {
+    listener.set_nonblocking(true).unwrap();
+
+    thread::spawn(move || {
+        let deadline = Instant::now() + Duration::from_secs(10);
+
+        loop {
+            match listener.accept() {
+                Ok((mut stream, _)) => {
+                    stream
+                        .set_read_timeout(Some(Duration::from_secs(5)))
+                        .unwrap();
+                    let mut request = [0; 1024];
+                    assert!(stream.read(&mut request).unwrap() > 0);
+                    write!(
+                        stream,
+                        "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
+                        body.len()
+                    )
+                    .unwrap();
+
+                    return true;
+                }
+                Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
+                    if Instant::now() >= deadline {
+                        return false;
+                    }
+                    thread::sleep(Duration::from_millis(10));
+                }
+                Err(error) => panic!("Failed to accept index request: {error}"),
+            }
+        }
+    })
+}
 
 mod schema_tool {
     use super::*;
 
+    #[tokio::test(flavor = "multi_thread")]
+    async fn loads_nested_index_versions_in_both_schema_formats() {
+        for format in [1, 2] {
+            let (endpoint, listener) = bind_json();
+            let sandbox = create_empty_proto_sandbox();
+            let schema = if format == 1 {
+                format!(
+                    "name = \"schema-test\"\ntype = \"cli\"\n\n[resolve]\nmanifest-url = \"{endpoint}\"\nmanifest-array-key = \"releases\"\nmanifest-version-key = \"tag\"\n"
+                )
+            } else {
+                format!(
+                    "format = \"2\"\n\n[metadata]\nname = \"schema-test\"\ntype = \"command-line\"\n\n[resolve]\nindex_url = \"{endpoint}\"\nindex_array_key = \"releases\"\nindex_version_key = \"tag\"\n"
+                )
+            };
+            sandbox.create_file("index.toml", &schema);
+            let plugin = sandbox
+                .create_schema_plugin("schema-test", sandbox.path().join("index.toml"))
+                .await;
+
+            let server = serve_json(
+                listener,
+                r#"{"releases":[{"tag":"1.2.3"},{"tag":"2.3.4"}]}"#,
+            );
+            let output = plugin.load_versions(LoadVersionsInput::default()).await;
+            assert!(
+                server.join().unwrap(),
+                "format {format} did not request the index"
+            );
+
+            assert!(
+                output
+                    .versions
+                    .contains(&VersionSpec::parse("1.2.3").unwrap()),
+                "format {format} failed to load 1.2.3"
+            );
+            assert!(
+                output
+                    .versions
+                    .contains(&VersionSpec::parse("2.3.4").unwrap()),
+                "format {format} failed to load 2.3.4"
+            );
+        }
+    }
+
     generate_resolve_versions_tests!(
         "schema-test",
         {