feat(providers): add profile-backed policy composition (#1037)

Foundation for providers v2. Add provider profiles and provider profile composition with user policies.

Author: johntmyers

.agents/skills/debug-openshell-cluster/SKILL.md

@@ -187,8 +187,11 @@ Component images (server, sandbox) can reach kubelet via two paths:
 Gateway and cluster image builds consume Rust binaries staged at `deploy/docker/.build/prebuilt-binaries/<arch>/`. In CI these come from the reusable Rust native build workflow; locally `tasks/scripts/docker-build-image.sh` runs `tasks/scripts/stage-prebuilt-binaries.sh` before invoking Docker unless `PREBUILT_AUTO_STAGE=0` is set.
 
 ```bash
-# Verify image refs currently used by openshell deployment
-openshell doctor exec -- kubectl -n openshell get statefulset openshell -o jsonpath="{.spec.template.spec.containers[*].image}"
+# Verify image refs currently used by openshell deployment.
+# The gateway image and server.supervisorImage should use the same build tag
+# in branch/E2E deploys; a stale supervisor image can make sandbox behavior
+# lag behind gateway policy/proto changes.
+openshell doctor exec -- kubectl -n openshell get statefulset openshell -o jsonpath="{.spec.template.spec.containers[*].image}{\"\n\"}{.spec.template.spec.containers[*].env[?(@.name==\"OPENSHELL_SUPERVISOR_IMAGE\")].value}{\"\n\"}"
 
 # Verify registry mirror/auth endpoint configuration
 openshell doctor exec -- cat /etc/rancher/k3s/registries.yaml

.github/workflows/branch-e2e.yml

@@ -46,8 +46,19 @@ jobs:
       component: cluster
       platform: linux/arm64
 
+  build-supervisor:
+    needs: [pr_metadata]
+    if: needs.pr_metadata.outputs.should_run == 'true'
+    permissions:
+      contents: read
+      packages: write
+    uses: ./.github/workflows/docker-build.yml
+    with:
+      component: supervisor
+      platform: linux/arm64
+
   e2e:
-    needs: [pr_metadata, build-gateway, build-cluster]
+    needs: [pr_metadata, build-gateway, build-cluster, build-supervisor]
     if: needs.pr_metadata.outputs.should_run == 'true'
     permissions:
       contents: read

.github/workflows/shadow-branch-e2e.yml

@@ -58,8 +58,21 @@ jobs:
       image-tag: shadow-branch-e2e-${{ github.sha }}
     secrets: inherit
 
+  build-supervisor:
+    needs: [pr_metadata]
+    if: needs.pr_metadata.outputs.should_run == 'true'
+    permissions:
+      contents: read
+      packages: write
+    uses: ./.github/workflows/docker-build.yml
+    with:
+      component: supervisor
+      platform: linux/arm64
+      image-tag: shadow-branch-e2e-${{ github.sha }}
+    secrets: inherit
+
   e2e:
-    needs: [pr_metadata, build-gateway, build-cluster]
+    needs: [pr_metadata, build-gateway, build-cluster, build-supervisor]
     if: needs.pr_metadata.outputs.should_run == 'true'
     permissions:
       contents: read

Cargo.lock

@@ -30,9 +30,8 @@ The `REGISTERED_SETTINGS` static array defines the allowed setting keys and thei
 
 ```rust
 pub const REGISTERED_SETTINGS: &[RegisteredSetting] = &[
-    RegisteredSetting { key: "log_level", kind: SettingValueKind::String },
-    RegisteredSetting { key: "dummy_int", kind: SettingValueKind::Int },
-    RegisteredSetting { key: "dummy_bool", kind: SettingValueKind::Bool },
+    RegisteredSetting { key: "providers_v2_enabled", kind: SettingValueKind::Bool },
+    RegisteredSetting { key: "ocsf_json_enabled", kind: SettingValueKind::Bool },
 ];
 ```
 
@@ -373,15 +372,14 @@ Set a single setting key at sandbox or global scope.
 
 ```bash
 # Sandbox-scoped
-openshell settings set my-sandbox --key log_level --value debug
+openshell settings set my-sandbox --key ocsf_json_enabled --value true
 
 # Global (requires confirmation)
-openshell settings set --global --key log_level --value warn
-openshell settings set --global --key dummy_bool --value yes
-openshell settings set --global --key dummy_int --value 42
+openshell settings set --global --key providers_v2_enabled --value true
+openshell settings set --global --key ocsf_json_enabled --value true
 
 # Skip confirmation
-openshell settings set --global --key log_level --value info --yes
+openshell settings set --global --key providers_v2_enabled --value true --yes
 ```
 
 Value parsing is type-aware: bool keys accept `true/false/yes/no/1/0/on/off` via `parse_bool_like()`. Int keys parse as base-10 `i64`. String keys accept any value.
@@ -392,7 +390,7 @@ Delete a setting key from the specified scope.
 
 ```bash
 # Global delete (unlocks sandbox control)
-openshell settings delete --global --key log_level --yes
+openshell settings delete --global --key providers_v2_enabled --yes
 ```
 
 ### `policy set --global --policy FILE [--yes]`
@@ -502,26 +500,26 @@ Settings are refreshed on each 2-second polling tick alongside the sandbox list
 
 ## Data Flow: Setting a Global Key
 
-End-to-end trace for `openshell settings set --global --key log_level --value debug --yes`:
+End-to-end trace for `openshell settings set --global --key providers_v2_enabled --value true --yes`:
 
 1. **CLI** (`crates/openshell-cli/src/run.rs` -- `gateway_setting_set()`):
-   - `parse_cli_setting_value("log_level", "debug")` -- looks up `SettingValueKind::String` in the registry, wraps as `SettingValue { string_value: "debug" }`
+   - `parse_cli_setting_value("providers_v2_enabled", "true")` -- looks up `SettingValueKind::Bool` in the registry, wraps as `SettingValue { bool_value: true }`
    - `confirm_global_setting_takeover()` -- skipped because `--yes`
-   - Sends `UpdateSettingsRequest { setting_key: "log_level", setting_value: Some(...), global: true }`
+   - Sends `UpdateSettingsRequest { setting_key: "providers_v2_enabled", setting_value: Some(...), global: true }`
 
 2. **Gateway** (`crates/openshell-server/src/grpc.rs` -- `update_settings()`):
    - Acquires `settings_mutex` for the duration of the operation
    - Detects `global=true`, `has_setting=true`
-   - `validate_registered_setting_key("log_level")` -- passes (key is in registry)
+   - `validate_registered_setting_key("providers_v2_enabled")` -- passes (key is in registry)
    - `load_global_settings()` -- reads `gateway_settings` record from store
-   - `proto_setting_to_stored()` -- converts proto value to `StoredSettingValue::String("debug")`
+   - `proto_setting_to_stored()` -- converts proto value to `StoredSettingValue::Bool(true)`
    - `upsert_setting_value()` -- inserts into `BTreeMap`, returns `true` (changed)
    - Increments `revision`, calls `save_global_settings()`
    - Returns `UpdateSettingsResponse { settings_revision: N }`
 
 3. **Sandbox** (next poll tick in `run_policy_poll_loop()`):
    - `poll_settings(sandbox_id)` returns new `config_revision`
-   - `log_setting_changes()` logs: `Setting changed key="log_level" old="<unset>" new="debug"`
+   - `log_setting_changes()` logs: `Setting changed key="providers_v2_enabled" old="<unset>" new="true"`
    - `policy_hash` unchanged -- no OPA reload
    - Updates tracked `current_config_revision` and `current_settings`
 

architecture/gateway-settings.md

@@ -41,13 +41,67 @@ The gRPC surface is defined in `proto/openshell.proto`:
 - `CreateProvider`
 - `GetProvider`
 - `ListProviders`
+- `ListProviderProfiles`
+- `GetProviderProfile`
 - `UpdateProvider`
 - `DeleteProvider`
 
+## Provider Type Profiles
+
+Provider type profiles are declarative metadata for provider types. Built-in profiles
+live as one YAML document per provider under the top-level `providers/` directory
+and are exposed through
+`ListProviderProfiles` and `GetProviderProfile`. The profile loader validates the
+YAML catalog and materializes the same proto-backed shape that future API imports
+will accept. Profiles describe credential names and environment variables, known
+network endpoints, expected binaries, category, and whether the provider is
+inference-capable. Categories are a proto enum so clients can group and filter
+provider types without parsing display strings. Current values are `other`,
+`inference`, `agent`, `source_control`, `messaging`, `data`, and `knowledge`.
+Agent profiles such as `claude`, `codex`, and `opencode` can still be
+inference-capable when their tool talks to an inference API.
+
+Profiles are additive to provider records. A provider record with only `type`,
+`credentials`, and `config` can be matched to built-in profile metadata by
+`provider.type`. Profile-generated policy is still opt-in: the gateway composes provider
+profile rules only when the gateway-global `providers_v2_enabled` setting is true.
+
+This keeps the compatibility boundary at the gateway. A gateway without
+`providers_v2_enabled=true` keeps the existing credential-only provider behavior, while a
+gateway with the flag enabled routes all attached known provider types through the
+profile-backed policy path.
+
+### Provider Policy Composition
+
+Sandbox policy fetch uses just-in-time composition:
+
+```text
+effective policy = base/static policy + provider profile rules + user rules
+```
+
+The composed policy is derived data. The sandbox still receives one normal
+`SandboxPolicy`, but provider-generated entries are not persisted as user-authored
+policy revisions. Full policy replacement and incremental policy updates continue to
+mutate the user-authored policy layer. Provider-generated rules are re-added during
+composition for each attached provider whose type has a built-in profile.
+
+Provider-generated network rules use reserved `_provider_*` names derived from the
+provider record name. If a user or global policy already has the same key, composition
+keeps the policy entry and adds a numeric suffix to the provider entry. Duplicate
+host/port endpoints across policy and provider rules are valid; OPA evaluates all
+rules, so allow decisions are the union of matching allows and deny rules continue to
+win globally.
+
+Gateway-global policy still overrides sandbox-authored policy. When `providers_v2_enabled`
+is true, provider layers compose JIT onto the effective policy source, whether that
+source is sandbox-scoped or global. The composed payload is derived data and is not
+persisted as a policy revision.
+
 ## Components
 
 - `crates/openshell-providers`
   - canonical provider type normalization and command detection,
+  - YAML-backed built-in provider profiles,
   - provider registry and per-provider discovery plugins,
   - shared discovery engine and context abstraction for testability.
 - `crates/openshell-cli`
@@ -174,6 +228,7 @@ Also supported:
 
 - `openshell provider get <name>`
 - `openshell provider list`
+- `openshell provider list-profiles`
 - `openshell provider update <name> ...`
 - `openshell provider delete <name> [<name>...]`
 
@@ -232,6 +287,8 @@ Key behaviors:
 - Only `credentials` are injected, not `config`.
 - Invalid env var keys (containing `.`, `-`, spaces, etc.) are skipped.
 - Credentials are never persisted in the sandbox spec's environment map.
+- Provider profiles do not change credential injection in the first iteration.
+  Injection still uses the existing placeholder environment path.
 
 ### Sandbox Supervisor: Fetching Credentials
 

architecture/sandbox-providers.md

@@ -299,11 +299,10 @@ const POLICY_EXAMPLES: &str = "\x1b[1mALIAS\x1b[0m
 const SETTINGS_EXAMPLES: &str = "\x1b[1mEXAMPLES\x1b[0m
   $ openshell settings get my-sandbox
   $ openshell settings get --global
-  $ openshell settings set my-sandbox --key log_level --value debug
-  $ openshell settings set --global --key log_level --value warn
-  $ openshell settings set --global --key dummy_bool --value yes
-  $ openshell settings set --global --key dummy_int --value 42
-  $ openshell settings delete --global --key log_level
+  $ openshell settings set --global --key providers_v2_enabled --value true
+  $ openshell settings set my-sandbox --key ocsf_json_enabled --value true
+  $ openshell settings set --global --key ocsf_json_enabled --value true
+  $ openshell settings delete --global --key providers_v2_enabled
 ";
 
 const PROVIDER_EXAMPLES: &str = "\x1b[1mEXAMPLES\x1b[0m
@@ -735,6 +734,10 @@ enum ProviderCommands {
         names: bool,
     },
 
+    /// List available provider profiles.
+    #[command(help_template = LEAF_HELP_TEMPLATE, next_help_heading = "FLAGS")]
+    ListProfiles,
+
     /// Update an existing provider's credentials or config.
     #[command(help_template = LEAF_HELP_TEMPLATE, next_help_heading = "FLAGS")]
     Update {
@@ -2764,6 +2767,9 @@ async fn main() -> Result<()> {
                 } => {
                     run::provider_list(endpoint, limit, offset, names, &tls).await?;
                 }
+                ProviderCommands::ListProfiles => {
+                    run::provider_list_profiles(endpoint, &tls).await?;
+                }
                 ProviderCommands::Update {
                     name,
                     from_existing,
@@ -3454,6 +3460,19 @@ mod tests {
         }
     }
 
+    #[test]
+    fn provider_list_profiles_parses() {
+        let cli = Cli::try_parse_from(["openshell", "provider", "list-profiles"])
+            .expect("provider list-profiles should parse");
+
+        assert!(matches!(
+            cli.command,
+            Some(Commands::Provider {
+                command: Some(ProviderCommands::ListProfiles)
+            })
+        ));
+    }
+
     #[test]
     fn settings_set_global_parses_yes_flag() {
         let cli = Cli::try_parse_from([

crates/openshell-cli/src/main.rs

@@ -22,16 +22,18 @@ use openshell_bootstrap::{
     get_gateway_metadata, list_gateways, load_active_gateway, remove_gateway_metadata,
     resolve_ssh_hostname, save_active_gateway, save_last_sandbox, store_gateway_metadata,
 };
+use openshell_core::proto::ProviderProfileCategory;
 use openshell_core::proto::{
     ApproveAllDraftChunksRequest, ApproveDraftChunkRequest, ClearDraftChunksRequest,
     CreateProviderRequest, CreateSandboxRequest, DeleteProviderRequest, DeleteSandboxRequest,
     ExecSandboxRequest, GetClusterInferenceRequest, GetDraftHistoryRequest, GetDraftPolicyRequest,
     GetGatewayConfigRequest, GetProviderRequest, GetSandboxConfigRequest, GetSandboxLogsRequest,
-    GetSandboxPolicyStatusRequest, GetSandboxRequest, HealthRequest, ListProvidersRequest,
-    ListSandboxPoliciesRequest, ListSandboxesRequest, PolicySource, PolicyStatus, Provider,
-    RejectDraftChunkRequest, Sandbox, SandboxPhase, SandboxPolicy, SandboxSpec, SandboxTemplate,
-    SetClusterInferenceRequest, SettingScope, SettingValue, UpdateConfigRequest,
-    UpdateProviderRequest, WatchSandboxRequest, exec_sandbox_event, setting_value,
+    GetSandboxPolicyStatusRequest, GetSandboxRequest, HealthRequest, ListProviderProfilesRequest,
+    ListProvidersRequest, ListSandboxPoliciesRequest, ListSandboxesRequest, PolicySource,
+    PolicyStatus, Provider, ProviderProfile, RejectDraftChunkRequest, Sandbox, SandboxPhase,
+    SandboxPolicy, SandboxSpec, SandboxTemplate, SetClusterInferenceRequest, SettingScope,
+    SettingValue, UpdateConfigRequest, UpdateProviderRequest, WatchSandboxRequest,
+    exec_sandbox_event, setting_value,
 };
 use openshell_core::settings::{self, SettingValueKind};
 use openshell_core::{ObjectId, ObjectName};
@@ -3851,7 +3853,7 @@ pub async fn provider_create(
                     created_at_ms: 0,
                     labels: HashMap::new(),
                 }),
-                r#type: provider_type,
+                r#type: provider_type.clone(),
                 credentials: credential_map,
                 config: config_map,
             }),
@@ -3978,6 +3980,68 @@ pub async fn provider_list(
     Ok(())
 }
 
+pub async fn provider_list_profiles(server: &str, tls: &TlsOptions) -> Result<()> {
+    let mut client = grpc_client(server, tls).await?;
+    let response = client
+        .list_provider_profiles(ListProviderProfilesRequest {
+            limit: 100,
+            offset: 0,
+        })
+        .await
+        .into_diagnostic()?;
+    let mut profiles = response.into_inner().profiles;
+    profiles.sort_by(|left, right| {
+        left.category
+            .cmp(&right.category)
+            .then_with(|| left.id.cmp(&right.id))
+    });
+
+    if profiles.is_empty() {
+        println!("No provider profiles found.");
+        return Ok(());
+    }
+
+    println!("{}", "Available Provider Profiles:".cyan().bold());
+    let mut current_category = i32::MIN;
+    for profile in profiles {
+        if profile.category != current_category {
+            current_category = profile.category;
+            println!();
+            println!("  {}", display_provider_category(current_category).bold());
+        }
+        print_provider_type_row(&profile);
+    }
+
+    Ok(())
+}
+
+fn display_provider_category(category: i32) -> &'static str {
+    match ProviderProfileCategory::try_from(category).unwrap_or(ProviderProfileCategory::Other) {
+        ProviderProfileCategory::Inference => "INFERENCE",
+        ProviderProfileCategory::Agent => "AGENT",
+        ProviderProfileCategory::SourceControl => "SOURCE CONTROL",
+        ProviderProfileCategory::Messaging => "MESSAGING",
+        ProviderProfileCategory::Data => "DATA",
+        ProviderProfileCategory::Knowledge => "KNOWLEDGE",
+        ProviderProfileCategory::Other | ProviderProfileCategory::Unspecified => "OTHER",
+    }
+}
+
+fn print_provider_type_row(profile: &ProviderProfile) {
+    let inference = if profile.inference_capable {
+        " inference"
+    } else {
+        ""
+    };
+    println!(
+        "    {:<12} {:<42} endpoints: {:<2}{}",
+        profile.id,
+        profile.display_name,
+        profile.endpoints.len(),
+        inference
+    );
+}
+
 pub async fn provider_update(
     server: &str,
     name: &str,