feat: add token/placeholder substitution to publish pipelines

Copilot · EMaher · commit e28051330c1e · 2026-06-10T02:43:57.000Z
Adds {#[TOKEN_NAME]#} substitution as a dedicated step in both GitHub Actions and Azure DevOps publish pipelines generated by `apiops init`, compatible with APIOps Toolkit configuration files. - GitHub Actions: cschleiden/replace-tokens@v1.3 with {#[ / ]#} - Azure DevOps: replacetokens@6 with {#[ / ]#} - Step runs before apiops publish so secrets are never committed - New docs/guides/token-substitution.md covering syntax, setup, migration from APIOps Toolkit, and common examples - Updated ci-cd/github-actions.md and ci-cd/azure-devops.md Closes #36
diff --git a/docs/README.md b/docs/README.md
@@ -42,6 +42,7 @@ Requires Node.js 22 or later.
 - **Incremental publish** — Deploy only changed resources via git diff
 - **Dry-run mode** — Preview changes before applying them
 - **CI/CD scaffolding** — `apiops init` generates GitHub Actions or Azure DevOps pipelines
+- **Token substitution** — Replace `{#[TOKEN_NAME]#}` placeholders in config files with pipeline secrets before publish
 - **Multiple auth methods** — Azure CLI, managed identity, workload identity (OIDC), service principal
 
 ## Documentation Structure
@@ -64,6 +65,7 @@ docs/
 │   ├── multi-environment.md           — Dev / staging / prod promotion
 │   ├── multi-team-workflows.md        — Selective extraction, CODEOWNERS
 │   ├── code-first-workflow.md         — IDE → git → CI/CD → APIM
+│   ├── token-substitution.md          — Pipeline token/placeholder substitution
 │   └── migration-from-v1.md           — Migrate from Azure/apiops toolkit
 ├── ci-cd/
 │   ├── github-actions.md              — GitHub Actions integration
diff --git a/docs/ci-cd/azure-devops.md b/docs/ci-cd/azure-devops.md
@@ -144,7 +144,8 @@ Each stage:
 2. **Uses a deployment job** — Wraps the publish step in a `deployment` job targeting an [Azure DevOps environment](https://learn.microsoft.com/en-us/azure/devops/pipelines/process/environments) for approval gates
 3. **Loads per-environment variables** — Each stage uses its own variable group (`apim-dev`, `apim-prod`)
 4. **Authenticates per-environment** — Uses environment-specific service connections (`AZURE_SERVICE_CONNECTION_DEV`, `AZURE_SERVICE_CONNECTION_PROD`)
-5. **Applies overrides** — Passes `--override configuration.{env}.yaml` to apply [environment-specific overrides](../guides/environment-overrides.md)
+5. **Substitutes tokens** — Replaces `{#[TOKEN_NAME]#}` placeholders in `configuration.<env>.yaml` with secret variable values before publishing
+6. **Applies overrides** — Passes `--override configuration.{env}.yaml` to apply [environment-specific overrides](../guides/environment-overrides.md)
 
 ### Publish Pipeline Walkthrough
 
@@ -293,6 +294,27 @@ In your `package.json`, pin to a specific version:
 }
 ```
 
+### Using Token Substitution
+
+To replace `{#[TOKEN_NAME]#}` placeholders in `configuration.<env>.yaml` with secret variable values:
+
+1. **Install the [Replace Tokens extension](https://marketplace.visualstudio.com/items?itemName=qetza.replacetokens)** in your Azure DevOps organization (if not already installed).
+
+2. **Add secret variables** to the `apim-<env>` variable group (e.g., `PROD_SECRET_VALUE`). Mark them as secret.
+
+3. The generated substitution step runs automatically before publish:
+
+   ```yaml
+   - task: replacetokens@6
+     displayName: 'Substitute tokens in configuration.prod.yaml'
+     inputs:
+       sources: 'configuration.prod.yaml'
+       tokenPrefix: '{#['
+       tokenSuffix: ']#}'
+   ```
+
+See the [Token Substitution Guide](../guides/token-substitution.md) for full details, including migration from APIOps Toolkit.
+
 ---
 
 ## Troubleshooting
@@ -317,4 +339,5 @@ In your `package.json`, pin to a specific version:
 - [apiops publish](../commands/publish.md) — publish command reference
 - [Authentication Guide](../guides/authentication.md) — auth methods and RBAC
 - [Environment Overrides](../guides/environment-overrides.md) — per-environment configuration
+- [Token Substitution](../guides/token-substitution.md) — pipeline placeholder substitution with `{#[TOKEN_NAME]#}`
 - [Filtering Resources](../guides/filtering-resources.md) — extract specific APIs
diff --git a/docs/ci-cd/github-actions.md b/docs/ci-cd/github-actions.md
@@ -100,7 +100,8 @@ The workflow runs automatically when changes are pushed to `main` in these paths
 1. **Resolves the commit ID** — captures `GITHUB_SHA` for incremental publish.
 2. **Checks out the repository** with `fetch-depth: 2` (needed for git diff).
 3. **Authenticates with Azure** using OIDC federated credentials.
-4. **Runs `apiops publish`** in one of two modes:
+4. **Substitutes tokens** — replaces `{#[TOKEN_NAME]#}` placeholders in `configuration.<env>.yaml` with pipeline secret values.
+5. **Runs `apiops publish`** in one of two modes:
    - **Incremental** (default): uses `--commit-id` to publish only changed files.
    - **Full**: publishes all artifacts in the repository (useful for recovery or initial setup).
 
@@ -212,6 +213,24 @@ Use GitHub environment protection rules for production deployments:
 
 The publish workflow will pause and wait for approval before deploying to prod.
 
+### Using Token Substitution
+
+To replace `{#[TOKEN_NAME]#}` placeholders in your configuration YAML with pipeline secrets, add the secret mappings to the `env:` block of the generated substitution step:
+
+```yaml
+- name: Substitute tokens in configuration.prod.yaml
+  uses: cschleiden/replace-tokens@v1.3
+  with:
+    tokenPrefix: '{#['
+    tokenSuffix: ']#}'
+    files: '["configuration.prod.yaml"]'
+  env:
+    MY_SECRET: ${{ secrets.MY_SECRET }}
+    BACKEND_URL: ${{ secrets.BACKEND_URL }}
+```
+
+See the [Token Substitution Guide](../guides/token-substitution.md) for full details, including migration from APIOps Toolkit.
+
 ### Adding Environment Overrides
 
 To use [environment-specific overrides](../guides/environment-overrides.md), add the `--overrides` flag to the publish step in the workflow:
@@ -271,4 +290,5 @@ For authentication issues, see the [Authentication Guide](../guides/authenticati
 
 - [Authentication Guide](../guides/authentication.md) — all auth methods and RBAC roles
 - [Environment Overrides](../guides/environment-overrides.md) — per-environment configuration
+- [Token Substitution](../guides/token-substitution.md) — pipeline placeholder substitution with `{#[TOKEN_NAME]#}`
 - [Scenarios and Workflows](../guides/scenarios-and-workflows.md) — portal-first vs. code-first patterns
diff --git a/docs/guides/token-substitution.md b/docs/guides/token-substitution.md
@@ -0,0 +1,256 @@
+# Token/Placeholder Substitution
+
+Token substitution lets you store secrets and environment-specific values in your pipeline's secret store rather than in your configuration YAML files. The publish pipeline replaces `{#[TOKEN_NAME]#}` placeholders in `configuration.<env>.yaml` with the actual values before running `apiops publish`.
+
+This feature is compatible with APIOps Toolkit configuration files — users migrating from APIOps Toolkit can use their existing configuration files without modification.
+
+---
+
+## How It Works
+
+1. **Define placeholders** in your `configuration.<env>.yaml` using the `{#[TOKEN_NAME]#}` syntax:
+
+   ```yaml
+   namedValues:
+     - name: my-api-secret
+       properties:
+         displayName: my-api-secret
+         secret: true
+         value: "{#[PROD_SECRET_VALUE]#}"
+     - name: backend-url
+       properties:
+         displayName: backend-url
+         value: "{#[BACKEND_API_URL]#}"
+   ```
+
+2. **Store actual values** in your pipeline's secret store (GitHub Actions Secrets or Azure DevOps variable groups / Key Vault).
+
+3. **Token substitution runs automatically** as a pipeline step before `apiops publish`. Placeholders are replaced with the actual values in memory — the files on disk are modified only within the pipeline run and the secrets are never committed to the repository.
+
+The substitution follows this pattern:
+
+```
+{#[TOKEN_NAME]#}  →  value of environment variable TOKEN_NAME
+```
+
+---
+
+## GitHub Actions Setup
+
+### Generated Step
+
+`apiops init` generates a substitution step in each environment's publish job:
+
+```yaml
+- name: Substitute tokens in configuration.prod.yaml
+  uses: cschleiden/replace-tokens@v1.3
+  with:
+    tokenPrefix: '{#['
+    tokenSuffix: ']#}'
+    files: '["configuration.prod.yaml"]'
+  env:
+    # Map pipeline secrets/variables to environment variables so that
+    # {#[TOKEN_NAME]#} placeholders in configuration.prod.yaml are replaced
+    # with their actual values before the publish step runs. Example:
+    #   MY_SECRET: ${{ secrets.MY_SECRET }}
+```
+
+### Mapping Secrets to Tokens
+
+To substitute a token, add the corresponding secret to your GitHub environment and map it in the `env` block:
+
+```yaml
+- name: Substitute tokens in configuration.prod.yaml
+  uses: cschleiden/replace-tokens@v1.3
+  with:
+    tokenPrefix: '{#['
+    tokenSuffix: ']#}'
+    files: '["configuration.prod.yaml"]'
+  env:
+    PROD_SECRET_VALUE: ${{ secrets.PROD_SECRET_VALUE }}
+    BACKEND_API_URL: ${{ secrets.BACKEND_API_URL }}
+```
+
+> **Important:** The environment variable name must exactly match the token name inside `{#[...]#}`. Token names are case-sensitive.
+
+### Step-by-Step for GitHub Actions
+
+1. **Add secrets to your GitHub environment:**
+   - Go to **Settings → Environments → prod → Add secret**
+   - Add a secret for each token (e.g., `PROD_SECRET_VALUE`)
+
+2. **Map secrets to env vars** in the substitution step's `env:` block as shown above.
+
+3. **Define placeholders** in `configuration.prod.yaml` using the matching token names.
+
+4. The substitution step runs during the publish job and replaces the placeholders before `apiops publish` is called.
+
+### Example
+
+`configuration.prod.yaml`:
+```yaml
+namedValues:
+  - name: payment-api-key
+    properties:
+      displayName: payment-api-key
+      secret: true
+      value: "{#[PAYMENT_API_KEY]#}"
+```
+
+GitHub environment secret: `PAYMENT_API_KEY = sk-live-abc123...`
+
+Workflow `env:` mapping:
+```yaml
+env:
+  PAYMENT_API_KEY: ${{ secrets.PAYMENT_API_KEY }}
+```
+
+Result: the placeholder `{#[PAYMENT_API_KEY]#}` is replaced with `sk-live-abc123...` before publish.
+
+---
+
+## Azure DevOps Setup
+
+### Generated Step
+
+`apiops init` generates a substitution step in each environment's deployment job using the [Replace Tokens](https://marketplace.visualstudio.com/items?itemName=qetza.replacetokens) extension:
+
+```yaml
+- task: replacetokens@6
+  displayName: 'Substitute tokens in configuration.prod.yaml'
+  inputs:
+    sources: 'configuration.prod.yaml'
+    tokenPrefix: '{#['
+    tokenSuffix: ']#}'
+```
+
+> **Prerequisite:** The [Replace Tokens extension](https://marketplace.visualstudio.com/items?itemName=qetza.replacetokens) (by Guillaume Rouchon / qetza) must be installed in your Azure DevOps organization from the Visual Studio Marketplace.
+
+### Mapping Variables to Tokens
+
+The `replacetokens` task automatically reads from pipeline variables (including those from variable groups). Add your secret values as variables in the `apim-<env>` variable group:
+
+1. Go to **Pipelines → Library → apim-prod**
+2. Add a variable for each token (e.g., `PROD_SECRET_VALUE`)
+3. Check **"Keep this value secret"** to mark it as a secret variable
+
+The substitution task will automatically replace `{#[PROD_SECRET_VALUE]#}` with the variable value from the group.
+
+### Step-by-Step for Azure DevOps
+
+1. **Install the Replace Tokens extension** in your Azure DevOps organization if not already present.
+
+2. **Add secret variables to your variable group:**
+   - Go to **Pipelines → Library → apim-prod**
+   - Add each token as a secret variable (e.g., `PROD_SECRET_VALUE`)
+
+3. **Define placeholders** in `configuration.prod.yaml` using matching variable names.
+
+4. The substitution step runs automatically before the publish task.
+
+### Example
+
+`configuration.prod.yaml`:
+```yaml
+backends:
+  - name: order-service
+    properties:
+      url: "{#[ORDER_SERVICE_URL]#}"
+      description: Order processing backend
+```
+
+Variable group `apim-prod`:
+| Variable | Value | Secret |
+|----------|-------|--------|
+| `ORDER_SERVICE_URL` | `https://orders.contoso.com/api` | ✓ |
+
+Result: `{#[ORDER_SERVICE_URL]#}` is replaced with `https://orders.contoso.com/api` before publish.
+
+---
+
+## Migration from APIOps Toolkit
+
+If you are migrating from APIOps Toolkit, your existing `configuration.<env>.yaml` files that use `{#[TOKEN_NAME]#}` placeholders work without modification. The same syntax is supported.
+
+The only difference is where secrets are stored and mapped:
+
+| | APIOps Toolkit | APIOps CLI |
+|---|---|---|
+| **Token syntax** | `{#[TOKEN_NAME]#}` | `{#[TOKEN_NAME]#}` (identical) |
+| **GitHub Actions** | `cschleiden/replace-tokens@v1.3` | `cschleiden/replace-tokens@v1.3` (same action) |
+| **Azure DevOps** | `qetza.replacetokens@6` | `replacetokens@6` (same extension) |
+| **Token prefix/suffix** | `{#[` / `]#}` | `{#[` / `]#}` (identical) |
+
+### Migration Steps
+
+1. Copy your existing `configuration.<env>.yaml` files to your new repository — no changes required.
+
+2. Run `apiops init` to generate the pipeline scaffolding. The publish pipeline includes token substitution steps out of the box.
+
+3. Re-create your secrets in the new pipeline:
+   - **GitHub Actions**: Add each secret to the corresponding GitHub environment.
+   - **Azure DevOps**: Add each secret to the corresponding `apim-<env>` variable group.
+
+4. For GitHub Actions, add the `env:` mappings to the substitution step as described in [GitHub Actions Setup](#github-actions-setup).
+
+---
+
+## Common Use Cases
+
+### Named Value Secrets
+
+```yaml
+namedValues:
+  - name: subscription-key
+    properties:
+      displayName: subscription-key
+      secret: true
+      value: "{#[SUBSCRIPTION_KEY]#}"
+```
+
+### Backend URLs
+
+```yaml
+backends:
+  - name: inventory-api
+    properties:
+      url: "{#[INVENTORY_API_BASE_URL]#}"
+```
+
+### Multiple Tokens in One File
+
+```yaml
+namedValues:
+  - name: db-connection
+    properties:
+      value: "{#[DB_CONNECTION_STRING]#}"
+  - name: auth-secret
+    properties:
+      secret: true
+      value: "{#[AUTH_SECRET_KEY]#}"
+backends:
+  - name: payment-service
+    properties:
+      url: "{#[PAYMENT_SERVICE_URL]#}"
+```
+
+### Using Different Values per Environment
+
+Use separate `configuration.<env>.yaml` files (e.g., `configuration.dev.yaml`, `configuration.prod.yaml`) each referencing the same token names. The pipeline substitutes values from the environment-specific secret store, so `{#[API_KEY]#}` resolves to the dev key in dev and the prod key in prod.
+
+---
+
+## Security Notes
+
+- Tokens are replaced **in the pipeline runner's memory** — they are never committed to the repository.
+- Use your pipeline platform's secret storage (GitHub Actions Secrets or Azure DevOps secret variables / Key Vault) — never store actual secret values in YAML files.
+- The replaced configuration YAML files are only visible within the single pipeline run and are discarded after the run completes.
+
+---
+
+## Related
+
+- [Environment Overrides](environment-overrides.md) — merge environment-specific configuration before publishing
+- [GitHub Actions Integration](../ci-cd/github-actions.md) — full GitHub Actions pipeline guide
+- [Azure DevOps Integration](../ci-cd/azure-devops.md) — full Azure DevOps pipeline guide
+- [apiops init](../commands/init.md) — generate pipeline scaffolding
diff --git a/src/templates/azure-devops/publish-pipeline.ts b/src/templates/azure-devops/publish-pipeline.ts
@@ -42,6 +42,13 @@ export function generatePublishPipeline(config: PublishPipelineConfig): string {
               - script: npm ci
                 displayName: 'Install dependencies'
 
+              - task: replacetokens@6
+                displayName: 'Substitute tokens in configuration.${env}.yaml'
+                inputs:
+                  sources: 'configuration.${env}.yaml'
+                  tokenPrefix: '{#['
+                  tokenSuffix: ']#}'
+
               - task: AzureCLI@2
                 displayName: 'Publish to ${env} (incremental - last commit only)'
                 condition: ne('\${{ parameters.COMMIT_ID_CHOICE }}', 'publish-all-artifacts-in-repo')
diff --git a/src/templates/github-actions/publish-workflow.ts b/src/templates/github-actions/publish-workflow.ts
@@ -48,6 +48,18 @@ ${autoDeployComment}
           tenant-id: \${{ secrets.AZURE_TENANT_ID }}
           subscription-id: \${{ secrets.AZURE_SUBSCRIPTION_ID }}
 
+      - name: Substitute tokens in configuration.${env}.yaml
+        uses: cschleiden/replace-tokens@v1.3
+        with:
+          tokenPrefix: '{#['
+          tokenSuffix: ']#}'
+          files: '["configuration.${env}.yaml"]'
+        env:
+          # Map pipeline secrets/variables to environment variables so that
+          # {#[TOKEN_NAME]#} placeholders in configuration.${env}.yaml are replaced
+          # with their actual values before the publish step runs. Example:
+          #   MY_SECRET: \${{ secrets.MY_SECRET }}
+
       - name: Publish to ${env} (incremental - last commit only)
         if: \${{ github.event.inputs.COMMIT_ID_CHOICE != 'publish-all-artifacts-in-repo' }}
         run: |
diff --git a/tests/unit/templates/azure-devops/publish-pipeline.test.ts b/tests/unit/templates/azure-devops/publish-pipeline.test.ts
diff --git a/tests/unit/templates/github-actions/publish-workflow.test.ts b/tests/unit/templates/github-actions/publish-workflow.test.ts
