Merge pull request #236 from cdayne/feat/publish-filtering

feat: add --filter support to apiops publish

Author: azaslonov

README.md

@@ -85,9 +85,11 @@ Publish local artifact files to an Azure APIM service.
 | `--service-name <name>` | *(required)* | APIM service name |
 | `--source <dir>` | `./apim-artifacts` | Source artifacts directory |
 | `--overrides <path>` | | Path to overrides file |
+| `--filter <path>` | | Filter YAML file (same format as `extract`) |
+| `--no-transitive` | | Publish only filter matches, without referenced dependencies |
 | `--commit-id <sha>` | | Git commit SHA for incremental publish |
 | `--dry-run` | | Preview changes without applying |
-| `--delete-unmatched` | | Delete resources not in artifacts (mutually exclusive with `--commit-id`) |
+| `--delete-unmatched` | | Delete resources absent from artifacts, or removed by an incremental commit (mutually exclusive with `--filter`) |
 
 ```bash
 apiops publish --help
@@ -109,8 +111,19 @@ apiops publish \
   --resource-group <rg> \
   --service-name <name> \
   --commit-id <sha>
+
+# Publish a filtered subset; transitive dependencies are included by default
+apiops publish \
+  --resource-group <rg> \
+  --service-name <name> \
+  --source ./apim-artifacts \
+  --filter ./filter.yaml
 ```
 
+The publish filter uses the same YAML file and matching rules as `apiops extract --filter`.
+Referenced named values, backends (including backend pool members), policy fragments, and version
+sets are included automatically. Add `--no-transitive` to publish only the exact filter matches.
+
 ### `apiops init`
 
 Scaffold a new APIM artifacts repository with CI/CD pipelines.

docs/architecture.md

@@ -109,6 +109,7 @@ flowchart TB
     extract_svc --> apim_client
     extract_svc --> store
 
+    publish_svc --> filter_svc
     publish_svc --> override_svc
     publish_svc --> git_svc
     publish_svc --> dry_svc

docs/architecture/overview.md

@@ -161,7 +161,7 @@ flowchart TD
 - **Dry-run mode** — `--dry-run` compares local artifacts against live APIM state and outputs a change report without modifying anything.
 - **Override merging** — Environment-specific override files are merged into artifact JSON before publishing, enabling promotion across environments (dev → staging → prod).
 - **Topological ordering** — Resources are published in dependency order (e.g., named values before APIs that reference them).
-- **Delete-unmatched** — Optionally removes APIM resources not present in the artifact source. Requires explicit opt-in and is mutually exclusive with `--commit-id`.
+- **Delete-unmatched** — Optionally removes APIM resources not present in the artifact source. With `--commit-id`, it deletes only resources removed by the selected commit. All deletion requires explicit opt-in.
 
 ## Design Principles
 

docs/commands/publish.md

@@ -44,6 +44,20 @@ apiops publish \
   --commit-id abc123def456
 ```
 
+### Publish a filtered subset
+
+```bash
+apiops publish \
+  --resource-group my-rg \
+  --service-name my-apim \
+  --filter ./configuration.extractor.yaml
+```
+
+The filter uses the same YAML format as `apiops extract --filter`. Referenced named values,
+backends (including backend pool members), policy fragments, and version sets are included by
+default; use `--no-transitive` to publish only exact matches. Product, gateway, and subscription
+links do not pull their API or Product targets into the publish set.
+
 ### Delete resources not in source
 
 ```bash
@@ -73,11 +87,13 @@ apiops publish \
 | `--service-name <name>` | string | — | Yes | APIM service instance name |
 | `--source <dir>` | string | `./apim-artifacts` | No | Source directory containing artifacts |
 | `--overrides <path>` | string | — | No | Override configuration YAML file |
+| `--filter <path>` | string | — | No | Filter YAML file shared with `extract` |
+| `--no-transitive` | boolean | `false` | No | Publish only exact filter matches |
 | `--commit-id <sha>` | string | env: `COMMIT_ID` | No | Git commit SHA for incremental publish |
 | `--dry-run` | boolean | `false` | No | Preview changes without applying |
-| `--delete-unmatched` | boolean | `false` | No | Delete APIM resources not present in source |
+| `--delete-unmatched` | boolean | `false` | No | Delete APIM resources absent from source, or removed by an incremental commit |
 
-> **Note:** `--commit-id` and `--delete-unmatched` are **mutually exclusive**. The CLI will error if both are specified.
+> **Note:** `--filter` and `--commit-id` can be combined. `--delete-unmatched` cannot be combined with `--filter`; with `--commit-id`, it explicitly enables commit-scoped deletions.
 
 ### Global flags
 
@@ -168,7 +184,7 @@ In CI/CD pipelines, this is typically set automatically:
 - run: npx apiops publish --commit-id ${{ github.event.before }}
 ```
 
-> **Tip:** Incremental publish cannot be combined with `--delete-unmatched` because delete-unmatched requires a full comparison between source and APIM.
+> **Tip:** Incremental publish is non-destructive by default. Add `--delete-unmatched` to delete resources removed by the selected commit; omit `--commit-id` for a full unmatched-resource cleanup.
 
 ## Dry run
 
@@ -185,7 +201,7 @@ The output lists each resource and the planned action (create, update, or delete
 
 ## Delete unmatched
 
-When `--delete-unmatched` is set, resources that exist in the APIM instance but are **not** present in the source artifacts are deleted. This enforces the source directory as the single source of truth.
+For a full publish, `--delete-unmatched` deletes APIM resources that are **not** present in the source artifacts. With `--commit-id`, it deletes only resources and Product associations removed by the selected commit.
 
 > **Warning:** Use with caution. Resources created manually in the Azure portal that are not in your artifact directory will be removed.
 

docs/guides/dry-run-workflow.md

@@ -73,6 +73,7 @@ apiops publish \
 ```
 --- Dry-Run Report ---
 3 creates/updates
+0 patches
 1 deletes
 0 skipped
 
@@ -83,11 +84,12 @@ Planned actions:
   DELETE NamedValue/old-key
 ```
 
-Each action shows the operation (`PUT`, `DELETE`, `SKIP`), the resource type, and the resource name.
+Each action shows the operation (`PUT`, `PATCH`, `DELETE`, `SKIP`), the resource type, and the resource name.
 
 | Operation | Meaning |
 |-----------|---------|
 | `PUT` | Resource would be created (new) or updated (existing) |
+| `PATCH` | Resource would be partially updated |
 | `DELETE` | Resource would be removed from APIM |
 | `SKIP` | Resource could not be checked (error reading from APIM) |
 
@@ -104,6 +106,7 @@ Each action shows the operation (`PUT`, `DELETE`, `SKIP`), the resource type, an
     ],
     "summary": {
       "creates": 3,
+      "patches": 0,
       "deletes": 1,
       "skips": 0
     }
@@ -219,11 +222,12 @@ This lets reviewers see _"this PR will create 2 APIs and update 1 backend"_ dire
 | `--dry-run` | Preview full publish |
 | `--dry-run --delete-unmatched` | Preview full publish + unmatched resource deletions |
 | `--dry-run --commit-id <sha>` | Preview incremental publish (changed files only) |
+| `--dry-run --commit-id <sha> --delete-unmatched` | Preview incremental publish including commit-scoped deletions |
 | `--dry-run --overrides config.yaml` | Preview publish with environment overrides applied |
 | `--dry-run --format json` | Machine-readable preview output |
 | `--dry-run --log-level debug` | Preview with verbose diagnostic logging |
 
-> **Note:** `--commit-id` and `--delete-unmatched` remain mutually exclusive, even in dry-run mode.
+> **Note:** Incremental deletion is disabled unless `--delete-unmatched` is explicitly provided.
 
 ---
 

docs/guides/filtering-resources.md

@@ -1,6 +1,6 @@
 # Filtering Resources
 
-By default, `apiops extract` pulls every resource from your APIM instance. For large instances or multi-team setups, you can filter extraction to specific resources using a YAML filter file.
+By default, `apiops extract` pulls every resource from your APIM instance and `apiops publish` publishes every artifact in the source directory. For large instances or multi-team setups, you can use the same YAML filter file to limit either operation to specific resources.
 
 ## Why Filter?
 
@@ -34,6 +34,18 @@ apiops extract \
 
 `petstore-api`, `orders-api`, and their transitive dependencies are extracted — along with every backend, named value, product, tag, workspace, and every other resource type, because those keys are omitted and therefore default to "include all". To narrow the extract to just these APIs, see [How To: Extract Just One API](#how-to-extract-just-one-api) below.
 
+The same filter can limit publishing to a subset of the extracted artifacts:
+
+```bash
+apiops publish \
+  --resource-group my-rg \
+  --service-name my-apim \
+  --filter configuration.extractor.yaml
+```
+
+Referenced dependencies are included by default; add `--no-transitive` to publish only direct filter
+matches.
+
 ---
 
 ## How To: Extract Just One API

docs/guides/incremental-publish.md

@@ -140,17 +140,17 @@ Force a full publish (omit `--commit-id`) when:
 - **Configuration drift** — someone changed APIM directly in the portal and you want to overwrite everything from git.
 - **Major refactoring** — renaming many APIs or restructuring directories. A full publish ensures nothing is missed.
 - **Override-only changes** — you updated an override file but no artifact files changed. See [Gotcha: Override-only changes are not published incrementally](environment-overrides.md#gotcha-override-only-changes-are-not-published-incrementally).
-- **You need `--delete-unmatched`** — see below.
+- **You need a full unmatched-resource cleanup** — incremental deletion only covers resources removed by the selected commit.
 
-### `--commit-id` and `--delete-unmatched` are mutually exclusive
+### Incremental deletion requires explicit opt-in
 
-You cannot combine incremental publish with `--delete-unmatched`. The CLI exits with an error if both are specified.
+By default, incremental publish does not delete resources. Add `--delete-unmatched` to delete resources whose artifacts or Product associations were removed by the selected commit:
 
-```
-Options --commit-id (or COMMIT_ID) and --delete-unmatched are mutually exclusive.
+```bash
+apiops publish --commit-id abc123 --delete-unmatched ...
 ```
 
-**Why?** `--delete-unmatched` removes APIM resources that don't exist in the artifact directory — it requires a full view of all artifacts. Incremental publish only sees one commit's diff.
+This does not perform a full unmatched-resource scan. Omit `--commit-id` when you need to remove every APIM resource absent from the complete artifact source.
 
 ---
 
@@ -162,7 +162,7 @@ Options --commit-id (or COMMIT_ID) and --delete-unmatched are mutually exclusive
 | `Commit <sha> not found; skipping incremental diff` | Shallow clone doesn't include the commit | Use `fetch-depth: 2` (or more) in your checkout step to include at least the parent commit. |
 | Nothing published, no errors | Commit diff returned no artifact file changes | Verify the commit actually touches files in the `--source` directory. Use `git diff --name-status HEAD~1 HEAD` locally to check. |
 | Nothing published after override change | Override file changed but no artifact files changed | Override files are not artifact files — they don't trigger resource selection. Run a full publish (omit `--commit-id`) or include an artifact file change in the same commit. See [Gotcha: Override-only changes](environment-overrides.md#gotcha-override-only-changes-are-not-published-incrementally). |
-| `mutually exclusive` error | Both `--commit-id` and `--delete-unmatched` specified | Remove one. Use `--commit-id` for incremental or `--delete-unmatched` for full sync — not both. |
+| Removed resources remain in APIM | Incremental deletion was not enabled | Add `--delete-unmatched` after reviewing a dry-run. |
 
 ### GitHub Actions: fetch depth
 

docs/reference/configuration.md

@@ -63,7 +63,7 @@ Available on all commands (`extract`, `publish`, `init`):
 | `--dry-run` | Preview changes without applying | `false` |
 | `--delete-unmatched` | Delete resources not in artifacts | `false` |
 
-> ⚠️ `--commit-id` and `--delete-unmatched` are **mutually exclusive**. You cannot use both.
+> ⚠️ Incremental deletion is disabled unless `--delete-unmatched` is combined with `--commit-id`. The combination deletes only resources removed by the selected commit.
 
 ### `apiops init` Flags
 

docs/troubleshooting/common-errors.md

@@ -141,23 +141,15 @@ apiops init --environments dev prod
 
 ## Publish Errors
 
-### "Options --commit-id and --delete-unmatched are mutually exclusive"
+### Removed resources remain after incremental publish
 
-**Cause:** Both `--commit-id` and `--delete-unmatched` were specified. These flags conflict because:
+**Cause:** Incremental publishing is non-destructive by default.
 
-- `--commit-id` publishes only changed resources (partial set)
-- `--delete-unmatched` deletes resources not in the source (requires full set)
-
-Deleting based on a partial set would remove resources that were simply unchanged.
-
-**Solution:** Use one or the other:
+**Solution:** Preview and then enable commit-scoped deletion:
 
 ```bash
-# Incremental publish (changed resources only)
-apiops publish --commit-id abc123 ...
-
-# Full publish with cleanup (all resources, delete extras)
-apiops publish --delete-unmatched ...
+apiops publish --commit-id abc123 --delete-unmatched --dry-run ...
+apiops publish --commit-id abc123 --delete-unmatched ...
 ```
 
 ---

src/cli/publish-command.ts

@@ -3,15 +3,15 @@
 /**
  * Publish command CLI registration
  * Commander subcommand with --resource-group, --service-name, --source,
- * --overrides, --dry-run, --delete-unmatched flags.
+ * --overrides, --filter, --no-transitive, --dry-run, --delete-unmatched flags.
  * Includes --format json: machine-readable JSON output mode.
  */
 
 import { Command } from 'commander';
 import { PublishConfig } from '../models/config.js';
 import { ApimServiceContext } from '../models/types.js';
 import { runPublish, PublishResult } from '../services/publish-service.js';
-import { loadOverrideConfig } from '../lib/config-loader.js';
+import { loadFilterConfig, loadOverrideConfig } from '../lib/config-loader.js';
 import { logger, parseLogLevel } from '../lib/logger.js';
 import { ApimClient } from '../clients/apim-client.js';
 import { ArtifactStore } from '../clients/artifact-store.js';
@@ -25,6 +25,8 @@ interface PublishOptions {
   serviceName: string;
   source: string;
   overrides?: string;
+  filter?: string;
+  transitive: boolean;
   commitId?: string;
   dryRun: boolean;
   deleteUnmatched: boolean;
@@ -40,6 +42,8 @@ export function createPublishCommand(): Command {
     .requiredOption('--service-name <name>', 'APIM service instance name')
     .option('--source <dir>', 'Source directory with artifacts', './apim-artifacts')
     .option('--overrides <path>', 'Override configuration YAML file')
+    .option('--filter <path>', 'Filter configuration YAML file')
+    .option('--no-transitive', 'Disable transitive dependency inclusion')
     .option(
       '--commit-id <sha>',
       'Git commit SHA for incremental publish (overrides COMMIT_ID env var)'
@@ -121,15 +125,24 @@ async function executePublish(
     }
   }
 
+  let filterConfig;
+  if (options.filter) {
+    filterConfig = await loadFilterConfig(options.filter);
+    if (!filterConfig) {
+      logger.error(`Filter file not found: ${options.filter}`);
+      process.exit(2);
+    }
+  }
+
   // Resolve commit ID for incremental publish
   const commitId = options.commitId ?? process.env.COMMIT_ID;
   if (commitId) {
     logger.debug(`Using incremental publish with commit ID: ${commitId}`);
   }
 
-  if (hasMutuallyExclusivePublishOptions(options.deleteUnmatched, commitId)) {
+  if (hasMutuallyExclusivePublishOptions(options.deleteUnmatched, commitId, Boolean(options.filter))) {
     logger.error(
-      'Options --commit-id (or COMMIT_ID) and --delete-unmatched are mutually exclusive.'
+      'Option --delete-unmatched cannot be combined with --filter.'
     );
     process.exit(2);
   }
@@ -138,6 +151,8 @@ async function executePublish(
   const publishConfig: PublishConfig = {
     service: context,
     sourceDir: options.source,
+    filter: filterConfig,
+    includeTransitive: options.transitive,
     overrides: overrideConfig,
     dryRun: options.dryRun,
     deleteUnmatched: options.deleteUnmatched,
@@ -167,9 +182,10 @@ async function executePublish(
  */
 export function hasMutuallyExclusivePublishOptions(
   deleteUnmatched: boolean,
-  commitId?: string
+  _commitId?: string,
+  hasFilter = false
 ): boolean {
-  return deleteUnmatched && Boolean(commitId);
+  return deleteUnmatched && hasFilter;
 }
 
 /**
@@ -182,6 +198,7 @@ function outputJson(result: PublishResult): void {
     exitCode: number;
     summary: {
       totalPuts: number;
+      totalPatches: number;
       totalDeletes: number;
       totalErrors: number;
       totalSkipped: number;
@@ -198,9 +215,11 @@ function outputJson(result: PublishResult): void {
         operation: string;
         type: string;
         name: string;
+        error?: string;
       }>;
       summary: {
         creates: number;
+        patches: number;
         deletes: number;
         skips: number;
       };
@@ -215,6 +234,7 @@ function outputJson(result: PublishResult): void {
     exitCode: result.exitCode,
     summary: {
       totalPuts: result.totalPuts,
+      totalPatches: result.totalPatches,
       totalDeletes: result.totalDeletes,
       totalErrors: result.totalErrors,
       totalSkipped: result.totalSkipped,
@@ -235,6 +255,7 @@ function outputJson(result: PublishResult): void {
         operation: a.operation,
         type: a.type,
         name: a.name,
+        error: a.error,
       })),
       summary: result.dryRunReport.summary,
     };
@@ -257,6 +278,7 @@ function outputText(result: PublishResult, dryRun: boolean): void {
     process.stdout.write(
       `${result.dryRunReport.summary.creates} creates/updates\n`
     );
+    process.stdout.write(`${result.dryRunReport.summary.patches} patches\n`);
     process.stdout.write(`${result.dryRunReport.summary.deletes} deletes\n`);
     process.stdout.write(`${result.dryRunReport.summary.skips} skipped\n`);
 
@@ -272,7 +294,7 @@ function outputText(result: PublishResult, dryRun: boolean): void {
     // Regular publish mode summary
     process.stdout.write('\n--- Summary ---\n');
     process.stdout.write(
-      `${result.totalPuts} creates/updates, ${result.totalDeletes} deletes, ${result.totalSkipped} skipped\n`
+      `${result.totalPuts} creates/updates, ${result.totalPatches} patches, ${result.totalDeletes} deletes, ${result.totalSkipped} skipped\n`
     );
 
     if (result.totalErrors > 0) {