diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..bcc7f34dd4 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# maintainers are the overall code owners +* @velero-io/Maintainer \ No newline at end of file diff --git a/.github/workflows/auto_assign_prs.yml b/.github/workflows/auto_assign_prs.yml index 9b915533c4..c2d7ea8e8d 100644 --- a/.github/workflows/auto_assign_prs.yml +++ b/.github/workflows/auto_assign_prs.yml @@ -14,6 +14,7 @@ permissions: jobs: # Automatically assigns reviewers and owner add-reviews: + if: github.repository == 'velero-io/velero' && github.event_name == 'pull_request_target' runs-on: ubuntu-latest steps: - name: Set the author of a PR as the assignee diff --git a/.github/workflows/auto_label_prs.yml b/.github/workflows/auto_label_prs.yml index 042cc7e95d..21540d8cb2 100644 --- a/.github/workflows/auto_label_prs.yml +++ b/.github/workflows/auto_label_prs.yml @@ -15,6 +15,7 @@ permissions: jobs: # Automatically labels PRs based on file globs in the change. triage: + if: github.repository == 'velero-io/velero' runs-on: ubuntu-latest steps: - uses: actions/labeler@v5 diff --git a/.github/workflows/e2e-test-kind.yaml b/.github/workflows/e2e-test-kind.yaml index b195771877..b0ddb53a31 100644 --- a/.github/workflows/e2e-test-kind.yaml +++ b/.github/workflows/e2e-test-kind.yaml @@ -87,12 +87,41 @@ jobs: id: set-matrix # everything excluding older tags. limits needs to be high enough to cover all latest versions # and test labels - # grep -E "v[1-9]\.(2[5-9]|[3-9][0-9])" filters for v1.25 to v9.99 + # grep -E "^v[1-9]\.(2[5-9]|[3-9][0-9])\.[0-9]+$" filters for well-formed v1.25.x to v9.99.x + # GA releases only, so a pre-release tag like v1.37.0-rc.1 can't reach the + # awk step below and be misparsed as a patch release (e.g. "1.37.1") # and removes older patches of the same minor version # awk -F. '{if(!a[$1"."$2]++)print $1"."$2"."$NF}' run: | + set -euo pipefail + candidates=$(wget -q -O - "https://hub.docker.com/v2/namespaces/kindest/repositories/node/tags?page_size=50" | grep -o '"name": *"[^"]*' | grep -o '[^"]*$' | grep -E "^v[1-9]\.(2[5-9]|[3-9][0-9])\.[0-9]+$" | awk -F. '{if(!a[$1"."$2]++)print $1"."$2"."$NF}' | sort -r | sed s/v//g) + + # Docker Hub's tag listing can include tags whose manifest was never + # published or has since been removed (e.g. kindest/node:v1.37.1). If such + # a tag reaches the matrix, its job is guaranteed to fail with + # "manifest unknown" once helm/kind-action tries to pull it, which clogs + # up the e2e queue on every PR with a red job unrelated to the change + # under test. Test-pull each candidate's manifest here and drop any tag + # that isn't actually available before building the matrix. + valid=() + while IFS= read -r v; do + [ -z "$v" ] && continue + echo "Verifying kindest/node:v${v} image is available..." + if docker manifest inspect "kindest/node:v${v}" > /dev/null 2>&1; then + valid+=("$v") + else + echo "::warning::kindest/node:v${v} manifest not found on Docker Hub; excluding from e2e test matrix" + fi + done <<< "$candidates" + + if [ ${#valid[@]} -eq 0 ]; then + echo "::warning::No kindest/node tags passed the manifest availability check; the e2e test matrix will have no Kubernetes versions to test" + fi + + k8s_json=$(printf '%s\n' "${valid[@]+"${valid[@]}"}" | jq -R -c -s 'split("\n") | map(select(length > 0))') + echo "matrix={\ - \"k8s\":$(wget -q -O - "https://hub.docker.com/v2/namespaces/kindest/repositories/node/tags?page_size=50" | grep -o '"name": *"[^"]*' | grep -o '[^"]*$' | grep -v -E "alpha|beta" | grep -E "v[1-9]\.(2[5-9]|[3-9][0-9])" | awk -F. '{if(!a[$1"."$2]++)print $1"."$2"."$NF}' | sort -r | sed s/v//g | jq -R -c -s 'split("\n")[:-1]'),\ + \"k8s\":${k8s_json},\ \"labels\":[\ \"Basic && (ClusterResource || NodePort || StorageClass)\", \ \"ResourceFiltering && !Restic\", \ diff --git a/.github/workflows/pr-changelog-check.yml b/.github/workflows/pr-changelog-check.yml index 0f296853a4..f9fb14f37f 100644 --- a/.github/workflows/pr-changelog-check.yml +++ b/.github/workflows/pr-changelog-check.yml @@ -7,6 +7,7 @@ on: jobs: build: + if: github.repository == 'velero-io/velero' name: Run Changelog Check runs-on: ubuntu-latest steps: diff --git a/.github/workflows/pr-ci-check.yml b/.github/workflows/pr-ci-check.yml index ba55e6ab0c..2b25ff24c2 100644 --- a/.github/workflows/pr-ci-check.yml +++ b/.github/workflows/pr-ci-check.yml @@ -2,11 +2,13 @@ name: Pull Request CI Check on: [pull_request] jobs: get-go-version: + if: github.repository == 'velero-io/velero' && github.base_ref == 'main' uses: ./.github/workflows/get-go-version.yaml with: ref: ${{ github.event.pull_request.base.ref }} build: + if: github.repository == 'velero-io/velero' && github.base_ref == 'main' name: Run CI needs: get-go-version runs-on: ubuntu-latest diff --git a/.github/workflows/pr-codespell.yml b/.github/workflows/pr-codespell.yml index 65d2a18855..b65ae7ae5d 100644 --- a/.github/workflows/pr-codespell.yml +++ b/.github/workflows/pr-codespell.yml @@ -3,6 +3,7 @@ on: [pull_request] jobs: codespell: + if: github.repository == 'velero-io/velero' name: Run Codespell runs-on: ubuntu-latest steps: diff --git a/.github/workflows/pr-linter-check.yml b/.github/workflows/pr-linter-check.yml index 79d7918b24..439b129201 100644 --- a/.github/workflows/pr-linter-check.yml +++ b/.github/workflows/pr-linter-check.yml @@ -28,5 +28,5 @@ jobs: - name: Linter check uses: golangci/golangci-lint-action@v9 with: - version: v2.5.0 + version: v2.13.1 args: --verbose diff --git a/.github/workflows/prow-action.yml b/.github/workflows/prow-action.yml index a38953ed89..0789b747b4 100644 --- a/.github/workflows/prow-action.yml +++ b/.github/workflows/prow-action.yml @@ -7,6 +7,7 @@ on: jobs: execute: + if: github.repository == 'velero-io/velero' runs-on: ubuntu-latest steps: - uses: jpmcb/prow-github-actions@v1.1.3 diff --git a/.github/workflows/rebase.yml b/.github/workflows/rebase.yml index 07c86b5342..064bef70ae 100644 --- a/.github/workflows/rebase.yml +++ b/.github/workflows/rebase.yml @@ -5,7 +5,7 @@ name: Automatic Rebase jobs: rebase: name: Rebase - if: github.event.issue.pull_request != '' && contains(github.event.comment.body, '/rebase') + if: github.repository == 'velero-io/velero' && github.event.issue.pull_request != '' && contains(github.event.comment.body, '/rebase') runs-on: ubuntu-latest steps: - name: Checkout the latest code diff --git a/.github/workflows/stale-issues.yml b/.github/workflows/stale-issues.yml index 58e1ca89fe..74d1b1ffb3 100644 --- a/.github/workflows/stale-issues.yml +++ b/.github/workflows/stale-issues.yml @@ -5,6 +5,7 @@ on: jobs: stale: + if: github.repository == 'velero-io/velero' runs-on: ubuntu-latest steps: - uses: actions/stale@v10.1.1 diff --git a/Dockerfile b/Dockerfile index dc5b5dd9f8..742cd8961c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -73,7 +73,7 @@ RUN mkdir -p /output/usr/bin && \ go clean -modcache -cache # Velero image packing section -FROM paketobuildpacks/run-jammy-tiny:0.2.139 +FROM paketobuildpacks/run-jammy-tiny:0.2.165 LABEL maintainer="Xun Jiang " diff --git a/Tiltfile b/Tiltfile index 63e3a231fc..143ea74eee 100644 --- a/Tiltfile +++ b/Tiltfile @@ -52,7 +52,7 @@ git_sha = str(local("git rev-parse HEAD", quiet = True, echo_off = True)).strip( tilt_helper_dockerfile_header = """ # Tilt image -FROM golang:1.25.11 as tilt-helper +FROM golang:1.26.7 as tilt-helper # Support live reloading with Tilt RUN wget --output-document /restart.sh --quiet https://raw.githubusercontent.com/windmilleng/rerun-process-wrapper/master/restart.sh && \ diff --git a/changelogs/CHANGELOG-1.18.md b/changelogs/CHANGELOG-1.18.md index fd1f3f5666..f0e5e56aa4 100644 --- a/changelogs/CHANGELOG-1.18.md +++ b/changelogs/CHANGELOG-1.18.md @@ -1,3 +1,40 @@ +## v1.18.3 + +### Download +https://github.com/vmware-tanzu/velero/releases/tag/v1.18.3 + +### Container Image +`velero/velero:v1.18.3` + +### Documentation +https://velero.io/docs/v1.18/ + +### Upgrading +https://velero.io/docs/v1.18/upgrade-to-1.18/ + +### All Changes + * Avoid duplicated InitContainer names generated in velero install CLI. (#10396, @blackpiglet) + * Bound WaitRestoreExecHook polling with resourceTimeout to avoid an infinite wait when restore exec hooks never complete. (#10394, @nitishmalang) + * fix log format string mismatches that produce wrong or mangled output (#10392, @samay43) + * Skip DeleteSnapshot when ProviderSnapshotID is empty (#10381, @kaovilai) + * Fail backup validation when built-in data mover is requested but no node-agent pods are running (#10360, @Joeavaikath) + * Fix issue #10341, avoid mutating the cached node-agent LoadAffinity so the OS node selector term is not appended repeatedly to data mover pods (#10348, @shubham-pampattiwar) + * Fix PodVolumeBackup metadata loss on fs-backup timeout, which caused all fs-backup volumes to become unrestorable (#9999, @shubham-pampattiwar) + * Fix repo connection contest of the two repositories with the same storage type (#10377, @Lyndon-Li) + * Only sync finished backups from object storage (#10347, @chlins) + * Support copying namespace-scoped secrets and configmaps for backup and restore PVC provisioning to enable datamover backup/restore of encrypted CSI volumes (#10335, @shubham-pampattiwar) + * Remove PVC and PV inclusion check during creating PVR. (#10319, @blackpiglet) + * Fix a potential deadlock when resultsLock is held by the informer but blocked on resChan because the early quit of RestorePodVolumes (#10263, @Lyndon-Li) + * Fix restore-wait init container ignoring pod-level securityContext, falling back to hardcoded runAsUser 1000 instead of the workload's own uid/gid, causing fs-backup restores to deadlock at Init:0/1 on owner-restricted volumes (#10224, @kaovilai) + * Add use guide for restore fine-grained filters via resource policy (#10164, @adam-jian-zhang) + * Add restore.velero.io/must-include-additional-items so RestoreItemActions can opt in to bypassing global restore filters for AdditionalItems (mirrors the backup-side must-include annotation; no default behavior change for existing restores/plugins). Stop force-including VolumeSnapshotContents via resourceMustHave on every restore; CSI VolumeSnapshot/PVC RestoreItemActions now set restore.velero.io/must-include-additional-items so bound snapshot dependencies are restored only when their parent is restored (fixes #9957) (#10101, @adam-jian-zhang) + * User guide for backup fine-grained filters via resource policy, and add set based label selectors for fine-grained filters (#10072, @adam-jian-zhang) + * Fix issue #10032, prioritize exact namespace match in restore (#10052, @adam-jian-zhang) + * Fix issue #9997, cancel ongoing PVB on timeout and wait for all PVBs to terminal state (#10039, @Lyndon-Li) + * Add fine-grained filters for restore via resource policy, introduced resourcePolicy field for restoreSpec, which contains ClusterScopedFilterPolicy and NamespacedFilterPolicy section (#10015, @adam-jian-zhang) + * Add support for matching PVCs by volume mode and access mode in resource policies, and introduce the `--global-backup-volume-policies-configmap` server flag to merge cluster-wide backup volume policies into every backup. (#10012, @chlins) + * Add fine-grained filters for backup via resource policy, introduced ClusterScopedFilterPolicy and NamespacedFilterPolicy section for resource policy (#10011, @adam-jian-zhang) + ## v1.18.2 ### Download diff --git a/config/crd/v1/bases/velero.io_restores.yaml b/config/crd/v1/bases/velero.io_restores.yaml index 1b92ea4fcc..c41fe88de2 100644 --- a/config/crd/v1/bases/velero.io_restores.yaml +++ b/config/crd/v1/bases/velero.io_restores.yaml @@ -404,6 +404,33 @@ spec: - name type: object x-kubernetes-map-type: atomic + resourcePolicy: + description: |- + ResourcePolicy specifies the reference to a ConfigMap containing resource + filter policies for this restore. The ConfigMap can contain a + namespacedFilterPolicies section that specifies per-namespace resource type + filters, label selectors, and resource name patterns, and a + clusterScopedFilterPolicy section for per-kind filtering of cluster-scoped + resources. The ConfigMap format is the same as for BackupSpec.ResourcePolicy. + nullable: true + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic restorePVs: description: |- RestorePVs specifies whether to restore all included diff --git a/config/crd/v1/crds/crds.go b/config/crd/v1/crds/crds.go index d26e9cfd81..1e46e16ce3 100644 --- a/config/crd/v1/crds/crds.go +++ b/config/crd/v1/crds/crds.go @@ -36,7 +36,7 @@ var rawCRDs = [][]byte{ []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcWMo\xe36\x10\xbd\xfbW\f\xd0K\v\xac\xe4\x06E\x8b·\xd6\xd9C\xb0\xe96\x88\xb7\xb9S\xd4HbC\x91,9t6E\x7f|1\xa4\xe4\x0fYv\x9c\xcb\xea\xe6\xe1p\xf8\xe6\xcd\xcc#]\x14\xc5B8\xf5\x84>(kV \x9c¯\x84\x86\x7f\x85\xf2\xf9\xd7P*\xbb\xdc\xde,\x9e\x95\xa9W\xb0\x8e\x81l\xff\x88\xc1F/\xf1\x16\x1be\x14)k\x16=\x92\xa8\x05\x89\xd5\x02@\x18cI\xb09\xf0O\x00i\ry\xab5\xfa\xa2ES>\xc7\n\xab\xa8t\x8d>\x05\x1f\x8f\xde\xfeX\xde\xfcR\xfe\xbc\x000\xa2\xc7\x15\xd4\xf6\xc5h+j\x8f\xffD\f\x14\xca-j\xf4\xb6Tv\x11\x1cJ\x8e\xddz\x1b\xdd\n\xf6\vy\xefpn\xc6|;\x84y\xccaҊV\x81>ͭޫ\xc1\xc3\xe9\xe8\x85>\x05\x91\x16\x832m\xd4\u009f,/\x00\x82\xb4\x0eW\xf0\x99a8!\xb1^\x00\f)&XŐ\xdd\xf6&\x87\x92\x1d\xf6\"\xe3\x05\xb0\x0e\xcdo\x0fwO?m\x8e\xcc\x005\x06镣D\xd4\x7f\xc5\xce\x0e\xd3\x04@\x05\x100\xc0\x01\xb2;\x84 \f\bO\xaa\x11\x92\xa0\xf1\xb6\x87J\xc8\xe7\xe8\xc0V\x7f\xa3$\bd\xbdh\xf1\x03\x84(;\x10\x1c%;\x1c\x9c\xa5m\v\x8d\xd2X\xeel\xce[\x87\x9e\xd4Hy\xfe\x0e\x1a\xea\xc0z)\v\xfe8\xf1\xbc\vj\xee,\f@\x1d\x8e\xe4a=p\x05\xb6\x01\xeaT\x00\x8f\xcec@\x93{\x8d\xcd\xc2\fٔ\x93\xd0\x1b\xf4\x1c\x06Bg\xa3\xae\xb9!\xb7\xe8\t\x1aE\xaf\xcb41\xaa\x8ad}XָE\xbd\f\xaa-\x84\x97\x9d\"\x94\x14=.\x85SEJĤQ+\xfb\xfa;?\ff8:\x96^\xb9!\x03yeڃ\x854\x1d\xef(\x0f\xcfK\xee\xae\x1c*\xa7\xb8\xaf\x02\x9b\x98\xbaǏ\x9b/0\"ɕ\x1aZl\xe7z\xc2\xcbX\x1ffS\x99\x06}ޗڔc\xa2\xa9\x9dU\x86\xd2\x0f\xa9\x15\x1a\x82\x10\xab^Q\x18{\x9dK7\r\xbbNR\x04\x15Bt\xb5 \xac\xa7\x0ew\x06֢G\xbd\x16\x01\xbfq\xad\xb8*\xa1\xe0\"\\U\xadC\x81\x9d:gz\x0f\x16Fy&j^\x01\x128\xe1[\xa4\xa9u\x82\xe5Kr\xe2\xe3_:q,X\xdfcٖ\xac9a\x00\x92\xf5\xe8\x87i\xa1.a\x80\xd9F\x9fE2\xf67\xd3\xc0\xbc\xb2\xa0\xb0\xd8\x1db:=\x9a?4\xb1\x9f?\xa0\x80\xdf\x13\xe6{\xdb^\\_[C<\x17\x17\x9d\x9e\xac\x8e=n\x8cp\xa1\xb3o\xf8\xde\x11\xf6\x7f:\xf4\xf9\x1a\xbe\xe8:\xde滫\xef\x82c\xd4g\xcf}D\xbeA\xf0|\xa6\x83\xc3UQ\xae\xc04x^\x95\xe8zs\xf7\x1e\nϸ\xbf\xa3Hw\xa6\xb1o\xa4\xb8w\x9c\xf5;#\x03\xe3\x97\xde\x10o\xf74\xbfBƞ\xe6-\xf9\xeeD\xf8\x14+\xf4\x06\t\xc3^\xa9_\x14u\xb3\x11\x01^:%\xbb\xb41\r\x04_\x02!X\xa9\xe6$\xf5\n\xf8\xac#\xca\xe3\xccP\x16iXg\xcc\f\xfe\xc4|F\xfd\xce\x1dP\f\x8at\x95\x82\x92\xa0\x18ޡ\xa1\xc9\x7f\xa4ZF\xef\xd3\x15\x95\xad\xfc2\x99n\xb8VDG\xe5\xf9\xeb\xf1\xfe\r%\xbd\xdd{\xa6\x17\xb7P&\xa3q\x1e\x8b\xa0Z~A\xf1\x1akiҸS2\xf2w\xfc\xc2;&j\xb6\xa2\xf8թ<\x80o@\xfc\xb8ŝ\x8f&\xdf\xf3\xd37l\n\x88\x81\x9f[ \x85\x99\xc1X!Ԩ\x91\xb0\x86\xea5\xdf\\\xaf\x81\xb0?\xc5\xddX\xdf\vZ\x01\xdf\xff\x05\xa9\x9962QkQi\\\x01\xf9x\xae\xcbf\x13w\x9d\b3cx\x94\xf3\x03\xfb\xcc5\xc6n\x18/v\x06\x9c\xbd_\n\xf8\x8c/3\xd6\ao%\x86\x80\xa7ct6\x93\xd9!81\x06~\xa4\xd5\a,\r\x7f\x19\x06\xcb\xff\x01\x00\x00\xff\xffx\xae@\xbaJ\x0e\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4:Ks\x1b7\xd2w\xfd\x8a.吤\xca$\xe3|ߦ\xb6x\xb3\xe5͖v\x13\xafʔ}I\xe5\xd0\x1c49\x88f\x00,\x80\x11\xcd\xcd\xe6\xbfo5\x80\xe1\xbc@R\xa2\x93\x18\x17\x89x4\xfa\xfd\xc2\xccf\xb3+4\xf2\x03Y'\xb5Z\x02\x1aI\x1f=)\xfe\xe5\xe6\x0f\x7fus\xa9\x17\x8f/\xaf\x1e\xa4\x12K\xb8i\x9c\xd7\xf5;r\xba\xb1\x05\xbd\xa1\x8dT\xd2K\xad\xaej\xf2(\xd0\xe3\xf2\n\x00\x95\xd2\x1ey\xda\xf1O\x80B+ouU\x91\x9dmI\xcd\x1f\x9a5\xad\x1bY\t\xb2\x01x{\xf5\xe37\xf3\x97\xdf\xcd\xffr\x05\xa0\xb0\xa6%\x18-\x1eu\xd5Դ\xc6\xe2\xa11n\xfeH\x15Y=\x97\xfa\xca\x19*\x18\xf6\xd6\xea\xc6,\xa1[\x88gӽ\x11\xe7;->\x040\xaf\x03\x98\xb0RI\xe7\xff\x99[\xfdA:\x1fv\x98\xaa\xb1XM\x91\b\x8bN\xaamS\xa1\x9d,_\x01\xb8B\x1bZ\xc2[F\xc3`A\xe2\n \x91\x18К\x01\n\x11\x98\x86՝\x95ʓ\xbda\b-\xb3f \xc8\x15V\x1a\x1f\x982\xc2\x0f\x9cG\xdf8pMQ\x02:xK\xbbŭ\xba\xb3zk\xc9E\xe4\x00~qZݡ/\x970\x8f\xdb\xe7\xa6DGi52w\x15\x16Ҕ\xdf3\xca\xce[\xa9\xb69$\xeeeM \x1a\x1b\x84\xca\xd4\x17\x04\xbe\x94n\x82\xdd\x0e\x1dch} ;\x8fKXg\x88\xcecm\xc6H\xf5\x8eF\xac\x04z\xca\xe1t\xa3kS\x91'\x01뽧\x96\x92\x8d\xb65\xfa%H\xe5\xbf\xfb\xff\xe3\xecH\xfc\x9a\x87\xa3o\xb4\x1a\xf2\xe65\xcfBo:b²ڒ\xcd2H{\xac>\x05\x11\xcf\x00^\xf7\xceGL\"\xdc\xfe\xfcYTnUa\xa9&u\x19B\xb2;=Ŧ\x0f\xba\xbfj\xac\xd4V\xfa\xfd\x12^~\xf3T4\xd9>@o\xc0\x97\x04IyV^[\xdc\x12\xfc\xa0\x8b\xa8h\xbb\x92lR\xb4u\xd2\xfeR7\x95\x80u+\x18\x00\xe7\xb5\xcd*\x9b\xa1b\x1eO%\xb8-ؑ\xc6\r\xef\xfc#\f\xa2\xb0\x84Y\x83h\x9d\xe6<\xec\x90Z\xe5\xad\xe2Ֆ\x9ed\x11}\x96*-\xe8\xc0?\x9a\xa0%\x1d\x18\xab\vr\ue1212\x8c\x01\"o\xbb\x89\xb3\f*)\xeci\xf1iL\xa5Q\x90\x05\xaf\xa1D%*b2\x10\xbcE\xe56IE\xa6\x02l\x8f\xdd\xef\xcd\x10\x95\xf7i\xe1\x18:q\xd7\xe3\xcb讋\x92j\\\xa6\xbdڐzuw\xfb\xe1\xffV\x83iVcm\xc8zن\x8f8z\xc1\xb17\vCr\xff;\x1b\xac\x01\xf0\x05\xf1\x14\b\x8e\x92\xe4\x02\x1bR \x91p\x8a\xec\x91\x0e,\x19K\x8eM+h\x94\xde\x00*\xd0\xeb_\xa8\xf0\xf3\x11\xe8\x15Y\x06\xd3\xdaB\xa1\xd5#Y\x0f\x96\n\xbdU\xf2?\a؎y͗V\xe8\xc9\xf9`\x8cVa\x05\x8fX5\xf4\x02P\x89\x11\xe4\x1a\xf7`\x89\xef\x84F\xf5\xe0\x85\x03n\x8cǏ\xda\x12H\xb5\xd1K(\xbd7n\xb9Xl\xa5oS\x86B\xd7u\xa3\xa4\xdf/B\xf4\x97\xeb\xc6k\xeb\x16\x82\x1e\xa9Z8\xb9\x9d\xa1-J\xe9\xa9\xf0\x8d\xa5\x05\x1a9\v\x84\xa8\x906\xcck\xf1\x85MI\x86\x1b\\;\x11t\x1c!\xd2?C<\x1c\xfb\xd9\b0\x81\x8a$vR\xe0)fݻ\xbf\xad\xee\xa1\xc5$J*\n\xa5\xdb:\xe1K+\x1f\xe6\xa6T\x1b\xd6y>\xb7\xb1\xba\x0e0I\t\xa3\xa5\xf2\xe1GQIR\x1e\\\xb3\xae\xa5g5\xf8wCγ\xe8\xc6`oBZ\x05k\xb6%\xf6\x00b\xbc\xe1V\xc1\r\xd6Tݠ\xa3?YV,\x157c!\x89wg\xf9\xc3c#\xa9\x12!s8\x7fwVsy\xdcn\"\x12!\"x\r\bFRA\x83h\fR9O(\xd2$;AKi\xedE\xf4\xf4G\x91\xe4\xd1Em\x96\t G\x1e)\xe0\x1f\xab\x7f\xbd]\xfc]G:\x00\vN\xcdB\xad\x17\xf2\xed\x17\x87zO\x90\x93\x96\x04Wo4\xafQ\xc9\r9?O\xd0Ⱥ\x9f\xbe\xfd9\xcf?\x80\xef\xb5\x05\xfa\x88\\5\xbd\x00\x19y~\bf\xad\xdaH\x17\t?@\x84\x9d\xf4e@\xd4h\x91\b\xdc\x05\x12<>\xb0%G\x12\x1a\x82J>d\xec'\x8e\xeb\x90\xcduh\xfe\xca\xd6\xf3\xdb5|\x15\x9d\xd75\xff\xbc\x8eh\x1cҖ\xbe\x81u\xe8D+\xb3r\xbb\xa5.\xef\x9f(\v\x87Y\x0eP_\x83\xb6L\xab\xd2=\x10\x010\xcb)\xc6\a\x12\x13\xf4~\xfa\xf6\xe7k\xf8jȃ#WI%\xe8#|\xcb\xde'\xf0\xc6h\xf1\xf5\x1c\xee\x83\x1e\xec\x95Ǐ|SQjG\n\xb4\xaa\xf61\x01~$p\xba&\xd8QU\xcdb\x82(`\x87{Л#\xf7\xb4\"b\xd5D0h\xfd\xc9$1\xf1\xe1\xb4\xd1L\xb3\xa6v<\xcd^B\x16\xf5$\xeb\xfdl\x19\xc8\x139\x11ʅO\xe0D\xbf\xf4\xba\x80\x13\x0f͚\xac\"O\x81\x19B\x17\x8e\xf9P\x90\xf1n\xa1\x1f\xc9>J\xda-v\xda>H\xb5\x9d\xb12\u03a2\xd4\xdd\"t\xbb\x16_\x84?\x97\x12\x1e\xdaT\x9fJ}\x00\xf2\xf9X\xc0\xb7\xbb\xc5%\x1ch\xb3\xfb\xa7Ǯ\xa3|X\xa5\x84s\f\x93m~Wʢlk\xbd\x9e\xb7\xadQDw\x8cj\xff\x99l\x87\xf9\xdcX\xc6h?K\xad\xda\x19*\xc1\xff;\xe9<\xcf_\xc2\xd8F~\x92sy\x7f\xfb\xe6sZT#/\xf1$Gj\x988>\xce:\xacf5\x9aY܍^ײ\x18\xed\xe6\x1c\xfeV\xb0\x906\x92\xec\x99\xf4\xef\xdd`s\x9b\xa0f\xaa\x81Þg\xe5\x9f\x1e\xb7\x99\x84\xaf\xdf\xc5>\x95\x16\x9e\xe4\xd7yU\xb8ǭ\x03\xb4\x04\b5\x1aֈ\a\xda\xcfb\xc6aPr\xba\xc0\x19\xc1\xa11\bhL\xc51=f\x11\x19\x88)\xffM\xecA\x17\xe8;Ɛ\xac(ۮԊ\xbc\x97\xea32\xe7\xfd\b\x91ߗQ\x87\x9e]\xa1\xd5FnS\xb7s\xca)\xd5T\x15\xae+Z\x82\xb7ͱ\x9a\xeb$#\xefy\xcbi\xfa\xdf\xf7\xb6\xb6\x1a~\xa6\xc1\x98\xa7j\xd0v\x9c\x12C\xaa\xa9\xa7\xa8\xcc\xe0A\x1b\x89\x99yK\xceO\xac\x97\x17\xae\xaf\x9fccQ)/)\xb9c\x19\x9c\xabJ\x93\xa2\xa7\x04\xbe\xadL\xbd\ueabc\xacП\xe1\x1b\xb8\xba\xe7rd\x88\xf7,\xdf.\x19\xed\xe9u\x97\xdb)\xa3\xc5hf\xe8\x06G\x8b\x91\xbe'\xf5\x90BC\xfb\x19]\xa4\xf8Ȗx\x1a\x83\xa3o\x9f\xde8\xed\xbe\xb4\x8fą\x9d\xf1$\x0e\x8d\xfeK$\xfej\f$\xf4~\xadHF!k:\x94\xfeC_\x17\x8b\xbb5\x81\xb1d0\xdb\x15\x82йw\xa1\x85\xf9\xa5\x8b\xc0\xa4\x83Ƒ\b\x1d\xb4\xc9\xdd\x13\b\xed;\x93@O3>\x7f\x99\xbf\xc87\xa6\xe2\x9b_\xff\xa5\xe4\xa2.\xd5\x14̔\x85\xd8r-<ᴏ\x8d9\x8eu\xe0\x0e\xfc\x8a\xd0H\x84*\x94\x8b\xe4\rʊ\x04\xb4/\xd9τ\xb2\xa6\r\xa78\xd1ǵ}\x9c\x84\xde\xf1\xfa\xef\xb4$3L\x98&<\x7f\xa40\xc7O\x8dg$y;\xda\x0e\xa5\xae\x92\xbcTS\xafɲa\x86\aOP\xb4㺿(Qm\xb3N\xae}\xb0#\xa8\xd0yXw\x1f\x06\xe4\x88\uffd8\x8e)\xeb\xbfpv\xa3&\xe7p{Ν\xff\x18w\xc5\xce]:\x02\xb8֍\xcf\xdb\xef\x97.\xb9\xa0\xe7u\x0f\xb3M\xb1\xa1\xf7C_\xb6\xcen\xd3TU8ӏ\x1b\xdd\a\x1c\x01\xab5\xe53\xfe\x13\xad\xc3S\b\x96\xe8α\xea\x8e\xf7\xe4\xfc\xf1!؝t\xc8p\"\xb0\xbf\xa5]f\xb6\xf5s\x99\xa5\xbb\xe4<3K\x93/1\xfa\x8b\xb17\x9e\xe3\\\xbb\x96\x85y\xf8\xce!\xb3\xf6}\xf0*\xcfbv\xc2\xef\x12\xb7y\xe8\xadw\x96\x17>[\x98\xd8\xdf0\xff@%\xfab\xcb5!\xba\xf3\xad\x06EH\xa9\x91\x96\x9e\x04\x82\xeb\xf2\x1a\x84t\xa6\xc2\xfd\x81\x96P\xfa\xb1\xa9\xe6\xdfG:\x8bj=\xa6\xa1c\xa9\xec\xe9\x0e\xf7\xe1k\x91|]{\xda_\xc0\x19\x9f\x11\xd6\xf5qg\xf8{\xdcp\"\x15w\n\x8d+\xb5\xbf}sF5V\x87\x8d\xad=vee\b,\xe1\xe9-mJ\xaa\x90A\xb5\xf3n\xcfr\x16Ï\x87.\xd1\xe2\xd5\x00\u0099\xb8\x9f\xbee\xcaE\xd7\x15{\x01v@\xe1a\xf7f\xfc\x05NjC\x90A\x9f\x1a\xe41\x1e\xe5\xba\nZ\x85:B\xdb\xe9+;\x9c\r\xe4C\x82\xfe\xcc\x18\x9eU\xa7\xc9d\xc0\\\xf4`\xa77\xcd\xfeL\xb3><\xf7/\xe1\xd7߮\xfe\x17\x00\x00\xff\xfff=C\x19\x96(\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Z͒\x1b\xb7\x11\xbe\xefSt\xad\x0f\xb6\xab4d\xa4$\xae\x14o\xd2*Nmbo\xb6DI\x17\x97\x0f\xe0\xa09\x03s\x06\x80\x01\f\xb9\xb4\xe3wO5\x80\x19\xce\x0fH.\xa9\x925\x17i\xf1\xd3\xf8\xf0u\xa3\xbb\xd1`\x96e7L\x8b\x8fh\xacPr\x01L\v|r(\xe9/;\xdb\xfc\xc3΄\x9ao_\xdel\x84\xe4\v\xb8k\xacS\xf5;\xb4\xaa19\xbeŵ\x90\xc2\t%ojt\x8c3\xc7\x167\x00LJ\xe5\x185[\xfa\x13 W\xd2\x19UUh\xb2\x02\xe5lӬpՈ\x8a\xa3\xf1\xc2ۥ\xb7\x7f\x99\xbd\xfcn\xf6\xf7\x1b\x00\xc9j\\\x80V|\xab\xaa\xa6F\x83\xd6)\x83v\xb6\xc5\n\x8d\x9a\tuc5\xe6$\xbc0\xaa\xd1\v8t\x84\xc9q\xe1\x00\xfaQ\xf1\x8f^λ \xc7wUº\xff$\xbb\x7f\x10\xd6\xf9!\xbaj\f\xab\x128|\xaf\x15\xb2h*f\xa6\xfd7\x006W\x1a\x17\xf0@P4ˑ\xdf\x00\xc4}zh\x190\xce=s\xacz4B:4w$\xa2e,\x03\x8e67B;\xcf\xcc\x18\"X\xc7\\c\xc16y\t\xcc\xc2\x03\xee\xe6\xf7\xf2Ѩ\u00a0\r\xf0\x00~\xb1J>2W.`\x16\x86\xcft\xc9,\xc6\xde@\xf1\xd2w\xc4&\xb7'\xcc\xd6\x19!\x8b\x14\x8a\xf7\xa2F\xe0\x8d\xf1\xaa\xa5\xfd\xe7\b\xae\x14v\no\xc7,A4\xceo<\r\xc6\xf7\x93H\xebX\xadǨzS\x03,\xce\x1c\xa6@ݩZW\xe8\x90\xc3j\xef\xb0\xdd\xcaZ\x99\x9a\xb9\x05\b\xe9\xbe\xfb\xdbq>\"a3?\xf5\xad\x92Cr\xdeP+\xf4\x9a\x03\x12\xd2V\x81&ɐr\xac\xfa\x14 \x8e\x04\xbc\xe9\xcd\x0fH\x82\xdc~\xfbY(dz\xa0\xd6\xe0J\x847,\xdf4\x1a\x96N\x19V \xfc\xa0\xf2\xa0\xc2]\x89\x06\xfd\x88U\x18A'\x18\x04\xe9N\x99\xa4\xea4\xe6\xb306\nke\x8d\xf47\\\xe8\xb3\xd8Wn\x90%\xed\xabuE3?B(\x996\xb2\xd7\x05>\xcb\xc0\xfaDJű\xc7\xda\x04\x97\xb0\xa0\x8d\xca\xd1\xda\x13\x86OB\x06H\x1e\x0e\rg)*яi\x015\xbaR\x8c\xa3\x01\xa7\xa0d\x92W\x18t\xe8\f\x93v\x1d-c\xaa\xc2v\xda\xfb\xbd\x1eB\xf9\xd0\xca\xeb\xf5L0\x85\xa1ۗ\xc1\r\xe6%\xd6l\x11\xc7*\x8d\xf2\xf5\xe3\xfdǿ.\a\xcd@\xb4h4N\xb4\x9e9|\xbd\xc0\xd3k\x85\xe1\x9e\xff\x97\r\xfa\x00h\x810\v8E \xb4\x9e\x8b\xe8_\x91GL\x81#a\xc1\xa06hQ\x86\x98D\xcdL\x82Z\xfd\x82\xb9\x9b\x8dD/ѐ\x18\xb0\xa5j*N\x81k\x8bƁ\xc1\\\x15R\xfc\xd6ɶD8-Z1\x87\xd6\xf9\x83h$\xab`˪\x06_\x00\x93|$\xb9f{0HkB#{\xf2\xfc\x04;\xc6\xf1\xa3\xb7&\xb9V\v(\x9d\xd3v1\x9f\x17µ\xe18Wu\xddH\xe1\xf6s\x1fYŪq\xca\xd89\xc7-Vs+\x8a\x8c\x99\xbc\x14\x0es\xd7\x18\x9c3-2\xbf\x11\xe9C\xf2\xac\xe6_\x99\x18\xc0\xed`ى\xa2\xc3\xe7\x83\xe8\x05ꡨJ'\x81EQa\x8b\a-P\x13Q\xf7\xee\x9f\xcb\xf7\xd0\"\t\x9a\nJ9\f\x9d\xf0\xd2\xea\x87\xd8\x14rM\x86O\xf3\xd6F\xd5^&J\xae\x95\x90\xce\xff\x91W\x02\xa5\x03۬j\xe1\xc8\f~m\xd0:R\xddX\xec\x9dOY`E\a\x8a\xfc\x00\x1f\x0f\xb8\x97p\xc7j\xac\xee\x98\xc5?YW\xa4\x15\x9b\x91\x12\x9e\xa5\xad~\"6\x1e\x1c\xe8\xedu\xb4Y\xd4\x11Վ\xfd\xdbRcN\x9a%ri\xaaX\x8b\x18I\xd6\xca\x00\x9b\x8c\x1f2\x95v\x01\xf4%#\xcax\xd09\xb3\xa3\xefMJP\x8bX\xf6\x1cy\x8cw6\x06\xaaj\x18\xa8\xfa\xdf$F\x1a\xd4\xca\n\xa7\xcc\xfe\x10)\xc7&qT;\xf4\xe5L\xe6X]\xb3\xbd;?\x13\x84\xe4\xc4;v&M\xce(H\xf5@\x95,\x14\x1d\xb2\x89:\xe0\xde\xd18\xb2s\x8b.\xbdYy4\xb2\t\t\x87\x1c\x13\xfa\xb9\xe4x\xdb+\xa5*dc6\xb5\xe2g6\xfd\xa8\xa2\xe30\xb8F\x83>\xfe\a7\xab\x95wƎ\tٺ\x8f\x90r\x83S\x89}\xac\xc8\xdd\x1cS\xcdq;\x84\x13!)\t\xf8\xf5\xe3}\x1bvZˊ\xd0'\x91\xa5\xcfO\xd2,\xe8[\v\xac\xb8\x0f\xd4\xe7\xd7NZ\b}\xf7\xeb\x00\xc2\xfb^\xa7\x80\x81\x16\x98\xe3 \ue050\xd6!㱑܍\xc1\xd8\xf7\"\xf8ԣ \xe9;\xc4GR\t0\xf2\xf1\x82ÿ\x97\xff}\x98\xffK\x85}\x00\xcb)\x13\xf2w\x15\xacQ\xba\x17\xdd}\x85\xa3\x15\x069\xdd>pV3)\xd6h\xdd,JCc\x7fz\xf5s\x9a?\x80\xef\x95\x01|b\x94\xf4\xbf\x00\x118\xef\xc2Fk5\u0086\x8dw\x12a'\\\xe9\x81j\xc5\xe3\x06w~\v\x8em\xe8Ą-4\b\x95\xd8`\x9a}\x80[\x9f<\x1d`\xfeN.\xe5\x8f[\xf8&8\x89[\xfa\xf36\xc0\xe8\x12\x84\xbe\xd79\xc0q%s\xe0\x8c(\n<$\xda\x13c\xa1\x80F\xa1\xe0[P\x86\xf6*UO\x84\x17Lz\n\x8e\x18\xf9\x04\xdeO\xaf~\xbe\x85o\x86\x1c\x1cYJH\x8eO\xf0\x8aθ\xe7F+\xfe\xed\f\xde{;\xd8KǞh\xa5\xbcT\x16%(Y\xedC\xbe\xb9E\xb0\xaaF\xd8aUe!\x15\xe3\xb0c{P\xeb#\xeb\xb4*\"\xd3d\xa0\x99q'ӱ\xc8\xc3\xe9C3\xcdO\xda\xefy\xe7\xc5\xe7+\xcf:\xbd_,\xd6?\x93\t\x9f\x98\x7f\x02\x13\xfd\xab\xce\x15Ll\x9a\x15\x1a\x89\x0e=\x19\\\xe5\x96x\xc8Q;;W[4[\x81\xbb\xf9N\x99\x8d\x90EFƘ\x05\xad۹/\xd9̿\xf2\xff\\\xbbq_g\xf9\xd4\xdd{!_\x8e\x02Z\xddίa\xa0ͣ\x9f\x1f\xbb\x8e\U000b0319\xddX&\x9d\xf9])\xf2\xb2\xbdU\xf5\xbcm\xcdxp\xc7L\xee\xbf\xd0\xd9!\x9e\x1bC\x88\xf6Y,8fLr\xfa\xbf\x15\xd6Q\xfb5\xc46ⓜˇ\xfb\xb7_\xf2D5\xe2\x1aOr\xe4\xb6\x10\xbe\xa7\xec\x80*\xab\x99\xce\xc2h\xe6T-\xf2\xd1hʕ\xef9)i-М\xc9\xfe\xde\r\x06\xb7Y{\"\xeb\xee\xc6\\\x94v[ɴ-\x95\xbb\x7f{\x06Dz\x1b\xd8b8\xe80&\x9d\xad,:\x12's\xcdg\xe0Y\x8a\xdf\x12n+\x89\x88\x86\xb6\x98*U\x88\x9cU`}\x9b\x8c\xc5\xca\b\xb3\x95=\x05\x94\xaaG\x8e\xe1\xf6\xab\x8a=\xbc\xde\x17<\x1c\xf7\xb4C\xc8\xc3\xd1-jeD!$\xab\x0e\x1e\xdb_\x1d%\xab\x99\xff+a\xab5\xd3Z\xc8\xe2\"n\xdb\xfa\xd6\x12\x9d\x13\xb2H$\xfa\xfd\xf2\xfb\xa9\xeb\xc0\xc9sr\xde\x05|\x18\x01\x01f\x10\x18\xed\x89T\xb5\xc1}\x16\xb2N\xcd\x04\xa5\x8c\x94\x15\xc6\xd4z\x85\xc0\xb4\xae(\xaf\v\x99d\xca7\xb5պ\\ɵ(b\xe5tʔl\xaa\x8a\xad*\\\x803ͱK[\xf2\xb8\xf7\v\x85g4\xfe\xa17\xb4U\xf7\x99RezW\x83\x02\xe6t3(\x9bz\n%\x83\x8d҂%\xda\xe9pN\x1c\x13u\xdc\xde^bR\xe1\xe4\x9f\xe1 ܙS\x05\x87\xe88\xe25$^\xb1\x83\xfbHG\xf3K\x1d\x8a\xc1_\x1b\xbaS\r\x11f\xe9\xda\xcah\x8cV\xfcfLZ\xdf\x17\x8f:\x0f\x9et\xdc1<\xf4\xa3\xde@\xc1\xb3\xcaR\xbeP~Ia*<\x87E\xdeC\x1a\xe0\xdaG2\xba`\\]\x9a\xa2;\xacvȻ7\x84k\xea6\xaf\xc7B|A\xd9\xf0xHD\x8d]\x91#ډ9\x94]B\x88\xd1\x065KZ\x04\xf8G\x01\xeb\v\xa3_\xdb MXh,r\xef['\x8b\x1f\x8d\t\x9c9\xcch\xfeu\x0e$]\xec\n\xcfs\xfdW\x98\xab*_S1S\x0eYG\x9b\x7f\x1fj\x1f\x06S\x94\x1d\xe4u\x84\x05q\xc8\xfd\x95\x1b\x94\x845\x13\x15r\xe8\x1e\x9f/f>\x01z\x9a\x8c}N\xf2k\xb4\x96\x15\xe7\x9c֏aT\xa8\xbc\xc5)\xc0V\xaaqG\xac\xf2k\x1b\x8f\xd6E1Y*~\x0eɃ\xe2\x1e\x86<\xfe\xe46E\x93PK\xff\x19\xee\"\x8c\xbe\xa8y\xaeHIcR\xae\xa6\x83|\xda\xd7\xc0\x89\x18\xf6\x80\xbbDk{\x82\x13]\x8f\xd1-$\xba&\xbf\a\xe8w\x86Jr*\xa7i\xfb\x922\xbb\xc7\xf6D\xdf\xf7\xfe\xb8\\\xc4v\xc4w\x8dC\xe8\xeaХ\xaaZ\x1f\xe0\x1f\xc9eS\xafА*V\xa9\x8c\x18\x98\xe4}ͥ\x8a\t\x9d\x846\f\aQ\xb1\x1e\x16\v\xe8\xfe\x94;\x05\\X]\xb1}\xb7\x19\x7f\x83\xa3#\x9d~N8\x9c\xab\xd6WQ\xe49\x92\xb7\x9d\xaeTw?ZH\xdfOOg\xfap&\xdb\xf7\xfdݏ\x11>\xcf\n'\xf2\xce\xe1\x8fC\xae1\x90\xe5@¹`\x11\x7f\xacr\xb9\x8f\x1f.\xf3g\xba\xf7${\x93F\x8f\x9c\xf7d\xc7'\xaf~K\xb3\xeaރ\x17\xf0\xfb\x1f7\xff\x0f\x00\x00\xff\xff;\xa8N\xc3\x13&\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xdc=[s\xdb8w\xef\xf9\x15\x98\xf4a\xdb\x19\xcbi\xa6\x97\xe9\xf8\xcd\xf5:\x8d\xfb}\xebx\xec4\xfb\f\x91G\">\x83\x00\x17\x00\xa5h\xdb\xfe\xf7\x0e\x0e.$%\x90\x84d˛-^2\xa6\x80\x03\xe0\xdc\xcf\xc1\x01\xb2X,\xdeц}\x03\xa5\x99\x14W\x846\f\xbe\x1b\x10\xf6/}\xf9\xfco\xfa\x92\xc9\x0f\x9b\x8f\uf799(\xaf\xc8M\xab\x8d\xac\x1fA\xcbV\x15\xf03\xac\x98`\x86I\xf1\xae\x06CKj\xe8\xd5;B\xa8\x10\xd2P\xfbY\xdb?\t)\xa40Jr\x0ej\xb1\x06q\xf9\xdc.a\xd92^\x82B\xe0a\xea\xcd?^~\xfc\xd7\xcb\x7fyG\x88\xa05\\\x11\x05\xdaH\x05\xfar\x03\x1c\x94\xbcd\xf2\x9dn\xa0\xb00\xd7J\xb6\xcd\x15\xe9~pc\xfc|n\xad\x8fn8~\xe1L\x9b\xbf\xf4\xbf\xfe\x95i\x83\xbf4\xbcU\x94w\x93\xe1G\xcdĺ\xe5T\xc5\xcf\xef\bхl\xe0\x8a\xdc\xdbi\x1aZ@\xf9\x8e\x10\xbft\x9cv\xe1W\xbd\xf9\xe8@\x14\x15\xd4ԭ\x87\x10ـ\xb8~\xb8\xfb\xf6OO\x83τ\x94\xa0\v\xc5\x1a\x83\b\xf8\x9fE\xfcN\xc2B\tӄ\x92o\xb8Q\xbb\x1aD<1\x155DA\xa3@\x830\x9a\x98\n\bm\x1a\xce\n\xc4;\x91\xab\x1e\xa40J\x93\x95\x92u\amI\x8b\xe7\xb6!F\x12J\fUk0\xe4/\xed\x12\x94\x00\x03\x9a\x14\xbc\xd5\x06\xd4e\x04\xd4(ـ2,`ٵ\x1e\xef\xf4\xbeNm\xcc6\x8b\v7\x8a\x94\x96\x89\xc0m\xc1\xe3\x13J\x8f>\"W\xc4TLw[\r\xdb#T\x10\xb9\xfc\x1b\x14\xe6r\x0f\xf4\x13(\v\x86\xe8J\xb6\xbc\xb4\xbc\xb7\x01e\x91Uȵ`\xbfG\xd8\xdan\xdcNʩ\x01m\b\x13\x06\x94\xa0\x9cl(o\xe1\x82PQ\xeeA\xae\xe9\x8e(\xb0s\x92V\xf4\xe0\xe1\x00\xbd\xbf\x8e_\x90xb%\xafHeL\xa3\xaf>|X3\x13$\xaa\x90u\xdd\nfv\x1fP8ز5R\xe9\x0f%l\x80\x7f\xd0l\xbd\xa0\xaa\xa8\x98\x81´\n>І-p#\x02\xa5\xea\xb2.\xff.\x12u0\xad\xd9Y\x1e\xd5F1\xb1\xee\xfd\x80\x02q\x04y\xac\xa88\xc6s\xa0\xdc\x16;*\xd8O\x16u\x8f\xb7O_\xfbLɴ'J\x8f7\xc7\xe8c\xb1\xc9\xc4\n\x94\x1b\x87\xacia\x82(\x1bɄ\xc1?\n\xce@\x18\xa2\xdbe͌e\x83\xdfZЖ\xdf\xe5>\xd8\x1b\xd4:d\t\xa4mJj\xa0\xdc\xefp'\xc8\r\xad\x81\xdfP\roL+K\x15\xbd\xb0DȢV_\x97\xeewv\xe8\xed\xfd\x104\xe2\bi\xbd\x16yj\xa0\x18H\x9a\x1d\xc6VA]\xac\xa4\x1a(\x19;d\x88\xa3\xb4\xf0\xdb洈U\x8b\xfb\xbf\xccq\x99m\xff\x1eG[~\xb3+k\x05\xfb\xad\x05T\xa6N\xfc\xe1P_\xa9\x9ej\x1f6\xcbF\xfb\xd4\x1dE\xb4m\xf0\xbd\xe0m\te\xd4\xeb\a\x1b\xcc\xd9\xc6\xed\x01\x144z\x94\t+D\xd6\xfaؽ\x88\xeeWT\xe0T\x01\x11\xd2$\xe01\xe1\xe0\x11&\x10\x03I\x9a`G\x03ubœ[&D\xb4\x9c\xd3%\x87+bT{\x88F7\x96*Ew#\xd8\n\x1e\xc0\x8b\x90\x15\x81xU\xc3Y\x81$\x8f\n\x05\xf1\xf5\xe7E\x15\xd3VQ\x86]>HΊ\xdd\f\xben\x93\x83\x82\xb4z\xd9\xf5;$K\xa8\xe8\x86I\x95\x12\x03\xa9\xb0kϞwjZZ-\xe9\x81\xec۸\xcc\r'\x91UI\xf9<\xc7\x10\x9fm\x9f\xce:\x90\x02\x1dʸ\x15Omo\xbb\x97@\xe0;\x14\xadI,\x93\x90\xb2E\xd3$\x15i\xa46\xe3t\x1fW]\xa4\xef\x1c\xa5~\x9c`\x9a\x83\x9d%Y\xdd5\xaf\x84\x03Q-\x0e\x06\nY\n\xb0ۨ-Q\xbb\xbeJ\xb6\xae\xef(RȒj(\x89\x14\xa33#\xbb\xb4\x1c\xb4\x9f\xabD\xce\xe8\xf4\xd0E\xb7\x7f\xf4x\b\xa7K\xe0D\x03\x87\xc2Hu\x88\xcc\x1c\x94\xba\x96\xa3XGP\x99ЦC\t\xe860\x01\x92XN\xdfV\xac\xa8\x9c\x87a\xd9\x13\xe1\x90R\x82\xb6\xda\x04]\xe6\xdd\xd8&\xc9\x1c\xf9\xfd$Sڣk3b\xb5\x0f/\xa5Q\xba\x96\xa1\x86\xbb\x96Dm\xa7{\x0ft\x8b\xffn\xe4\xe4\xb6\xff\x7f\"6\x18\x93\x13\x98vB\xfe\t\xba\x9f\xd9<=ʷ\x18ၾ$w+\x02ucv\x17\x84\x99\xf0uN\x12(\xe7\xbd9\xfeĴ9\x9e\xe93I\x93#\x13g\"L\x9c\xe2OH\x174\x19O\xdebd\xd3\xe4\xaf\xfdQ\x17\x84\xad\"\xd2\xcb\v\xb2b܀\xda\xc3\xfeI\xaa>P\xe65\x90\x91c\xf5\b\xe6\tLQ\xdd~\xb7.\x8e\xee\x92`\x99x\xd9\x1f\xec|\xe3\x10A\f\xcd\xf3\f\\\x82\xf12SPc\x1cN\xbe\"6\xbb/\xe8T_\xdf\xff|\x18+\xef\xb7\f\xce;\xd8Ȍйv\xbd\xb7\xa3\xfe\xfa|T\x10~A\x1f(\x06U.\xe7rA(y\x86\x9ds]\xa8 \x96>4tΘ^\x01&\x7f\x90Ϟa\x87`\xd2ٜÖ\xcb\r\xae=C\xc2\xf5O\xb5\x01\x0e\xed\x9a|X\xec\xf0d? \"0\x86\xcfe\x03\u05fc($r'閩KB\v\xb8?a\x9bY\xacҟ\xa3\x9f\xfaD\x0e\xf8I;ZZ\x89\xa9\x98\xcfij@\x99\xc9%\xa8k\xdf(ge\x9c\xc8\xc9ȝ\xb8 \xf7\xd2\xd8\x7f0@\xd3\xc8(?K\xd0\xf7\xd2\xe0\x97\xb3`\xd4-\xfc\x9c\xf8t3\xa0\xa0\t\xa7\xe5-\xc2\xfa9?g\xd3,\xb7E\xdc3M\ue10dW\x1cJ2\xa7\xc2\xf4\xae\x9b\xceMT\xb7\x1a\xd3uB\x8a\x05\xda\xcc\xe4L\x1e\xdfR\r\xd0\xfd\xe2I\xfd\x84_\xad\xb1p\xbf\xb8$3\xa7\x05\x94!\xb2\xc4\xec'5\xb0fE\xe6|5\xa85\x90ƪ\xf0<\x8e\xc8T\xac~7DZO\x9e\xf5\xee\xb7\xef\x8b\xe7\x98/XX\x93\xb3\xf0\x10\x8c\xac3p\xe0uw9\xbf\x9f\x85\x95ٌ^\x81\x13f\xbb\x8e$Gǻ\xe6 \xe5\x05\xe8@+\x8e.\xce,uiY\xe2\x11\x1a\xe5\x0fGX\x94#x\xe1X\xd5\xd0[\xbb3\xc15m\xacZ\xf8okiQ\x9a\xfe\x974\x94)}I\xae\xf1\xa4\x8c\xc3\xe07\x9f\x87\xeb\x81ɘ\xb2\xb1SY\xfe\xd9Pnm\xbfU\xe0\x82\x00w\x9e\x80\\\x1d\xf8E\x17d[I\xed\xcc\xf6\x8a\x01\xc7\xf3\x8a\xf7ϰ{\x7fa\xa7\x9f\x9d\xb2\xafd\xde߉\xf7·8P\x18\xd1ᐂ\xef\xc8{\xfc\xed\xfdK\\\xa9LN\xcd\xec6`њ6y\x1c*\x92\xc9\xfa\xae\r8\xa6\x9f\x9b\xef\x92\xf2\xdeɞ\xdam\x16\x8b6R\x9b\xcf\xe9\xbc\xe1\xc8z\x1e\u0088\xa1g\x9cȱ\xcdF\f>\x8f\x16\xf5\xbdu\"W\x06\x94\xcf%:\x1b\x10\xe2\x8f\x17Ff\xa9S\x99\xfebc2\x90\xc6\xfc\xaeE\xf0\f7\xb9\x83\x9b\x9c%\x1e\xe3\xb0Z\xbc\x1c\xe9\xed\xdf~\xef\xe53\xad\xe4ڿ\xfb\x1bym\x87\xba\x90uM\xf7O5\xb3\x96z\xe3F\x06\x9e\xf6\x80\x1c\xf5պEyε\xc8\x1d\x0f\xe1\xf9喙\x8a\tB\x83\xda\x00\xe5\x19\x8a\x92F\xa6rةVQM\x96\x00\"\xa6\xe8\x7f\x04W\xa2f\xe2\x0e' \x1f\xcf\xe0zDt\x9d\xd3ٽ\x894\x89\x94\x8f\x1f\x9c\xc9jdI\xb6\x15(\x180\xc6a\xde\x1d=U!M/eq\x84C\xda\xc8\xf2'MVLi\xd3_\x82&\xadΥ\xf5\x91\xe4\xb3\xeb\xfe\xcaj\x90\xad9'\x82o\xbbi\x06g\xcd5\xfd\xce\xea\xb6&\xb4\x96\xad3\xe6\x86\xd5\xf1TףwK\x99\x89\xc7V\x98\xbf1Ғ\xa0\xe1`\x80,a\x95>\xefM\xb5B\n\xcdJP\xa1J\xc1\x91\x8dI+\x98+\xcax\x9b:%J\xb5c#`q\xab\xd4I\x01\xf0\x177\xb2\x97w\xac\xe4v\x88\xa0̽\xe3A\x1a\x10\xb6\"\xcc\x10\x10\x85\xc58(\xa7\x92q\n\x8f\fD\r\xcb\xd5sy\n\xdc6\x10m\x9d\x87\x80\x05\n$\x13\x93)\xb7~\xf7O\x94\xf1s\x90\xcdr\xde'\xa9\x1e\x81\x96\xa7\xe4h~\xed\r' t\xab\xf0\xf0\xdf\xe9\x8e-\xe3yk\xb6\x94#\x9c\xb6\xa2\xa8\x00\x95\x90\x18\xea\x06\a\x9e\tm\x80\xe6\xf2\x82\xf5\x8aZ!\x98X\xe7\xd1.;\x11\xda5\x87\ua954\x1c\xe8\xf8)d\xd7,\xae\xdf@\x13\xfd\xdaM\xf3BM\xd4\x11\xc1\x1d\x9b#\x1d\xb2)j\x95\x16\xa1\xc6@\xdd8\x91\x93D\xb5\xa2o]Π\x88\x8e\t\xc3\xfd*^3\xbef\x82e\xd0v@\xd7;\xc1L\xdfy\xb4 \xce\xea<\xda\t\xa2;pJ\x86\xedn\x00\xc0\nh\x88Cp\xed\x91k\x8ep$\x97@hYB\xe9r\x97\xd6\x15\xf1a\x89+|\x1b)nH\xee\xeexO0\x8b\xb2\xa1\r\x82N\xccê\r,Z\xf1,\xe4V,0\x18\xd7G\xeb\x90\x13\xb3T/\x9dޜ\xac\x8c\xe6\xf5K\xbe\x9a\x9e\xd3BC~\xcd\xe7\xa9\xe0?\x9dA\xcbd\xf3\xcdQ\t\x8f).\x98\xd3k\xae\x00{\xe4\xc7\xd9UL\xcd?1\xd8\x1fJ߸b\xe9\x17\x95\xc5ݥA\xf5\x9c\xc2m\x05\xa6\x02\x15J\xb3\x17X\x92^N\x9e\x90v\xc1K\xac\x93\xb3L\x15\\dW\xfe\xb9W9\x87\xd1M\xcb\xf9\x85\xe5m\xda\xf2d8l$\x8a\xd8!geՏ\xa5=\x86\x9c\xea\x8bl<\xf6+-\x86\xf5\x85\xb1\n\"\x14\x18\xca0\xb3\xa7qj\xbfXX\xda;\xdf\x1f\x96S`\xfe/,\xff\x0f/=̨\x94\xc8Gcn\x95fDb\x02V\x82\xc1zh\xec\xea+|?_\xe8\xfbc\xe1\xd4@\xfd\xa5\xf1\x123\xea\xc2f\xa05\x01g\xaf\xde\x04\xadA\xab\x9d+\x10\xed\x80\xcf\x19\xda\xf1ׅ\xbb\x05\x11\xc0\xa4\xf8\xf5k\x05A|}\xf5>\xd3\xe4\x9fI%\xdbDU\xdf\x04\xcaf\xaa;\xe67<(\xf4\xf0\a\n`\xe8\xe6\xe3\xe5\xf0\x17#}\xd9\af\xd1\x12\x800(\xea2\xb3L\x94l\xc3ʖ\xf2 \xb5\xdd\x1d\x02\xc7@\x1d\x9f%\xa0IE\x04\xe3\x8e\x01\xc3\xf8\x01Ñ/\x8d;\x969Z\xc5M\xfb\xa2y\xd5!'ׄ\fk>F\xac\xe1\xb1\xc7\x17\xafR\x05\xfb\x87\xd4z\x1c_\xe1\x91\x13I\xccTs\x9cPÑY,\xf6\xe2\xf3\x96\x9c*\x8dcb\xee\xb3Ud\xbc~\x1dF\x16~\xe6k.\x8e\xc1\xce\xd9\xeb+ް\xaa\xe2mj)2+(^\xaf\x142/\xfa<\xa9\x14`>`\x19\xaf\x82\x98\xad}xQ@sҖfk\x1a\x8e\xa9d\x98\xa5N\x9e\x98\xbdY\xad\u009bU(\xbcm]\xc2$\x17M\xfexL\xe5A\x8c\x93~\xa1M\xc3\xc4\xfa\x90)rYg\x92m\xe6Y\xe6~o!\x03\x9e\xe9\x873]t8\x12\xfa\xba\xeb҉H2\xa4-\x990\xf2\x92\\\x8b\x9d\x87\x9b\x80\xd3\v\x1f\x854\a\x17\xd9첶\x8c\xf3\xfem-\x04;\r\xcaߙԴv\xab\x1a\xf3\xf6\x93t\x95j\xe0\x94\x9f\x148~ك\xd1ώ\xbe\xa5\xe7_\xb7ܰ\x86\x83\xf5\xe86\xacL\xde!3\x15\xec\"\x92\xff&\xf1\x86\xd4r\x87\x90\xbe]\xf4\x9eQ\xec\x9eq\x926\xbf\xc9\x13\xb6\x97Q\xcc~\\\x11{\x06\xcdrE\xf1\r\x8b\xd5߰H\xfd\xad\x8b\xd3g8k\xe6\xe7\xe3\x8a\xd0O>\x81\tG\xfd\xf7\xb2\x84\a\xa9\xcc\\p\xf2\xb0\xdf?q\x92\xda\v\xd8$/\x89\b]\x13\xbb\xc4\x10Ç\x17\xa7m*}\xe8\x19\xdc\xe9_di\xd76w\xc6\xf2\xb8\xd7\xfd\xe0\xae\xf2\n\x14\b\xf7\xcc\xc7\x7f>}\xb9\x8f\xf0S>\xaf\xf7\x8c\xf7\x9e\x97p\x1eL\xe9\x91\xe3\x8f\xe6|1\x93\xc3\x16\xfa\x00\xaf|.B\x1b\xf6\x1f\xf8\xaa\xdb\v\xd2A\xd7\x0fw\b#\xf8i\xf8L\\\xac\xa2\x88'\x96K\xb0\x16+\xa2jT,\xeeV\x03\x88Ê\xdf\xfe3JP\xba'\xb3\x82\xc5d\xa1\xc6\xcb\n\xdeÝ[\xc7\xd8,\x9f\xac\xd3(vD:\x8e\xac\x98*\x17\rUf\x87l\xa3/\x06k\bff*\x9d3\xaaX\x0f\x9f\x01K\xa27\xbc\xfe\x85g\x91\xbbfxڻ\x8f\xbbS\xd61~\xffd\xf6\xe6\xc9+\xaec\xdcb/\x10S\x89\xcf\xc9\x02\x93WK\x93yM\xf4\xf0\xed\xa4\xb4\xcbc\x1c=\xad\xe7l\x14\x1dRM\t0v<\xaa:-h\xa3\xabēK/\xd3u\xf8\x1a\x99\xa1\xa6}\xc9&\x1d\x80\xc1>YQ\xf5\xb4\xd5\x16\x82>\v\xdbFi\xc5a)\xddnm\xb3\xab{a\xfc\xa2\x97)x\x9b#\xe1\xcc\xe7\\N~\xc8šgD\xfd`\xf6˪\xb6CL\x9dp\x18<\xeb\xdae\x14\x19O;\xb1\x99π\xe4\x19\x8c\x13\x9e\xfe@|\xe5\xe2\x8a$_\x04\xc9|\xf5\xe3\x0fE\xf4\x84V\xd3E\x05e\xcb\xe1\xd47\xff\x9ez\xe3\xe7_\xfd\v\xb3e\xbc\xfbg\x91\xdd3\xd0\xd6g\x1e\xbe/\xe8)\xe1!\xf7)9\xe6\xf0ap\xe0\x9e\x17+\xdcK\x94E\x01Z\xafZ\x1e\xaa\x94\n\x05\xd4@\x19\xba3\x1dW|T\x9dM\xdbpIKP7R\xacX\xe2\x84d\x80\xd6\xff\x1at\xde\xe3\xd9\x02?\xb6\xaa{\xdaq\xf2Y\xbc\x17i\xae\x86*\xca9\xf0O\x8c\x83\xfeYn\x85]W\x86@>\xa4\xc6\xf5\xeee\x15\xad\xb2f}GD[/\xad\x93\vƌ\a\x8b+\xa9\xa6+\xa4\x1dޙ0\xb0\x86T|\xbdU\xcc\xc0SC\x95\x06\\Q\xc6\x0e~\xdd\x1b\xe2\xa2\xcf\x15\xa7kW\nW\xb2\x82\x1a\x88\x06\x18g\x18[>\x8e\xd7\b\x8b\xef\xb02I\x8e$\xbd\xb2\x85z\xecJƨX\x8f=/\x9a0\xd5\xc9\aF\x9dE.hc\xf0\x02\f\xd2\x11\x89h<\f|\xb4w\xef\x8d\xd1\x01\xd8qN\xf3e̾`N\x1bZ'\xa2\x84y\xbdss\b\x06\x9f\x05Ve\xaf\xee\xae\xff\xc0b,\xb0#[\xaac1u\xd2\xf7\xee`;0\xe8\xaa[\xd0P\x12\u0600 V\x14)\xe3PNq\xeaWL$\xab\r\xa8\x9ft\x84\x83\x95\x80\x96ş\fU&.\xfdЏYIUSsEJj`aG\x9f溥\x9fIU\xea\xc4\xe3@\xbc\xd9\xe6ţ\b\xd7n\xac\xf5s\xf7\xd1jК\xaeC\x10\xba\x05\x05d\r\xc2\xe2=\xe6\x16\x93\x1eS\xb8\xd2\xe7\x8dE,-\xb5(\xa4\x85i\xa9\x9f\xc0\xb9p\xf1\xf44\xbcO\x8cQ\xeczTE\xa7U\x85\xbf<\xf8\bT\xef?w}\x80\x8bO\xfd\xbe>I\xecv\xec\xceF\xa8+\xf0\xc4\a\x8f\r\x8b\x91uJ\xa6\x8dę\x8f2'\x95\x94\xcfYn\xf6\xe7رK'1\xe1X\t\xafL.ekz~\x8eGxb\x99\xf8\xfc\xe7+\xdb\x17\x84y\xed.P\x8d\xe5V\xf3<\xbd\xcf\x03H1\xbc\x95\x86\xf2`d,_\xc6\x0e\xd5\xc4\x03\x02O\xe1\xf1d\xcew\x17\xfb\x90\xf7^e\xef`W\xddS\x9e^\x13t\xd7\xc7\xc7Ҫ>\xeb\x97\x04\x12_\x01\xed|\x92\xb17\x17\xe7\xec\x1fB\xfd\x84\x8b\xca\xc0\xf1\xe7\xae\xf7\x18\x1e\xdd2\x9d\xc3\f\"\x1di\x12\f>L\x15%ㄥOx\xa9ME\xf5\x9c{\xfa`\xfbD\xb7\xa3g\xae\xa2\x13\xfa8\"\x95\xe9{\xae\vr\x0f\xdb\xc4W\x87,<\xfdB\xa9Jt\xb9\x13\x0fJ\xae\x15\xe8C\xa6[\xe0}F&֟\xa4z\xe0횉/\xe3\x95\xdfS\x9d\x1f\xa82\xcc2\xad[Ob\xecM\xb0q\x89\xdf\xe6G\x8f\xff\xc0\x04\xe5\xec\xf7\x94.\xef\xff87Ä\xbek<\xf2N\xb1P\x01\xf1s\n\xd0k\xe8\x9ft\xcf\xfc\x84y/ɽL\x8a\xb1? fC\xa0L\x93%h\xb3\x80\xd5J*\xe3\xf2\xf7\x8b\x05a\xab\xe0 Y\r\x81q\xa2{͞\xb0T\xe2=\x1e\xbd\x05\x87e\xe5S\x89\n\xad\x0e\x86\x9c5ݹ\x8c$-\n\x1b\x13\xc0\amh*6y\x91\x9e\xc6P\xd5\xcbJ\x8e\n\xb9\xeb\xf7\x8f9\xbe\xa8>\x10\x9cC\x1d^gw\x06\x9d\x8f\x9di\r^\xcb \xdab\xef\x14eB\x9c\x1a\xbb\x1b\x0f\xbb\xf3L\xcd\xd7\beL=\xfa\xfd\r\x1e\xe2\xf6\a\xac\xbe\x93%[QQ\xb1\x1e\xbd\xd0V)ٮ\xab\xc0\x9bc\x0e\x11)[\x8c\x9c\x1bT\x05:\xfc\xc7!\xa6U\xa2wh\xe7k,ƴt\\\uee0f\xf2\x02E\xad\xba\x8b-\x9d\xaa\x9a\xb0\xf9\xd9Y\xc2\x11\x88\xb3\xb6?\x01\x91\xea\x9d(&\xaf\xe0\xf8@\x9bM\xdc՝\xc2P\x12\tQ\x1b\xbf\x1a\x12\"\xc41$\xf4}\x89.\xe2\xf9a02棜\x88\x8ei'\x06\xb78\rj~\xd3}'h\xe8\xee\x1c\x87\x0e=\b\xfeNJ\xbb\r \x1c\x13\xf9\xe2\xdc\xe9\xb8\xf7ǍX7\xd1ۺ=9v\xfd\xb6\ac\xef\n\xa4\x8db\xbbiB\xbc\xf9\xf7l\x95\x92\x17\xf7\xbf3-9\xfc\xc3\xc1\xafo|\x95qK\x95`b}\x12F~\xf5c\x13\xf1\xbc\a{Έ>\xac\xfc\xd5b\xfa\xa4Y:\xf8\x88\f^\xf6\xf0\xecg\xf2_\xfe/\x00\x00\xff\xffP\a\xb5\x16Cm\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=Ks\xdc8sw\xff\n\x94sؤJ#ǕG\xa5ts\xb4v\xac|\xdf\xca*\xc9\xf1\x9e1d\xcf\x10\x9f@\x80\v\x80\x1a\xcf&\xf9\xef)4\x1e|\fHbF\x1a\xednjqQ\x89$\x1a@\xbf\xbb\xd1\xc0\xacV\xab7\xb4a\xdf@i&\xc5\x15\xa1\r\x83\xef\x06\x84\xfdO_>\xfe\x9b\xbed\xf2\xdd\xd3\xfb7\x8fL\x94W\xe4\xba\xd5F\xd6\xf7\xa0e\xab\n\xf8\x116L0äxS\x83\xa1%5\xf4\xea\r!T\bi\xa8}\xac\xed\xbf\x84\x14R\x18%9\a\xb5ڂ\xb8|lװn\x19/A!\xf00\xf4\xd3?^\xbe\xff\xd7\xcb\x7fyC\x88\xa05\\\x11\x05\xdaH\x05\xfa\xf2\t8(y\xc9\xe4\x1b\xdd@aan\x95l\x9b+ҽp}\xfcxn\xae\xf7\xae;>\xe1L\x9b\xbf\xf4\x9f\xfe\x95i\x83o\x1a\xde*ʻ\xc1\xf0\xa1fb\xdbr\xaa\xe2\xe37\x84\xe8B6pEn\xed0\r-\xa0|C\x88\x9f:\x0e\xbb\xf2\xb3~z\xef@\x14\x15\xd4\xd4͇\x10ـ\xf8pw\xf3\xed\x9f\x1e\x06\x8f\t)A\x17\x8a5\x06\x11\xf0?\xab\xf8\x9c\x84\x89\x12\xa6\t%\xdfp\xa1v6\x88xb*j\x88\x82F\x81\x06a41\x15\x10\xda4\x9c\x15\x88w\"7=H\xa1\x97&\x1b%\xeb\x0eښ\x16\x8fmC\x8c$\x94\x18\xaa\xb6`\xc8_\xda5(\x01\x064)x\xab\r\xa8\xcb\b\xa8Q\xb2\x01eX\xc0\xb2k=\xde\xe9=\x9d[\x98m\x16\x17\xae\x17)-\x13\x81[\x82\xc7'\x94\x1e}Dn\x88\xa9\x98\xee\x96\x1a\x96G\xa8 r\xfd7(\xcc\xe5\b\xf4\x03(\v\x86\xe8J\xb6\xbc\xb4\xbc\xf7\x04\xca\"\xab\x90[\xc1~\x8d\xb0\xb5]\xb8\x1d\x94S\x03\xda\x10&\f(A9y\xa2\xbc\x85\vBE9\x82\\\xd3=Q`\xc7$\xad\xe8\xc1\xc3\x0ez<\x8f\x9f\x90xb#\xafHeL\xa3\xaf\u07bd\xdb2\x13$\xaa\x90u\xdd\nf\xf6\xefP8غ5R\xe9w%<\x01\x7f\xa7\xd9vEUQ1\x03\x85i\x15\xbc\xa3\r[\xe1B\x04J\xd5e]\xfe]$\xea`X\xb3\xb7<\xaa\x8dbb\xdb{\x81\x02q\x04y\xac\xa88\xc6s\xa0\xdc\x12;*\xd8G\x16u\xf7\x1f\x1f\xbe\xf6\x99\x92iO\x94\x1eoN\xd1\xc7b\x93\x89\r(\xd7\x0fY\xd3\xc2\x04Q6\x92\t\x83\xff\x14\x9c\x810D\xb7\xeb\x9a\x19\xcb\x06\xbf\xb4\xa0-\xbf\xcb1\xd8k\xd4:d\r\xa4mJj\xa0\x1c\x7fp#\xc85\xad\x81_S\r\xafL+K\x15\xbd\xb2DȢV_\x97\x8e?v\xe8\xed\xbd\b\x1aq\x82\xb4^\x8b<4P\f$\xcdvc\x9b\xa0.6R\r\x94\x8c\xed2\xc4QZ\xf8msZĪ\xc5\xf1\x9b%.\xb3\xed\xdfco\xcbovf\xad`\xbf\xb4\x80\xcaԉ?\x1c\xea+\xd5S\xed\xc3f\xd9hL\xddID\xdb\x06\xdf\vޖPF\xbd~\xb0\xc0\x9ce|<\x80\x82F\x8f2a\x85\xc8Z\x1f\xbb\x16ѽE\x05N\x15\x10!M\x02\x1e\x13\x0e\x1ea\x021\x90\xa4\t~h\xa0N\xccxvɄ\x88\x96s\xba\xe6pE\x8cj\x0f\xd1\xe8\xfaR\xa5\xe8~\x02[\xc1\x03x\x16\xb2\"\x10\xafj8+\x90\xe4Q\xa1 \xbe\xfe\xb8\xa8b\xda*ʰ\xca;\xc9Y\xb1_\xc0\xd7\xc7d\xa7 \xad^v\xfd\n\xc9\x1a*\xfaĤJ\x89\x81T\xf8iϞwjZZ-遌m\\悓Ȫ\xa4|\\b\x88\xcf\xf6\x9b\xce:\x90\x02\x1dʸ\x14Omo\xbb\xd7@\xe0;\x14\xadIL\x93\x90\xb2E\xd3$\x15i\xa46\xd3t\x9fV]\xa4\xef\x1c\xa5^\xce0\xcd\xc1ʒ\xac\xee\x9aW\u0081\xa8\x16\a\x03\x85,\x05\xd8eԖ\xa8ݷJ\xb6\xee\xdbI\xa4\x905\xd5P\x12)&GFvi9h?V\x89\x9c\xd1顋n\xfd\xe8\xf1\x10N\xd7\xc0\x89\x06\x0e\x85\x91\xea\x10\x999(u-G\xb1N\xa02\xa1M\x87\x12\xd0-`\x06$\xb1\x9c\xbe\xabXQ9\x0fò'\xc2!\xa5\x04m\xb5\t\xba\xcc\xfb\xa9E\x92%\xf2\xfbA\xe6\xb4G\xd7\x16\xc4j\f/\xa5Q\xba\x96\xa1\x86\xbb\x96Dm\xa7{\x0ft\x8b\x7fn\xe4\xec\xb2\xff\x7f\"6\x18\x93\x13\x98vF\xfe\t\xba\x9f\xd9<=ɷ\x18ၾ$7\x1b\x02uc\xf6\x17\x84\x99\xf0tI\x12(\xe7\xbd1\xfe\xc0\xb49\x9e\xe93I\x93#\x13g\"L\x1c\xe2\x0fH\x174\x19\x0f\xdebd\xd3\xe4\xaf\xfd^\x17\x84m\"\xd2\xcb\v\xb2a܀\x1aa\xff$U\x1f(\xf3\x12\xc8ȱz\x04\xf3\x04\xa6\xa8>~\xb7.\x8e\xee\x92`\x99x\x19wv\xbeq\x88 \x86\xe6y\x01.\xc1x\x99)\xa81\x0e'_\x11\x9b\xdd\x13t\xaa?\xdc\xfex\x18+\x8f[\x06\xe7\x1d,dA\xe8\\\xfb0ZQ\x7f~>*\bo\xd0\a\x8aA\x95˹\\\x10J\x1ea\xef\\\x17*\x88\xa5\x0f\r\x1fg\f\xaf\x00\x93?\xc8g\x8f\xb0G0\xe9l\xcea\xcb\xe5\x06\xd7\x1e!\xe1\xfa\xa7\xda\x00\x87vN>,vx\xb2\x0f\x10\x11\x18\xc3粁k^\x14\x12\xb9\x93t\xcb\xd4%\xa1\x05ܟ\xb0\xcc,V\xe9\x8f\xd1O}\"\a\xfc\xa0\x1d-\xad\xc4T\xcc\xe745\xa0\xcc\xe4\x12Եo\x94\xb32\x0e\xe4d\xe4F\\\x90[i\xec\x1f\f\xd042ʏ\x12\xf4\xad4\xf8\xe4,\x18u\x13?'>\xdd\b(h\xc2iy\x8b\xb0~\xce\xcf\xd94\xcbm\x11\xf7L\x93\x1ba\xe3\x15\x87\x92̡0\xbd\xeb\x86s\x03խ\xc6t\x9d\x90b\x85639\x92ǷT\x03t?{P?\xe0Wk,\xdc\x1b\x97d洀2D\x96\x98\xfd\xa4\x06\xb6\xac\xc8\x1c\xaf\x06\xb5\x05\xd2X\x15\x9e\xc7\x11\x99\x8aկ\xe68\xf6ɳ\xde\xfd\xf6}\xf5\x18\xf3\x05+krV\x1e\x82\x91u\x06\x0e\xbc\xee.\x97׳\xb22\x9b\xf1U\xe0\x84\xc5O'\x92\xa3ӟ\xe6 \xe5\x19\xe8@+\x8e.\xce\"uiY\xe2\x16\x1a\xe5wGX\x94#x\xe1X\xd5Л\xbb3\xc15m\xacZ\xf8okiQ\x9a\xfe\x974\x94)}I>\xe0N\x19\x87\xc1;\x9f\x87\xeb\x81\xc9\x18\xb2\xb1CY\xfey\xa2\xdc\xda~\xab\xc0\x05\x01\xee<\x01\xb99\xf0\x8b.Ȯ\x92ڙ\xed\r\x03\x8e\xfb\x15o\x1fa\xff\xf6\xc2\x0e\xbf8d_ɼ\xbd\x11o\x9d\x0fq\xa00\xa2\xc3!\x05ߓ\xb7\xf8\xee\xeds\\\xa9LN\xcd\xfcl\xc0\xa25m\xf28T$\x93\xf5]\x1bpL?7\xdf%当=\xb7\xda,\x16m\xa46\x9f\xd3yÉ\xf9܅\x1eC\xcf8\x91c[\x8c\x18|\x1e-\xea{\xebDn\f(\x9fKt6 \xc4\x1fό\xccR\xbb2\xfd\xc9\xc6d \x8d\xf9]\x8b\xe0\x05nr\x1b79S<\xc6a\xb5x9\xd2\xdb\xff\xf8\xbd\x97ϴ\x92k\xff\xef/\xe4\xa5\x1d\xeaB\xd65\x1d\xefjfM\xf5\xda\xf5\f<\xed\x019\xea\xabm\x8b\xf2\x9ck\x91;\x1e\xc2\xfd\xcb\x1d3\x15\x13\x84\x06\xb5\x01\xca3\x14%\x8dL\xe5\xb0S\xad\xa2\x9a\xac\x01DL\xd1\xff\x1e\\\x89\x9a\x89\x1b\x1c\x80\xbc?\x83\xeb\x11\xd1uNg\xf7:\xd2$R>>p&\xab\x91%\xd9U\xa0`\xc0\x18\x87yw\xf4T\x854\xbd\x94\xc5\x11\x0ei#\xcb\x1f4\xd90\xa5M\x7f\n\x9a\xb4:\x97\xd6G\x92\xcf\xce\xfb+\xabA\xb6\xe6\x9c\b\xfe\xd8\r3\xd8k\xae\xe9wV\xb75\xa1\xb5l\x9d17\xac\x8e\xbb\xba\x1e\xbd;\xcaLܶ\xc2\xfc\x8d\x91\x96\x04\r\a\x03d\r\x9b\xf4~o\xaa\x15RhV\x82\nU\n\x8elLZ\xc1\xdcP\xc6\xdb\xd4.Q\xaa\x1d\x1b\x01\x8b\x8fJ\x9d\x14\x00\x7fq={y\xc7J\xee\x86\b\xca\\;n\xa4\x01a\x1b\xc2\f\x01QX\x8c\x83r*\x19\x87\xf0\xc8@\u0530\\=\x97\xa7\xc0m\x03\xd1\xd6y\bX\xa1@21\x9br\xeb\x7f\xfe\x892~\x0e\xb2Y\xce\xfb$\xd5=\xd0\xf2\x94\x1c\xcdϽ\xee\x04\x84n\x15n\xfe;ݱc,q\x85o\x13\xc5\r\xc9\xd5\x1d\xef\tfQ6\xb4AЉyX\xf5\x04\xabV<\n\xb9\x13+\f\xc6\xf5\xd1:\xe4\xc4,\xd5s\x877'+\xa3e\xfd\x92\xaf\xa6\x97\xb4А_\xf3y*\xf8Og\xd02\xd9|sT\xc2c\x8e\v\x96\xf4\x9a+\xc0\x9ex\xb98\x8b\xb9\xf1g:\xfbM\xe9kW,\xfd\xac\xb2\xb8\x9b4\xa8\x9eS\xb8\xab\xc0T\xa0Bi\xf6\nK\xd2\xcb\xd9\x1d\xd2.x\x89ur\x96\xa9\x82\x8b\xec\xca?G\x95s\x18ݴ\x9c_Xަ-O\x86\xc3F\xa2\x88\x1drVV\xfdX\xdacȩ\xbe\xc8\xc6c\xbf\xd2bX_\x18\xab B\x81\xa1\f#{\x1a\xa7\u058b\x85\xa5\xbd\xfd\xfda9\x05\xe6\xff\xc2\xf4\x7f\xf3\xd2ÌJ\x89|4\xe6ViF$&`%\x18\xac\x87Ʈ\xbe\xc2\x7f\xe7\v}\x7f_85P\x7fi\xbc\xc4L\xba\xb0\x19hM\xc0\x19՛\xa05h\xb5s\x05\xa2\x1d\xf09C\xdb\xffC\xe1NA\x040)~\xfdZA\x10__\xbd\xcf4\xf9gR\xc96Q\xd57\x83\xb2\x85\xea\x8e\xe5\x05\x0f\n=\xfc\x86\x02\x18\xfa\xf4\xfer\xf8\xc6H_\xf6\x81Y\xb4\x04 \f\x8a\xba\xcc,\x13%{beKy\x90\xda\xee\f\x81c\xa0\x8e\xcf\x12Ф\"\x82qǀ\xa1\xff\x80\xe1ȗ\xc6m\xcb\x1c\xad\xe2\xe6}ѼꐓkB\x865\x1f\x13\xd6\xf0\xd8\xed\x8b\x17\xa9\x82\xfdMj=\x8e\xaf\xf0ȉ$\x16\xaa9N\xa8\xe1\xc8,\x16{\xf6~KN\x95\xc611\xf7\xd9*2^\xbe\x0e#\v?\xcb5\x17\xc7`\xe7\xec\xf5\x15\xafXU\xf1:\xb5\x14\x99\x15\x14/W\n\x99\x17}\x9eT\n\xb0\x1c\xb0LWA,\xd6><+\xa09iI\x8b5\r\xc7T2,R'O\xcc^\xadV\xe1\xd5*\x14^\xb7.a\x96\x8bf_\x1eSy\x10㤟h\xd30\xb1=d\x8a\\֙e\x9be\x96\xb9\x1dMd\xc03\xfdp\xa6\x8b\x0e'B_w\\:\x11I\x86\xb4%\x13F^\x92\x0fb\xef\xe1&\xe0\xf4\xc2G!\xcd\xc1A6;\xad\x1d\xe3\xbc\x7fZ\v\xc1\u0383\xf2g&5\xadݬ\xa6\xbc\xfd$]\xa5\x1a8\xe5'\x05\x8e_F0\xfa\xd9\xd1\xd7\xf4\xfc\xeb\x96\x1b\xd6p\xb0\x1e\xdd\x13+\x93g\xc8L\x05\xfb\x88\xe4\xbfI\xfb\xa5\x05\xb5'\xf2\tK\x18\xbc\xf7֝U\xf0\xeaF\xdb\x183(@\xaf\x8c\xa76\x15\x0eB\x99NA\x91\x0f\xc2\xf9\x12\xe3\xf9`\x1f\xab\xf9\xbaPͪs\x1b\x85%ǘ\xe8.d\xec\x9d\xe8\xb6\xe4\xf6\xe7\x16\xf5\x9f7p;>t[\xf4\x95\xf2\xfd\xd9ߨX\xff\x94\"\xfd\xbc\xed\xa0Ţ\xfcs\x05rK\xa1\\\xb6\xf7\x9aWt\x7f\xdc&\xea\x19\x8b\xec\xcfQ\\\x9f\x89\xa9\x9cb\xfa\xe3\xf0\xf4\n\xc5\xf3\xafZ4\xffZ\xc5\xf2\xd9E\xf2Y\xfb\x98ٛV\xb9ی'V}/\xef\xba\xcf\x17\xbdg\x14\xbbg\xec\xa4-/\xf2\x84\xe5e\x14\xb3\x1fWĞA\xb3\\Q|\xc5b\xf5W,R\x7f\xed\xe2\xf4\x05\xceZx}\\\x11\xfa\xc9;0a\xab\xffV\x96p'\x95Y\nN\xee\xc6\xdf'vR{\x01\x9b\xe4%\x11\xe1\xd3\xc4*1\xc4\xf0\xe1\xc5i\x8bJoz\x06w\xfa'Yڹ-\xed\xb1\u070f>?8\xab\xbc\x01\x05\xc2]\xf3\xf1\x9f\x0f_n#\xfc\x94\xcf\xeb=\xe3\xd1\xf5\x12\u0383)=r\xfc֜/fr\xd8B\x1f\xe0\x85\xf7Eh\xc3\xfe\x03ou{F:\xe8\xc3\xdd\r\xc2\b~\x1a^\x13\x17\xab(\xe2\x8e\xe5\x1a\xacŊ\xa8\x9a\x14\x8b\x9b\xcd\x00\xe2\xb0\xe2\xb7\x7f\x8d\x12\x94\xeeʬ`1Y\xa8\xf1\xb2\x82ww\xe3\xe615\xca'\xeb4\x8a=\x91\x8e#+\xa6\xcaUC\x95\xd9#\xdb\xe8\x8b\xc1\x1c\x82\x99\x99K\xe7L*\xd6\xc3k\xc0\x92\xe8\r\xb7\x7f\xe1^\xe4\xbe\x19\xee\xf6\x8eqw\xca<\xa6ϟ,\x9eg\x91\x0e\xc0`\x9d\xac\xa8z\x1e\xe4\x0e\x82\x8f\x19\x96\x8dҊݒ\x1a\x1c\xb8?i\xc5\xf8E/{\xfb:e:\x99Wl\x9d|\xb9\x96Cτ\xfa\xc1\x1d\t\xab\xda\x0e1uB\x81\xceb\xb8\x9dq\xf0c>\xb1\x90y5S\x9e\xc18\xe1:&\xc4W.\xaeH\xf2\x96\xa6̛\x98~SD\xcfh5]TP\xb6\x1cN\xbd\x87\xf5\xa1\xd7\x7f\xf9&\xd60Z\xc6]\xac\x16\xd9=\x03m=\xacᝯ\x9e\x12\x1er\x9f\x92SA8&lܕ\x8f\x85\xbb\x1d\xb8(@\xebM\xcbC\xe5h\xa1\x80\x1a(\xc3\xe7L\xc7\x19\x1fU\xfb\xd86\\\xd2\x12\x94s\xc9\x16\xd0\xfa_\x83\x8fG<[\xe0\xc3Vu\xd7\xed\xce^U\xfa,\xcd\xd5PE9\a\xfe\x89q\xd0?ʝ\xb0\xf3\xca\x10ȻT\xbf\xdeY٢U֬\xef\x89h\xeb5(\xa2\xc1\x98\xe9\x04\xdeF\xaa\xf9S+\x0e\xefL\x18\xd8B*\xe7\xb9S\xcc\xc0CC\x95\x06\x9cQ\xc6\n~\x1euq\x19\xc1\r\xa7[W\x9e\\\xb2\x82\x1a\x88\x06\x18G\x98\x9a>\xf6\xd7\b\x8b\xef\xb1ZTNlDd\v\xf5\xd41\xb9I\xb1\x9e\xba\xf29a\xaa\x93\x97>;\x8b\\\xd0\xc6\xe0\xa1D\xa4#\x12\xd1x\x18x\x91\xfa\xe8\xde\xe7\x01\xd8iN\xf3GK|\x11\xb36\xb4ND\t\xcbz\xe7\xfa\x10\f^ծ\xca^-t\xff\xd2\xdbX\xf4LvT\xc7\x03.I\u07fb\x83\xed\xc0\xa0\xabnACI\xe0\t\x04\xb1\xa2H\x19\x87r\x8eS\xbf\xe2\xe6\x9ez\x02\xf5\x83\x8ep\xb0:۲\xf8\x83\xa1\xcaĩ\x1f\xfa1.\x86\xbb\"%5\xb0\xb2\xbdOs\xdd\xd2WW+ub\x89\x06\x9e6\xf6\xe2Q\x84\xa3\x90\xd6\xfa\xb93\xc25hM\xb7!1\xb8\x03\x05d\v\xc2\xe2=\xee\xf7$=\xa6p\xcc\xda\x1b\x8bAb\x80\x16\xa6\xa5~\x00\xe7\xc2Ŋ\x96pg\x0f \xc5\xf0V\x1aʃ\x91\xb1|\x19?\xa8f.uy\b\x17\xdas\xbe\xbf\x18C\x1e\xfdRF\a\xbb\xea\xaeW\xf6\x9a\xa0\xbb\xd2cb\xa0\xb0\x13\x93\x04\x12of\xee|\x92\xa9{p\x97\xec\x1fB\xfd\x84\x93\xca\xc0\xf1\xe7\xee\xeb)<\xbai:\x87\x19D:\xd2$\x18|\x98*J\xc6\tS\x9f\xf1R\x9b\x8a\xea%\xf7\xf4\xce~\x13ݎ\x9e\xb9\x8aN\xe8\xfd\x84T\xa6\xef\x1eX\x91[\xd8%\x9e:daE\x02JU\xe2\x93\x1bq\xa7\xe4V\x81>d\xba\x15\x9e1gb\xfbI\xaa;\xden\x99\xf82}\x1ag\xee\xe3;\xaa\f\xb3L\xeb\xe6\x93\xe8{\x1dl\\\xe2\xddr\xef\xe9\x17LP\xce~M\xe9\xf2\xfe˥\x11f\xf4]\xe3\x91w\x8a\x85\n\x88_R\x80^C\xff\xa0{\xe6'\x8c{IneR\x8c}\xd1\x0e\x1b\x02e\x9a\xacA\x9b\x15l6R\x19\xb7\xa7\xbaZ\x11\xb6\t\x0e\x92\xd5\x10\x18'\xba_\x18!,\xb5\x19\x1a\xcb!\x82ò\xf1\xa9D\x85V\aCΚ\xee]F\x92\x16\x85\x8d\t\xe0\x9d64\x15\x9b\x10\x9cC\x1d^1\xe2\f:\x9f\xaa3\x18\xdc`D\xb4\xc5\xde)ʄ85v3\x1dv癚\xaf\x11ʔz\xf4\xeb\x1b\xfc8\x82/z\xf1\x1fY\xb2\x15\x15\x15\xdb\xc9Cƕ\x92\xed\xb6\n\xbc9\xe5\x10\x91\xb2\xc5ȹAU\xa0Ï9\x99V\x89^!\x85\xaf{\x9b\xd2\xd2q\xba\xd3>\xca3\x14\xb5\xea\x0e\x1bv\xaaj\xc6\xe6gg\t' .\xda\xfe\x04D\xaa\xf7\xa2\x98=\x16y\xb8Gu\x94k\x99DB\xd4\xc6/\x86\x84\bq\n\t}_\xa2\x8bx~7\x18\x99\xf2QNDǼ\x13\x83K\x9c\a\xb5\xbc\xe8\xbe\x134tw\x8eC\x87\x1e\x04\x7f'\xa5\xdd\x06\x10\x8e\x89|q\xect\xdc\xfb\xfb\x8dX\x9f\xa2\xb7\xf5\xf1\xe4\xd8\xf5\xdb\b\xc6\xe8X\xba\x8db\xbbaB\xbc\xf9\xf7l\x93\x92\x17\xf7\x8byk\x0e\xffp\xf0\xf6\x95\x8f\x97\xef\xa8\x12LlO\xc2\xc8Ͼo\"\x9e\xf7`\xcf\x19ч\x99\xbfXL\x9f4K\a\x0f\x91\xc1\xcb\x1e\x9e\xfdH\xfe\xc9\xff\x05\x00\x00\xff\xff\xbc\x9a$\xa6\xd7r\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]s\x1c)\x92\xef\xfa\x15\x84\xeea?B\xdd^\xc7}ą\xde|\xb2gO\xb1\x1e[ai\xf4\xbctU\xb6\x9aQ\x15\xd4\x00\xd5r\xdf\xde\xfe\xf7\x8dL\xa0\xbe\xba\xe8\xa2Z-ygǼت\x86$\xc9L\xf2\x03\x12X,\x16g\xbc\x12\xf7\xa0\x8dP\xf2\x92\xf1J\xc0W\v\x12\xff2\xcb\xc7\xff6K\xa1\xdelߞ=\n\x99_\xb2\xab\xdaXU~\x01\xa3j\x9d\xc1{X\v)\xacP\xf2\xac\x04\xcbsn\xf9\xe5\x19c\\Je9~6\xf8'c\x99\x92V\xab\xa2\x00\xbdx\x00\xb9|\xacW\xb0\xaaE\x91\x83&\xe0\xa1\xebퟖo\xffk\xf9\x9fg\x8cI^\xc2%3\xd9\x06\xf2\xba\x00\xb3\xdcB\x01Z-\x85:3\x15d\b\xf4A\xab\xba\xbad\xed\x0f\xae\x91\xef\xd0!{\xeb\xdbӧB\x18\xfb\x97\xde\xe7\x8f\xc2X\xfa\xa9*j͋N\x7f\xf4\xd5\b\xf9P\x17\\\xb7\xdf\xcf\x183\x99\xaa\xe0\x92}®*\x9eA~Ƙǟ\xba^0\x9e\xe7D\x11^\xdch!-\xe8+U\xd4e\xa0Ă\xe5`2-*K#\xbe\xb5\xdcֆ\xa95\xb3\x1b\xe8\xf6\x83\xe5g\xa3\xe4\r\xb7\x9bK\xb64ToYm\xb8\t\xbf:\x129\x00\xfe\x93\xdd!n\xc6j!\x1f\xc6z{Ǯ\xb4\x92\f\xbeV\x1a\f\xa2\xccrb\xa0|`O\x1b\x90\xcc*\xa6kI\xa8\xfc\x0f\xcf\x1e\xebj\x04\x91\n\xb2\xe5\x00O\x8fI\xff\xe3\x14.w\x1b`\x057\x96YQ\x02\xe3\xbeC\xf6\xc4\r\xe1\xb0V\x9aٍ0\xd34A =l\x1d:\x1f\x87\x9f\x1dB9\xb7\xe0\xd1\xe9\x80\n»\xcc4\x90\xdcމ\x12\x8c\xe5e\x1f\xe6\xbb\aH\x00F$\xaaxmH8\xda\xd67\xddO\x0e\xc0J\xa9\x02\xb8\x80vX4\xb6\nu%\xa0\x80\xe6\f\xddN\x8d\x16FH\xb6\xae\xd1#]2\xd4\x12Q\x19\x11\xd2X\xe0\x11a>\x01\xef\xe0kV\xd49\xe4WEm,\xe8\xdbLU\x90\x87E\xa6Q͜\xca\xc3\x0f\a!\xfb\xf8\xa5\x10\x19 \x1f2WiA\x8b<1\xd1nC\x99]\x05n\xcd\tY\xed\x87\xd0\xc6(\x93\xbaŀņ\xe7\x7f<\xbf \t\xe8\xf7\xde\xef\xc70\xae\xa1!\xd3,\xddL\x16\x7f\xbc\x85\xb0PF\xa8;\xa9\xa3f\xf0\x9dk\xcdw\a\xb8\xde,\xa6\xbd\x00\xdfc\xb0\a\x9c\x97\xa1\xda7\xe2\xfd\xb0\xff\xdf\"\xf7O\xcboC\x8b\xce\\H\xe4s!\x8c\xed\xb1ٸU,$\xebX\b\xe9\t$\x1dLT\x93S\\\xfd'!\xe6I\xe7Nl\xb24\xb2\xe9'\xc0\xbf\x14%7J=\xa6P\xef\x7f\xb1^\xbb\x84\xc52\xda\x18a+\xd8\xf0\xadP\xda\f\x97I\xe1+d\xb5\x8dj\x16nY.\xd6k\xd0\b\x8b\x96\xf9\x9b]\x81C\xc4:\x1c\xbe\xb0\x8eʊV\x18\x8c\xabe:\xb2\x94\xa8\x11\x1b\n\x05\xa8Q\xa8\xce\xc1\xc1Ђ\x1c\x88\\lE^\xf3\x82|\t.37>\xde\xe0\x17\xd3j\x13\x02\xb1\x87\x7fT\xaa]q\x0eM\x18$2\xb1\xb7\xea\xa5$\xa0\x8f_bl\xb4_5N\x89\xb0\x94p\xb0od\xa6\xae\v0\xbe\xbb\x9c\xdc\xe4V']\xb4\xccrk\f\x05_A\xc1\f\x14\x90Y\xa5\xe3\x14J\x91\x03WR\x95n\x84\xb8#Z\xb6\x1fm\xb5\x83\x99\x00\xcb(\xc4݈l\xe3\xdcW\x144\x82\xc5r\x05\x86VExU\x15\x11\xd3ՖI\xe1\xf0\x9dM鍶$h\x90!ܘ.iK\xa2~n\xcb(\xd9۹٧\xfa\xf8:\xff(\xbe\xbf%\xa2\a\xabs\xa4\xb0Oh\x12F\xfb\x05\xc9\xf3!Jz\xa4\xb8\x00\xb3\xec\xac\xce\t\x1b\xbe\xa60\xb4\xe7?\xeem\xa5\xec\x11\xe5\xd7Ż\xe3&\xcc\f\xd6MΩ\x97e\\\xd3Ϳ\b\xdf\xc8d\xddz\x8b5\x8bg\x1f\xbb-/hW\xc03$\xbf`kQX \xa7j\nQ6\x83s\xa7$P\xaa\x05f\xb4Il\xb3͇f\xef(\xa1ŀVC\x00\xceA\x0fQ\x0e\xf1 \x01$k\\\v\xda4\x15\x1aJڌ\xa5H\xb2\xfb\x85\\\xc1w\x9f\xde\xc7c\xcfnI\x94ԽA%LZW\xde\r\x1c\xa3.\xae>T\t\xbf\x90\xbf\xd6\x04\x82n\x13\xfe\x82q\xf6\b;\xe7bqɐo)\x8b\xff|\xf8*\fv,s\xf6^\x81\xf9\xa4,}yQ*\xbbA\xbc\x06\x8d]O4A\xa5\xb3$H\xc4n\U00088ce5(\xa8\r?\x84a\xd7\x12C2G\xa2\x19\xddQ\xae\x90\xeb\xd2uVֆ\xb6Z\xa5\x92\v\xb7,6֛\xe7\x81\xd2=\x16\x9c\xa4c\xdf\xe9\x1d\x1a#\xf7\x8b\xcbZ*x\x06yآ\xa3t\x1an\xe1Ad3\xfa,A?\x00\xab\xd0,\xa4K\xcb\fE\xedG6_\xbc\xd2=\x87n\xf9\xbax\xacW\xa0%X0\v4k\v\x0fŪ2\x91.\xde&\x8c䜌\x95\x05\xce\xf5ĚAZ\x92\xaaG2r\x0eWO%\xd63\xc9D^\x04\xb9]IR\xd0Ml\x9dg\xbdf\xca\xcd1*\xa63\x16\xe7\x02\x94\x9c\xb6\xd6\xfe\x86\x96\x9ef\xe3\xdfYŅ6K\xf6\x8e2{\v\xe8\xfd\xe6\x17&;`\x12\xbb\xadh\x95\xfd\x97Zly\x81\xfe\a\x1a\bɠpވZ\xef\xf9j\x17\xeci\xa3\x8cs\x1b\x9aM\xbb\xf3Gع\x1d\xe5\xa4n\xbb\n\xeb\xfcZ\x9e;_fO\xf14\x8e\x8f\x92Ŏ\x9d\xd3o\xe7\xcfu\xeffH\xf4\x8c\xaa=Q.y\x95.ɔ7;'\xd0\xc0`=8DظI \xc5\x00a\x8a\x02ɢ\\)\x13I\x16\x89\xa0\x95 \xe87\xcaX\xb7\x0e\xd9\xf3\xf7G\x17*UX\x9cd|mA3c\x95\x0e)\x99\xa8\xf8S\x96\xe2\xbb\xe5n\x03\x06\xfc>\x94_\xf4t\x801\x8a=ou\x83\xb3*\xe7n/\x8c:\xe2\x19yOԶ\xd2*\x03\x13͋hK\xa2m\xeaQp\x9f\x0eͺ.w\xd1\xdf:Ik\xa7,J\x872ϑG\xd2\x1d\x11\x19}\xf8\xdaY\xa2F\xed\x82\x7f\xa7H\xeb182:\xafQ\x96|\x98\x0e\x9c\x8c\xee\x95k\x1d\xe6\x98\a\xe6\xc2-\xfdP\x93Ι\xe3u4\xa2\xfc\xcf\xe6ڔB^SG\xec\xed\v\xbaC^\x8b\xc7ң\xc6\xca\xf1N\xfaU\xe8\xac\xe5^\xf3\xc1\xe7\xd4)\xda\xf8\xd1\xd0c\xee\xfe\x9e\by\xd7R\xd9\xce2\xceL'\xbaR\xf9\xef\f[\vml\x17\rs \xb1j\x14\xd4\x11\xa1\xa7\xfc\xa0\xf5ё\xe7g\u05fa\xb3\xa0\xb8QO>qzN\xbc\x1dH\xba\xe1[\xf0\x99\xab 3UKZ\nC=\x80\xdd̀\xe8X\xe3\xac@\xa2\xbd\xeb4\x96u\x99N\x90\x05I\x92\x90\x93\xebf\xdd&?p\x91\xb6nŎc\xab=\x94\xc39V\x8e\x9fG!\xc1\xb3\x9bN_\U000af8acK\xc6K\xe4!\xb9\x1d\xa2\x84&\xa3ޱ\xbbI\xfb\xc4\x16d\xb4\xac\xc2YV\x15`\xc1\xa7m\xce\xc0#S҈\x1c\x1a\xd3\xefE@I\xc6ٚ\x8b\xa2\xd63\xb4\xeal\x92\xcf\r¼69}d\x95\x8eȂH\x94\xb8\xce>\xc3\v\x9e\xd6\xf8\x95\x9e\xe7Ǧ8\x8c\x1a\xe6\xfb\x8b\x95\x16\xca\x1d\x068\xbd\xcb\xe8ӎ\xb9\xdc}\xf7\x19\xbf\xfb\x8c\xdf}\xc69\x1d}\xf7\x19'\xcaw\x9f\xf1\xbb\xcfx\xb8|\xf7\x19S\xcaw\x9fq&\"\xdf\xcagL\xc1pAk\x9c\a*$a\x95\x98\n1\x85\xf6D_>\xe9ǟ\xd58I.\xf3\xf58ȑC<\x91\xe3\x171\xaf\xa35^Mr3\xce\xc00w\xdc)\xca\x04\x87\xf9\x04\xa7g\x02\x02\xa7?=s}\x10\xf2\tO\xcf\xf8!\xa4E\x18G\x9d\x9d\tD\x9a\x7fz\xe2\xc2'\x11\x95\xc0\xc3V\x8aK\xff\x88\x8d1&I\tx|\xe3\xe4\xf7\xbd\x8c\xc9\x17\x90\xa5W9\x913K\x9eFY\x7f\xfe\xc7\xf3_\a\x8bN˔(\x1b\xf6i\xeb\xd4xL?b,\xdfM\x8d\xecg\xa9\xfez\xa6\xc2Ie?\xf5DMC\xe4\b\xbc\xbeX\x0f\xa8\xfck\xd27\x16\xcaϕ\xb7\x96'8a\x7f=\x02/\xe9\x8c=7;\x99m\xb4\x92\xaa6~M\ba\xbd\xcbܽ\x03\x01dL\xd8G5\xc8\x7f\xb0\x8d\xaa#\xa76&H\x9b\x90E\x9bF\x90^R\xadO\x8c\x00˷o\x97\xfd_\xac\xf2)\xb6\xecI\xd8M\x04\x18\xddG\xc1\xf3\x1c\xe3\x82\u0381\x1e\xaf\a\xc2UIC\xa1\x8c\x00S\x9aIQ8\x89\r\x10z\xf2\xca>Wnu\xf0h\xbfiz\r+=\x11wn\xfam\x93-9\xed\xbe?#\xe9\xf6\xa4G\xa3\xbeYZ\xedqɴ\xa9+\x94\t\x89\xb3\xe9\xe9\xb2)lu%=I69BNM\x88\x9d\xbb\x02\xf1\xa2ɯ/\x93\xf2\x9aL\xb3\xb4\xf4ֹ\x14{\x95T\xd6WN`}\xbd\xb4\xd5\x19ɪ\xa7?\xf5\x92\xbe\x96~tveڲ\xcc\xe1\x84Ӥ4Ӥ\xa5\x9b\x94\x01\x1f5Ԥ\xf4ѹI\xa3I\x9cL\x9f\xae\xaf\x9a\x16\xfa\xaaɠ\xaf\x9f\x02:)m\x93\x15\xe6&y\x8e_r\x18ʴ\x03P|\v\xe1|.\x99\x94\xee\xb9\xe6ϊ;?\x0f`\xa1\xb0\x047\xf5\x15〲.\xac\xa8\x8a\xf6>\xb6X\xc0\xb9\x81]sY\xd1ϊ\x8e\xc8\xfb\x9b\xba>\x7fi$~9\x88j\xb8aOP\x14\x8c\xc7\xe6\xe6\x1e\x152w\x0fh\xa6\x16\x80\xb6\x11g\xb9\xbf\x8c\xc9_\x1ez\xe1\xa6\v\xdd\x06@\x16\xb6\x8c-\xf5qy\xf8\xa6\xaf\x83\x06,U\x8f\xedy\xe6.ޠo\xbfԠw\x8c\xee\x1dk|\xb3\xf6P\xa9\x9f\xe8\x06\x03Ӡ~\xbc:<\xb4g\xb2\x17\xe0\xb4ꁽ\x93\xce#\x18\xe2DmP\xef\xb4\x01\x1d*U\x8cӢ\xfdD@H\xd5@\x884Mq\xfe眲|\x89\xf0\xee\x14\x01^\x92\a4\xcf{\xfd\x86\xa7'\x8f=5\x99\x9e\x8c\x92tJ\xf2%½9\x01\xdf,\x7f5\xfd\x14\xe4\xfc\x8d\xe7\x17>\xf5\xf8R\xa7\x1dgP/\xf5t\xe3|ڽ\xd2i\xc6W?\xc5\xf8\x9a\xa7\x17g\x9dZLNϚ\x95q0'\xb5\xea\x19\xc7\xed\xd2r\t\xa6O!&\x9e>L\xcc4H\x1b\xfc\x91\xc3N<]8\xffTa\"\x7f\xe7L\xe9W>=\xf8ʧ\x06\xbf\xc5i\xc1\x04\tL\xa82\xffT\u0cf7\xa4\x94\xceAOn\xfb͑\xdaIyM\x8d\xe5\xfa\x88\r\xf6\xb5\xc2m\xb2X\xab\x17\x03\x90Y\xf2\x17\xf9ӣ\r\x87\xb6\xc1Q2;\x1eQo_\xb2u\xd7\xfa\x0e\xb1\x7f\xcd\xc1m]\x1a\xa88\x1a\x00\n\xdc(5+\xea*|\xe0\xd9f\xd0Æ\x1b\xb6V\xba䖝7\x9b\xc5o\\\a\xf8\xf7\xf9\x92\xb1\x1fT\x93\xabӽ/͈\xb2*v\x18\x89\xb1\xf3n\x83\xe7IIT:C\xcf7\xaa\x10Y\xc4\xe7\x1c\xbdW\xcf5ػl\x88n\xfe\xcb:\xd9\"\xb1\xc0\a\x9b\x8bp\xebb\xffJfw\x9f\xfb\x91k%\xbc\x12\x7f\xa6'\x95N\xb0\xea\xf6\xee\xe6\x9a`\x051\xa2\xb7\x9a\x9a\x04ņ\xe5+@\x97\xa1\x1d\xfb!}r\xbd\xeeA\xed\xe7\bw\x1f\xab\x80ܽL\x12\xdc\x16\xaf\x9a3\x85Z\xeb\xe6\xda\xe1r\xa8'\x94/.wL\xf9\xa7'\x84\xce\x17\x15\xd7v璉.zx\x04\xbb>\xb5jv\xd0Z\xed\xbf\xbc\xd2-=\xb2\x87GWh'{W\xf5\x93\a\x86\xf4|\x0eN\x87OUO\x9e\xa7~\x01\x9c\x0e\xbbP\v\xa2b\xe4\xa7h\x06\xe4\xc9W,\x8d\xbf\xa1\xffG\xb5\x85\xf7ѕ\xcb\xfe\xeb+\x83&#\xa9\x89\x01*]2\x1f\xa1`\x9b\x8fHw|?O\xed\xc5s\r\x03*\xfe\x8e\xf0\xe7,N\xde\xf6A\x8d?HB7\xa8\x87Nc^\x15=\xf5\xb4c7\xf7\x14\xb76\xaa\xd4O}\x1f\xb7\x86\xe5ɐ`\x10\x81%\xe4\xc17ZNEF\xab4\x7f\x80\x8fʽ\xad\x93\"&\xfd\x16\xbd\x97\x97\xbc\xe7\x16\xf2\xb5\xfd$\x8c)z?\xb6!\xc0\xf6|\xc6\xdeE\xff\x88\xed\x91O\x19X[\xba\x91ғ&\xef\xfd\xeb$\xa8\x8f\r \v\x02\x05\x1c\xb4\x15\xfew\xa3\x9e\xe8\x02\xfc\xf8\x1asx@\xa4\xf3\x86\x19\xd0A\x11J\xe1=j\x98uU(\x9e\x83\xbe\xa2GT\x12F\xfcS\xaf\xc1\xc0\x1d\xe8?\xc5\xe2\xedfd<\xa1\xe7\x17̒A\x8f\xae(\xa0\xf8A\x14`\x1c≦\xe1f\xbfec)\xear\xe5<\xd55\xfe\xd8tr\xc02\xbb\xa1\xd2\x06C\x05\x1a\xfdD\xb7\x15Q\x9b \xf9\x87\x89\xc1\x1a>\ni\xe1\x01\xc6c\xe8\t\x9b\xe0\xdeh \a (0\x8a\xf8\xfe\x12[y\xec\x11\xe4>\xdez \x03\xcdbdL\x8e\x95w\xabn\xee\xaf\f\xabeN\x1b\x00\xf7\x7f\xbe=J~\xb7\xbd\xf7e\x82NHQ\xef\xf7\xe3-;!BG;\x91O\x1fW\xe21X\xdc\x18\x95\t\x8a*\x9e\x84\xf5\xd79\xbe\xdc\x1d\xe2\x87\x02\xc4\x03\xd2Q\x1b\xf8\xfc$A\x7f\t\x16\xc8\\\xcbػ-\xd3\xda\xef\xa7=h\xd1\xf7Z\xac¾G`\f\x000\x15\xf6\xb9\x8c{\t(l\xaf\t\xd3H\x1c\xc4>\x01\xdcyG\xc8ik\x85\xb4\xe3\x9c!n\x9bVt\xd8tDCN\x8b\xed\xfd\x00\xc6 \x93\x9d\x1e}j\xaa\xb8Ӧ\x86\xfd^\x8cy\xa3\xb4c\x96\xe1@\xff\xb0\xf7kT\x83\x1f\xd4\xde1\xcd=\xaaF\xf6>\xd2CxyGr\xbc\x97\xde\xfdR\xaf\xda\a\x15\xd8\xdf\xfe~\xf6\x8f\x00\x00\x00\xff\xff)\x00\x87w>{\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcV\xcfo+5\x10\xbe\xe7\xaf\x18\x89+\xbb\xa1B \x94\x1b*\x1c*\xe0\xa9j\x9ezw\xbc\x93d\xa8\xd7^f\xc6)A\xfc\xf1\xc8\xf6n\x9b\xee:\xb4\x8f\x03\xbe\xad\xed\xf9\xe6\x9bo~x\x9b\xa6Y\x99\x81\x1e\x91\x85\x82߀\x19\b\xffT\xf4\xe9Kڧ\x1f\xa4\xa5\xb0>ݬ\x9e\xc8w\x1b\xb8\x8d\xa2\xa1\x7f@\t\x91-\xfe\x84{\xf2\xa4\x14\xfc\xaaG5\x9dQ\xb3Y\x01\x18\uf0da\xb4-\xe9\x13\xc0\x06\xaf\x1c\x9cCn\x0e\xe8ۧ\xb8\xc3]$\xd7!g\xf0\xc9\xf5\xe9\x9b\xf6\xe6\xfb\xf6\xbb\x15\x807=n@\x90ә\x1a\x8d\xc2\xf8GDQiO\xe8\x90CKa%\x03ڄ\x7f\xe0\x10\x87\r\xbc\x1e\x14\xfb\xd1w\xe1\xbd\xcdP\xdb\f\xf5P\xa0\xf2\xa9#\xd1_\xae\xdd\xf8\x95\xc6[\x83\x8bl\\\x9dP\xbe \xc7\xc0\xfa\xe9\xd5i\x03\"\\N\xc8\x1f\xa23\\5^\x01\x88\r\x03n \xdb\x0e\xc6b\xb7\x02\x18\x05\xc9Xͨ\xc5\xe9\xa6\xc0\xd9#\xf6\xa68\x01\b\x03\xfa\x1f\xef\xef\x1e\xbfݾ\xd9\x06\xe8P,ӠYֿ\x9b\x97}\xa8\x85\t$``\xa4\x04\x1a\xc0X\x8b\"`#3z\x85B\x19\xc8\xef\x03\xf79\xad`v!\xea\x05\xaa\x1e\x11\x1e\xb3\xfec\x98\xed\xcb\xe1\xc0a@V\x9a\xa4)\xeb\xa2\xe2.v\xff\x8dxZ)\xd6b\x05]*=\x94\xecy\xd4\v\xbbQ\x1e\b{\xd0#\t0\x0e\x8c\x82\xbe\x14c\xda6\x1e\xc2\xeew\xb4\xdaΠ\x8b.\x922\x19]\x97*\xf6\x84\xac\xc0h\xc3\xc1\xd3_/ؒ\x04JN\x9dѬ\x9dWdo\x1c\x9c\x8c\x8b\xf85\x18\xdf͐{s\x06\xc6\xe4\x13\xa2\xbf\xc0\xcb\x062\xe7\xf1[`\xccRo\xe0\xa8:\xc8f\xbd>\x90N}hC\xdfGOz^疢]\xd4\xc0\xb2\xee\xf0\x84n-th\f\xdb#)Z\x8d\x8ck3P\x93\x03\xf1\xb9\x17۾\xfb\x8a\xc7Ε7n\xf5\x9cjP\x94\xc9\x1f.\x0er\xeb|AzR#\x95b*P%\xc4\xd7,\xa4\xad$\xdd\xc3\xcf\xdb\xcf01)\x99*Iy\xbd\xba\xd0e\xcaOR\x93\xfc\x1e\xb9\xd8\xed9\xf4\x19\x13}7\x04\xf2\x9a?\xac\xa3\\\xb8qד\xcaT\xda)us\xd8\xdb<\xab`\x87\x10\x87\xce(v\xf3\vw\x1enM\x8f\xee\xd6\b\xfeϹJY\x91&%\xe1Cٺ\x9c\xc0\xf3\xcbEދ\x83iv^ImeJl\a\xb4)\xb9I\xdfdM{\xb2\xa5\xad\xf6\x81\xc1\xd4L\xda\x0f1\xc9\x16_\xc8e\x9cH\x85\xcdlN\xa5.\x7f\x9fM},哣\x11\x9co\xce8ݧ;s\xff\x8e\xf6h\xcf\xd6a\x81(S\bߧ\x92\x16\xfa\xd8/}6\xf0\t\x9f+\xbb\xf7\x1c҄\xc6\xf9\xa8\xb9Z\x1bP\x1e\xb1\x03\xf9E\xb8\xf3\xc8ʭ\xfc0.G~\x0eh\x04\x02\x8eާ\x96\x0e~\x01Yy\x11\x16wH\xb1\xaf\xb0\xa9\xf2\xb9\xf3\xfb\x90\xff\"Lrl\xb4\xb4\x13\x8e\xc9\x1e\xfd\x14^\x15\xc0\xeb\xb9.k9\xe7>$hY\xf9y\xfeo\xc6i.\x11c\xd5w\x93YU\x0f\x92ǚ\xe2\xf5\xfe\x1aYF\xe7\xcc\xce\xe1\x06\x94\xe3Һ\xd8\x1afs\x9eW\xcdTj\x9f\xa9GQ\xd3\x0f\xef\x14\xd0\xe2UH\xeb~\x81\x92\x9a\xe7\xf9\x88\xfeZ\x8b\xc0\xb3\x91W\xe7\x15\xc8\xdd\xf9\x9a\xe9\xed\xcb\xdf\xe6\xb2\xcfJ=o \xcd\xfaF\xa9\"䇔\xaa\xa6\xb4\xd4y\xf5\xb7f\xa1\xd2\xf6\xf2\xee4H\xde\xf4\xcb\xf4W\xb3\x8c\xe1*\x85j\x05,63|w\x11\x9eh`s\x98\x02\xfe'\x00\x00\xff\xff\xef\xf8\xa6>\x10\f\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcVM\x8f\xdb6\x10\xbd\xfbW\f\x92kd7(Z\x14\xbe\x05\xdb\x1e\x82&\xc5\"N\xf7N\x93#{j\x8ad\x87C9.\xfa\xe3\v\x92Ү-\xcb\xc9nQT\x17\xc3\xe4\xf0q>\u07bca\xd34\v\x15\xe8\x019\x92wkP\x81\xf0\x8b\xa0\xcb\xff\xe2\xf2\xf0S\\\x92_\xf5o\x17\arf\rw)\x8a\xef>a\xf4\x895\xfe\x8c-9\x12\xf2nѡ(\xa3D\xad\x17\x00\xca9/*/\xc7\xfc\x17@{'\xec\xadEnv薇\xb4\xc5m\"k\x90\v\xf8xu\xff\xdd\xf2\xed\x8f\xcb\x1f\x16\x00Nu\xb8\x86\xde\xdb\xd4at*Ľ\x17\xebu\xc5\\\xf6h\x91\xfd\x92\xfc\"\x06\xd4\xf9\x8a\x1d\xfb\x14\xd6\xf0\xb4Q!\x86\xeb\xab\xeb\x0f\x05m3\xa0}\x18Њ\x81\xa5(\xbf~\xc5\xe8\x03E)\x86\xc1&V\xf6\xa6g\xc5&\xee=\xcboO\xb77\xd0G[w\xc8\xed\x92U|\xeb\xfc\x02 j\x1fp\r\xe5xP\x1a\xcd\x02`\xc8O\x81k\xc6Լ\xad\x88z\x8f\x9d\xaa\xf7\x00\xf8\x80\xee\xdd\xfd\xfb\x87\xef7\x17\xcb\x00\x06\xa3f\nR\xb2<\x1f\"P\x04\x05\xa3'p\xdc##<\x94|B\x14\xcf\x18\a\xa7\x1fA\x01F\xff\xe3\xf2q1\xb0\x0f\xc8Bc\xf0\xf5;\xe3\xd7\xd9\xeaį\xbf\x9b\x8b=\x80\x1cJ=\x05&\x13\r#\xc8\x1e\xc7t\xa0\x19\xa2\a߂\xec)\x02c`\x8c\xe8*\xf5\xf2\xb2r\xe0\xb7\x7f\xa0\x96\xe5\x04z\x83\x9car\xad\x925\x99\x9f=\xb2\x00\xa3\xf6;G\x7f=bG\x10_.\xb5J0\n\x90\x13d\xa7,\xf4\xca&|\x03ʙ\tr\xa7N\xc0\x98\xef\x84\xe4\xce\xf0ʁ8\xf5\xe3\xa3g\x04r\xad_\xc3^$\xc4\xf5j\xb5#\x19\xbbN\xfb\xaeK\x8e\xe4\xb4*\rD\xdb$\x9e\xe3\xca`\x8fv\x15i\xd7(\xd6{\x12Ԓ\x18W*PS\x02q\xb5K:\xf3\x9a\x87>\x8d\x17\xd7\xca)S,\n\x93\u06ddm\x94.yAyr\xc3T\xd6T\xa8\x1a\xe2S\x15\xf2RNݧ_6\x9fa\xf4\xa4V\xaa\x16\xe5\xc9\xf4*/c}r6ɵ\xc8\xf5\\˾+\x98\xe8L\xf0\xe4\xa4\xfcі\xd0\tĴ\xedH2\r\xfeL\x18%\x97n\n{W\x94\t\xb6\b)\x18%h\xa6\x06\xef\x1dܩ\x0e흊\xf8?\xd7*W%6\xb9\bϪֹ\xdeN\x8dkz\xcf\x1bu\x90\xc9\x1b\xa5\x9dW\x84M@}\xd1x\x19\x85Z\x1a\x14\xa2\xf5i\x8b\x15\x10|;ý\x17\xb9\x9c?t\xa9\x9b#\xe2\xbb^\x91U[{-\t\r\xfc\xee\xd4\xcdݛş\xad\xe7\xd5b̏=\xb3\x06\xe1T\xb1\a\x96\r+\xff\x04\x00\x00\xff\xffNy\xc1Q\xa1\x0e\x00\x00"), diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index ea669c709f..f8f27a5212 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,6 +4,16 @@ kind: ClusterRole metadata: name: velero-perms rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - create + - delete + - get + - list - apiGroups: - "" resources: diff --git a/design/backup-filter-enhancement/fine-grained-backup-filters-design.md b/design/backup-filter-enhancement/fine-grained-backup-filters-design.md new file mode 100644 index 0000000000..3fef82974e --- /dev/null +++ b/design/backup-filter-enhancement/fine-grained-backup-filters-design.md @@ -0,0 +1,858 @@ +# Fine Grained Backup Filters via Resource Policies + +## Glossary & Abbreviation + +**Backup Filter**: The mechanism in Velero that determines which Kubernetes resources are collected from the cluster and written into the backup archive. Backup filters currently operate on four dimensions: namespace, resource type, label, and cluster scope. +**Global Filter**: A filter that applies uniformly across all namespaces in a backup. All existing Velero backup filters are global filters. +**Namespace-Scoped Filter**: A filter that applies only within specific namespaces, overriding the global filter for those namespaces. This is the capability introduced by this design. +**ClusterScopedFilterPolicy**: A global filter for cluster-scoped resources that allows per-kind label selectors and name patterns, functioning similarly to `NamespacedFilterPolicy` but applied to cluster-scoped resources globally. +**Resource Filter**: A filter rule that pairs one or more resource kinds with their own label selector and/or name patterns. Multiple resource filters within a namespace-scoped policy allow different filtering criteria for different resource types. +**Resource Name Filter**: A filter that matches individual resource instances by their metadata.name, using glob patterns. This is a new filter dimension introduced by this design. +**Resource Policy**: An existing Velero mechanism where backup behavior rules are defined in a ConfigMap and referenced from `BackupSpec.ResourcePolicy`. Currently used for volume policies and global include/exclude policies. + +## Background + +Velero's backup filter system allows users to specify which resources to include or exclude from a backup. The filters operate on three dimensions: + +1. **Namespace** — `IncludedNamespaces`/`ExcludedNamespaces` select which namespaces to back up +2. **Resource Type** — `IncludedResources`/`ExcludedResources` (or the newer scoped variants `Included/ExcludedClusterScopedResources`, `Included/ExcludedNamespaceScopedResources`) select which Kubernetes resource types to back up +3. **Labels** — `LabelSelector`/`OrLabelSelectors` filter individual objects by their labels + +All three dimensions are applied **globally** — the same resource type filter, the same label selector, and the same namespace list apply uniformly throughout the entire backup operation. Specifically: + +- In `item_collector.go`, the `ResourceIncludesExcludes.ShouldInclude()` check is a single global check applied to every resource type across all namespaces. +- In `listResourceByLabelsPerNamespace()`, the same `LabelSelector` is passed to every Kubernetes API list call regardless of namespace. +- There is no mechanism to filter resources by their individual `metadata.name`. + +This creates three critical gaps for common backup scenarios: + +**Gap 1: Different resource needs per namespace.** When multiple applications share the same cluster, different namespaces often require different backup strategies. For example, a namespace running a database workload may need all resource types backed up, while a namespace running a stateless frontend may only need Deployments, ConfigMaps, and Services. Setting `IncludedResources: [configmaps]` means *all* ConfigMaps in *all* included namespaces — you cannot say "only ConfigMaps in namespace-a but everything in namespace-b." + +**Gap 2: Same resource type, different workloads.** Resources of the same type (e.g., ConfigMaps or Secrets) in the same namespace may belong to different workloads. For instance, a namespace may contain `app-config`, `app-secret`, `monitoring-config`, and `monitoring-secret`. Without name-based filtering, you cannot selectively back up only the `app-*` resources — the only option is label-based selection, which requires workloads to have been pre-labeled appropriately. + +**Gap 3: Different kinds need different selectors.** Within a single namespace, different resource types may belong to different workloads with different labels. For example, Deployments labeled `app=workload-1` and StatefulSets labeled `app=workload-2` in the same namespace. The current single-label-selector-per-namespace model cannot express this — the label selector applies identically to all resource types. + +## Goals + +- Extend the `ResourcePolicies` ConfigMap format with a `namespacedFilterPolicies` section that allows per-namespace, per-kind resource filtering with independent label selectors and name patterns for each resource type +- Extend the `ResourcePolicies` ConfigMap format with a `clusterScopedFilterPolicy` section that allows per-kind resource filtering with independent label selectors and name patterns for cluster-scoped resources globally +- Support resource name filtering by glob patterns using the same `gobwas/glob` library that Velero uses for namespace patterns, ensuring consistency across the codebase +- Support per-kind label selectors, so that different resource types within the same namespace can be filtered with different labels +- Maintain full backward compatibility — existing backups with no `namespacedFilterPolicies` behave exactly as they do today +- Define clear precedence rules for how per-namespace filters interact with global filters +- Add corresponding validation within the Resource Policies validation pipeline using existing Velero wildcard validation functions +- Update `velero backup describe` output to display the referenced ResourcePolicy ConfigMap name when configured +- Ensure the restore process works correctly with backups produced by namespace-scoped filters, without requiring restore-side code changes in the initial phase + +## Non-Goals + +- Adding namespace-scoped filters to `RestoreSpec` or the restore pipeline is not part of the initial implementation. Restore from a namespace-filtered backup works automatically because the restore process reads whatever is in the backup archive. Restore-side namespace filters will be addressed in a follow-up. +- Changing existing `BackupSpec` fields (`IncludedResources`, `LabelSelector`, etc.) or adding new CRD fields is explicitly avoided by this design. +- Supporting regex patterns for resource names is not included. Glob patterns (already used throughout Velero) are sufficient and consistent. +- Modifying the plugin `ResourceSelector` system (`AppliesTo()` / `resolvedAction.ShouldUse()`) is not part of this design. +- CLI flags for inline specification of namespace-scoped filters are not part of the initial implementation. The configuration is expressed in the ResourcePolicy ConfigMap YAML. + +## Architecture of Namespace-Scoped Filters + +### Configuration Model + +The namespace-scoped filters and fine-grained global filters are defined in the same ResourcePolicy ConfigMap that is already referenced by `BackupSpec.ResourcePolicy`. The YAML format is extended with two new top-level keys: `clusterScopedFilterPolicy` and `namespacedFilterPolicies` + +```yaml +version: v1 +volumePolicies: + # existing volume policies (unchanged) + - conditions: + capacity: "0,100Gi" + action: + type: skip +includeExcludePolicy: + # existing global include/exclude policy (unchanged) + includedNamespaceScopedResources: + - configmaps +clusterScopedFilterPolicy: + # NEW: global overrides for cluster-scoped resources + resourceFilters: + - kinds: [ClusterRole, ClusterRoleBinding] + names: ["my-app-*"] + - kinds: [CustomResourceDefinition] + labelSelector: + matchLabels: + app: my-app +namespacedFilterPolicies: + # NEW: per-namespace filter overrides + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret, Deployment] + labelSelector: + matchLabels: + app: my-app + - namespaces: + - ns-b + resourceFilters: + - kinds: [Deployment] + names: [app-1, app-2] + - kinds: [ConfigMap] + labelSelector: + matchLabels: + app: my-service +``` + +All four sections coexist in the same ConfigMap. They are independent — `volumePolicies` handles volume backup strategy, `includeExcludePolicy` handles global resource type filtering, `clusterScopedFilterPolicy` handles cluster-scoped resource filtering by kind/name/label, and `namespacedFilterPolicies` handles per-namespace, per-kind overrides. + +### The `resourceFilters` Model + +Each `namespacedFilterPolicies` entry targets one or more namespaces and contains a `resourceFilters` array. Each entry in `resourceFilters` pairs one or more resource kinds with their own label selector and name patterns: + +```yaml +namespacedFilterPolicies: + - namespaces: [ns-a] + resourceFilters: + - kinds: [ConfigMap, Secret] # these kinds share a selector + labelSelector: + matchLabels: + app: my-app + names: ["app-*"] + - kinds: [Deployment] # this kind has its own selector + names: [workload-1, workload-2] + - kinds: [StatefulSet] # this kind has no extra filtering +``` + +This model has one way to express filters — there is no ambiguity about how to structure the configuration. Only resource kinds listed in `resourceFilters` entries are included in the backup for the matched namespaces; unlisted kinds are implicitly excluded. + +#### Label selectors (`matchLabels` / `matchExpressions`) + +`labelSelector` and each entry of `orLabelSelectors` use the standard Kubernetes selector shape (same as `BackupSpec.labelSelector`): + +```yaml +labelSelector: + matchLabels: + app: my-app + matchExpressions: + - key: environment + operator: In + values: [prod, staging] + - key: do-not-backup + operator: DoesNotExist +``` + +Supported `matchExpressions` operators: `In`, `NotIn`, `Exists`, `DoesNotExist`. Prefer `In` for value-OR on one key; use `orLabelSelectors` for OR across independent multi-key groups. `labelSelector` and `orLabelSelectors` cannot co-exist in the same `resourceFilters` entry. + +#### Catch-All Resource Filter (Empty `kinds` or `["*"]`) + +A `ResourceFilter` entry with an empty (or omitted) `kinds` field, or a field explicitly set to `["*"]`, acts as a **catch-all**. Its `labelSelector` or `orLabelSelectors` (if provided) is applied to **all resource types in the namespace that are not already matched by a kind-specific filter entry**. If no selectors are provided, all unlisted resources are included. Using `["*"]` is highly recommended as it makes the catch-all intention explicit and self-documenting. + +**Rules for catch-all entries:** +- At most **one** catch-all entry is allowed per `NamespacedFilterPolicy`. +- `names` and `excludedNames` are **not** supported on catch-all entries. Name patterns are kind-specific by nature and cannot be applied across arbitrary kinds; use kind-specific entries for name-based filtering. +- The catch-all applies to kinds that are **not listed in any other `resourceFilters` entry** in the same policy. Kind-specific entries take precedence over the catch-all. +- A catch-all entry **does not inherit or fall back to `BackupSpec.LabelSelector`**. If a catch-all entry has no `labelSelector`/`orLabelSelectors`, all unlisted resource kinds in the namespace are included with **no label filtering** — the global label selector is not applied. Define a catch-all with an explicit `labelSelector` if label-based filtering is desired for unlisted kinds. + +**Evaluation order within a namespace filter policy:** +1. For each resource kind encountered during backup, the system first checks whether a kind-specific `resourceFilters` entry exists for that kind. +2. If a kind-specific entry exists, it is used exclusively (its label selectors and name patterns apply). +3. If no kind-specific entry exists but a catch-all entry is present, the catch-all's `labelSelector`/`orLabelSelectors` is applied to that kind. +4. If neither a kind-specific entry nor a catch-all entry exists, the kind is excluded from the backup for that namespace. + +### Filter Precedence Model + +The namespace-scoped filter system and fine-grained global filter system layer on top of the existing global filter system. They intentionally behave differently: +- **`namespacedFilterPolicies`** acts as an **exclusive allowlist (boundary)**. Only kinds explicitly listed (or matched by a catch-all) are backed up from that namespace. This gives namespace owners complete and isolated control over their namespace's backup contents, preventing unexpected data spillage from global fallbacks. +- **`clusterScopedFilterPolicy`** acts as a **refinement overlay (tweak)**. Unlisted cluster-scoped kinds fall back to the standard global filters. This allows administrators to selectively adjust filtering for a few specific cluster-scoped kinds without rewriting the entire global inclusion list. + +**For Namespace-Scoped Resources:** + +The evaluation order is: + +1. **Global namespace filter** (`BackupSpec.IncludedNamespaces`/`ExcludedNamespaces`) is checked first. A namespace must pass this filter to be considered at all. `namespacedFilterPolicies` cannot override namespace exclusion — if a namespace is excluded globally, no filter policy entry can bring it back. + +2. **Per-namespace filter lookup.** For each namespace that passes the global namespace filter, the system checks whether any `namespacedFilterPolicies` entry matches (by namespace name or glob pattern). If a match is found, the `resourceFilters` array determines what gets backed up for that namespace: + - Only resource kinds listed in `resourceFilters[].kinds` are included + - Each kind uses its own `labelSelector`/`orLabelSelectors` (if specified) + - Each kind uses its own `names`/`excludedNames` patterns (if specified) + +3. **Namespaces without a matching filter policy** continue to use the global filters (`BackupSpec.IncludedResources`, `BackupSpec.LabelSelector`, etc., combined with `includeExcludePolicy`) exactly as they do today. + +4. **If multiple filter policy entries could match the same namespace** (e.g., `team-*` and `team-frontend-*` both matching `team-frontend-prod`), the **first matching policy in the list** is used. **Important: Place more specific patterns before broader patterns** to achieve the intended filtering behavior. + +5. **The `velero.io/exclude-from-backup=true` label** always takes precedence over all filters, regardless of whether the item matches global or per-namespace filters. + +6. **Interaction with `includeExcludePolicy`**: `namespacedFilterPolicies` is a **refinement** of the global resource filter system, not a replacement. Global exclusions defined in `includeExcludePolicy` (e.g., `excludedNamespaceScopedResources: [secrets]`) are applied first at the resource-type level before per-namespace filter policies are consulted. A namespace-scoped filter policy cannot re-include a resource kind that has been globally excluded by `includeExcludePolicy`. For example, if `secrets` is listed under `excludedNamespaceScopedResources`, no `Secret` resources will be backed up from any namespace, even if a `namespacedFilterPolicies` entry explicitly lists `Secret` for that namespace. Users who need per-namespace secret selection must remove `secrets` from the global exclusion list. + + To help users catch this misconfiguration early, Velero logs a warning at backup start when a `namespacedFilterPolicies` entry lists a kind that is globally excluded by `includeExcludePolicy`: + ``` + level=warn msg="namespacedFilterPolicies entry lists a kind that is globally excluded by includeExcludePolicy; the per-namespace filter entry has no effect" kind="secrets" namespacePattern="ns-a" + ``` + +**For Cluster-Scoped Resources:** + +1. If `clusterScopedFilterPolicy` is present, it acts as a **refinement overlay** over the existing global filters for cluster-scoped resources. It is NOT an exclusive allowlist. + - To back up cluster-scoped resources in a namespace-filtered backup, you must still explicitly include them via `BackupSpec.IncludedClusterScopedResources`. + - If a cluster-scoped kind is listed in its `resourceFilters`, its specific `labelSelector`/`orLabelSelectors` and `names`/`excludedNames` patterns are applied. + - If a cluster-scoped kind is **not listed**, it falls back to the standard global filters (`BackupSpec.LabelSelector`, etc.) and is included in the backup. + +2. If `clusterScopedFilterPolicy` is absent, Velero falls back to the existing global filters (`IncludedClusterScopedResources`, `IncludedResources`, `LabelSelector`, etc.) for cluster-scoped resources. + +3. **The `velero.io/exclude-from-backup=true` label** always takes precedence over all filters. + +```mermaid +flowchart TD + A["BackupSpec Global
IncludedNamespaces / ExcludedNamespaces"] + B{Namespace passes
global filter?} + C[Namespace excluded
from backup] + D{namespacedFilterPolicies
lookup by namespace} + E{"For each resource kind:
is kind in resourceFilters?"} + F["Apply kind-specific filters:
- labelSelector / orLabelSelectors
- names / excludedNames"] + G[Kind skipped for
this namespace] + H["Use global filters:
- BackupSpec IncludedResources
- BackupSpec LabelSelector
- includeExcludePolicy"] + + I{"Is resource
cluster-scoped?"} + J{"Is clusterScopedFilterPolicy
present?"} + K{"Is kind in resourceFilters?"} + L["Apply kind-specific filters:
- labelSelector / orLabelSelectors
- names / excludedNames"] + + I -- Yes --> J + J -- Yes --> K + K -- Yes --> L + K -- No --> H + J -- No --> H + + I -- No --> A + A --> B + B -- No --> C + B -- Yes --> D + D -- Match found --> E + E -- Yes --> F + E -- No --> G + D -- No match found --> H +``` + +### Data Flow in the Backup Pipeline + +The existing backup pipeline has two stages: item collection and item backup. Namespace-scoped filters and fine-grained global filters are applied at both stages: + +**Stage 1 — Item Collection (`item_collector.go`).** Resources are listed from the Kubernetes API. + +- **Resource type check** in `getResourceItems()`: Before iterating namespaces, the global resource type check still applies. + - **For Cluster-Scoped Resources:** The global resource type check (`ShouldInclude`) determines if the kind is collected. `ClusterScopedFilterPolicy` does not skip unlisted cluster-scoped kinds at this stage. + - **For Namespace-Scoped Resources:** Within the namespace loop, a per-namespace resource type check is added. If a filter policy matches the current namespace, only resource kinds listed in `resourceFilters[].kinds` are included — if the current resource type is not listed, it is skipped for that namespace. +- **Label selector** in `listResourceByLabelsPerNamespace()` and `listResourceByLabelsGlobally()`: The function looks up the filter policy (either the namespace-specific one or the fine-grained global one). If found, it retrieves the `ResourceFilter` entry for the current resource kind and uses that entry's `labelSelector`/`orLabelSelectors` for the Kubernetes API list call. If no filter policy is found, the global selectors are used as before. + +**Stage 2 — Item Backup (`item_backupper.go`).** Collected items are validated and written to the archive. + +- **Name pattern check** in `itemInclusionChecks()`: After the existing namespace and resource type re-validation, the item's `metadata.name` is checked against the `ResourceFilter` entry's `names`/`excludedNames` glob patterns for the item's kind (checking the cluster-scoped map for cluster resources and namespace map for namespace resources). If the name doesn't match, the item is excluded. + - **Important:** If the item's kind is not listed in the namespace filter map **and** there is no catch-all entry, the item passes through Stage 2 without a name check. This is intentional — see [Plugin AdditionalItems and Auto-Backed Up CRDs](#edge-cases-and-behavior-documentation) below. + +### Impact on Restore + +The restore process (`pkg/restore/restore.go`) is **not modified** in this design. The reason: + +- Restore reads the backup archive as-is. Items excluded by namespace-scoped filter policies during backup are simply absent from the archive. The restore process iterates what's in the tarball and applies `RestoreSpec` filters on top. No items excluded during backup will appear during restore. +- Restore plugins that request "additional items" (via `RestoreItemAction`) may reference items excluded from the backup. These items won't be in the archive, so the restore will skip them silently. This is the same behavior that occurs today with any incomplete backup — no new risk is introduced. +- Users can still use `RestoreSpec.IncludedNamespaces` to selectively restore from a namespace-filtered backup. + +A follow-up design will add namespace-scoped filters to the restore pipeline. + +### Edge Cases and Behavior Documentation + +This section documents the system behavior in edge cases and error conditions: + +**Plugin AdditionalItems and Auto-Backed Up CRDs:** +Cluster-scoped resources injected dynamically (such as `VolumeSnapshotClass` from the CSI plugin or `CustomResourceDefinition` from Velero's auto-backup loop) do not require hardcoded exceptions. In `itemInclusionChecks()`, Velero natively allows unlisted cluster-scoped resources to pass through unless explicitly excluded by the user. `ClusterScopedFilterPolicy` preserves this permissive behavior: if a dynamically injected cluster-scoped resource is NOT listed in the policy, it passes through untouched. If it IS listed, its specific `names` and `excludedNames` filters are strictly enforced. + +**Plugin-injected namespace-scoped additional items** follow the same permissive principle. When a `BackupItemAction` returns additional items whose kind is not listed in the matched `namespacedFilterPolicies` entry and there is no catch-all entry, those items still pass through Stage 2 (`itemInclusionChecks`). This is intentional: blocking plugin-injected items at Stage 2 would break backup completeness — for example, a CSI plugin may inject a `VolumeSnapshotContent` that is required for a correct restore even when the user's filter policy only lists application resource types. + +The kind-level exclusion that makes `namespacedFilterPolicies` an exclusive allowlist applies only during **Stage 1** (the primary collection pass in `item_collector.go`). At Stage 2, `itemInclusionChecks` enforces only: +- The `velero.io/exclude-from-backup=true` label (always takes precedence). +- The `names`/`excludedNames` patterns for **listed** kinds. + +Plugin-injected items of unlisted kinds are therefore included as long as they are not explicitly excluded by label. Users who need to suppress a specific plugin-injected kind should apply the `velero.io/exclude-from-backup=true` label to those resources. + +**Multiple Glob Patterns Matching Same Namespace (Incorrect Order):** +```yaml +namespacedFilterPolicies: + - namespaces: ["team-*"] # Broader pattern listed first + resourceFilters: + - kinds: [Deployment, Service] + - namespaces: ["team-frontend-*"] # More specific pattern listed second + resourceFilters: + - kinds: [ConfigMap, Secret, Deployment, Service] +``` +**Behavior:** For namespace `team-frontend-prod`, the broader `team-*` pattern matches first, so only `Deployment` and `Service` are backed up. The more specific `team-frontend-*` rule is never reached. + +**Multiple Glob Patterns Matching Same Namespace (Correct Order):** +```yaml +namespacedFilterPolicies: + - namespaces: ["team-frontend-*"] # More specific pattern listed first + resourceFilters: + - kinds: [ConfigMap, Secret, Deployment, Service] + - namespaces: ["team-*"] # Broader pattern listed second + resourceFilters: + - kinds: [Deployment, Service] +``` +**Behavior:** For namespace `team-frontend-prod`, the specific `team-frontend-*` pattern matches first, backing up all specified resources. For `team-backend-dev`, the broader `team-*` pattern matches, backing up only `Deployment` and `Service`. This achieves the intended behavior. + +**Namespace Included Globally But No Matching Filter Policy:** +```yaml +# BackupSpec includes "production" namespace +# ResourcePolicy has no namespacedFilterPolicies entry for "production" +``` +**Behavior:** The namespace uses global filters exactly as it does today. This is the backward compatibility behavior — only namespaces with explicit filter policies get namespace-scoped filtering. + + +**Empty ResourceFilters Array:** +```yaml +namespacedFilterPolicies: + - namespaces: ["test-namespace"] + resourceFilters: [] # empty array +``` +**Behavior:** Validation error during backup creation: +``` +namespacedFilterPolicies[0]: at least one resourceFilter must be specified +``` + +**Namespace Pattern with No Matches:** +```yaml +namespacedFilterPolicies: + - namespaces: ["nonexistent-*"] + resourceFilters: [...] +``` +**Behavior:** No error. The filter policy is loaded but never applied since no namespaces match the pattern. This allows for conditional filtering based on namespace existence. + +**Resource Kind Not Present in Target Namespaces:** +```yaml +resourceFilters: + - kinds: ["StatefulSet"] # namespace has no StatefulSets + names: ["workload-1"] +``` +**Behavior:** No error. The filter is applied but finds no matching resources. Empty result set is valid. + +**Conflicting Name Patterns:** +```yaml +resourceFilters: + - kinds: ["ConfigMap"] + names: ["app-*"] + excludedNames: ["app-config"] # conflicts with names pattern +``` +**Behavior:** The `excludedNames` takes precedence. Resources matching `app-*` are included, then `app-config` is excluded. Net result: includes `app-secret`, `app-data`, etc., but excludes `app-config`. + +**Invalid Label Selector Syntax:** +```yaml +resourceFilters: + - kinds: ["Pod"] + labelSelector: + matchLabels: + "invalid label key!": "value" # invalid key syntax +``` +**Behavior:** Validation error during backup creation when `metav1.LabelSelectorAsSelector()` fails: +``` +namespacedFilterPolicies[0].resourceFilters[0]: invalid label selector: "invalid label key!" is not a valid label key +``` + +**Out-of-Scope Kinds in Filter Entries:** +A user may accidentally list a cluster-scoped kind (e.g., `ClusterRole`) inside a `namespacedFilterPolicies` entry, or a namespace-scoped kind (e.g., `ConfigMap`) inside `clusterScopedFilterPolicy`. The system silently ignores such entries at the Kubernetes API level — namespace-scoped items are never listed globally, and cluster-scoped items are never listed per-namespace, so no matching resources will ever be found. A warning is logged at backup start to help the user detect the misconfiguration. No validation error is raised — the entry is harmless but ineffective. + +**Discovery Helper Unavailable:** +If the discovery helper is completely unavailable during backup initialization, the backup fails with: +``` +failed to resolve namespace filter policies: discovery client unavailable +``` +This is consistent with how other discovery-dependent features handle this error condition. + +# Detailed Design + +## ResourceFilter Field Notes + +**`labelSelector`** uses the standard Kubernetes shape: `matchLabels` (equality) and `matchExpressions` (set-based: `In`, `NotIn`, `Exists`, `DoesNotExist`). All requirements within one selector are AND-ed. Example: + +```yaml +labelSelector: + matchLabels: + app: my-app + matchExpressions: + - key: environment + operator: In + values: [prod, staging] + - key: do-not-backup + operator: DoesNotExist +``` + +**`orLabelSelectors`** is a list of the same selector shape. Match if **any** entry matches (AND within each entry, OR across the list). Prefer `In` for value-OR on one key; use `orLabelSelectors` for OR of independent multi-key groups. `labelSelector` and `orLabelSelectors` cannot co-exist in the same entry. + +```yaml +orLabelSelectors: + - matchLabels: + tier: frontend + matchExpressions: + - key: track + operator: In + values: [canary] + - matchLabels: + tier: backend +``` + +**`names` / `excludedNames`** accept exact resource names or glob patterns. If `names` is empty, all resource names are included (subject to label filters). `excludedNames` takes precedence over `names` when a name matches both. + +## Validation + +**Validation functions for `namespacedFilterPolicies`:** + +1. **Each filter policy must specify at least one namespace:** + ``` + namespacedFilterPolicies[N]: at least one namespace must be specified + ``` + +2. **Each filter policy must specify at least one resource filter:** + ``` + namespacedFilterPolicies[N]: at least one resourceFilter must be specified + ``` + +3. **Each resource filter without kinds can only be defined once, and cannot specify names/excludedNames.** + ``` + namespacedFilterPolicies[N]: only one resource filter with empty kinds is allowed + namespacedFilterPolicies[N].resourceFilters[M]: names or excludedNames cannot be specified when kinds is empty + ``` + +4. **No duplicate kinds across resource filter entries** within the same namespace filter: + ``` + namespacedFilterPolicies[N]: kind "Pod" appears in both resourceFilters[0] and resourceFilters[2] + ``` + +5. **`labelSelector` and `orLabelSelectors` mutual exclusion** within each resource filter: + ``` + namespacedFilterPolicies[N].resourceFilters[M]: labelSelector and orLabelSelectors cannot co-exist + ``` + +6. **No duplicate namespace patterns across filter policies.** This validates only exact duplicates - runtime behavior handles overlapping patterns. + + **Rationale:** Detecting all possible pattern overlaps (like `team-*` vs `team-frontend-*`) is computationally complex and may reject valid configurations. Instead, the runtime uses first-match semantics - the first matching filter policy in the list is applied. This allows users flexibility while preventing obvious configuration errors. + +7. **Namespace patterns must be valid globs.** + +8. **Resource name patterns must be valid globs.** + +9. **Resource kind validation with discovery helper** (performed during backup initialization). + +**Validation functions for `clusterScopedFilterPolicy`:** + +1. **At least one resourceFilter must be specified.** + ``` + clusterScopedFilterPolicy: at least one resourceFilter must be specified + ``` + +2. **No duplicate kinds across resource filters.** + +3. **`labelSelector` and `orLabelSelectors` mutual exclusion.** + +4. **Resource name patterns must be valid globs.** + +Additionally, in `backup_controller.go`, a validation check ensures that `namespacedFilterPolicies` and `clusterScopedFilterPolicy` are not used with old-style resource filters (`IncludedResources`/`ExcludedResources`/`IncludeClusterResources`), similar to the existing check for `includeExcludePolicy`. + +## ConfigMap Examples + +### Per-Namespace Resource Type Filtering + +Back up only ConfigMaps, Secrets, and Deployments (with label `app=my-app`) from `ns-a`, but everything from `ns-b`: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: backup-filter-policy + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret, Deployment] + labelSelector: + matchLabels: + app: my-app + # ns-b has no filter policy entry, so global filters apply (include everything) +``` + +Backup CR referencing it: + +```yaml +apiVersion: velero.io/v1 +kind: Backup +metadata: + name: selective-backup + namespace: velero +spec: + includedNamespaces: + - ns-a + - ns-b + resourcePolicy: + kind: configmap + name: backup-filter-policy + storageLocation: default + ttl: 720h0m0s +``` + +### Per-Kind Label Selectors (Different Labels per Kind) + +Back up Deployments with one label and StatefulSets with a different label from the same namespace: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: vm-filter-policy + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + - namespaces: + - target-namespace + resourceFilters: + - kinds: [Deployment] + labelSelector: + matchLabels: + app: production-workload-1 + - kinds: [StatefulSet] + labelSelector: + matchLabels: + app: production-workload-2 +``` + +### Per-Kind Exact Names + +Back up specific Deployments, Configmaps, and Secrets by name: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: named-resource-filter + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + - namespaces: + - target-namespace + resourceFilters: + - kinds: [Deployment] + names: [workload-1, workload-2] + - kinds: [ConfigMap] + names: [p1, p2] + - kinds: [Secret] + names: [c1, c2] +``` + +### Name Pattern Filtering with Exclusion + +Back up only `app-*` ConfigMaps and Secrets from `production`, excluding temporary and debug resources: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: app-config-filter-policy + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [ConfigMap, Secret] + names: ["app-*"] + excludedNames: ["*-tmp", "*-debug"] +``` + +### Catch-All with No Label Selector (Override-Only) + +A user may want to use the global configuration for 99% of resources in a namespace, but only apply a specific name filter to a single kind. To achieve this without explicitly listing all other kinds or adding dummy labels, a catch-all filter without a label selector can be used: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: override-only-filter-policy + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [Secret] + names: [my-secret] # Specific override for Secrets + - kinds: ["*"] # Catch-all: NO label selector + # Includes all other kinds unconditionally +``` + +**Result:** +- `Secret` resources: only `my-secret` is backed up. +- All other resource types: backed up unconditionally (acting like a global fallback). + +### Catch-All Label Selector (Back Up Everything with a Specific Label) + +When a user wants to back up any resource type in a namespace that carries a particular label — without enumerating every kind — the catch-all entry (empty `kinds` or `["*"]`) achieves this with a single rule: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: label-based-filter-policy + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: ["*"] # catch-all: applies to every kind not listed below + labelSelector: + matchLabels: + backup: "true" # back up any resource carrying this label +``` + +**Result:** Every resource type in `production` that has the label `backup=true` is backed up. Resources without that label are excluded. No kind enumeration is required. + +### Catch-All with Per-Kind Name Overrides + +A more advanced pattern: use exact names for specific kinds and fall back to a label selector for all remaining kinds. Kind-specific entries take precedence over the catch-all: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: mixed-filter-policy + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [Deployment] + names: [api-server, worker] # these exact Deployments by name + - kinds: [Secret] + names: [db-credentials, tls-cert] # these exact Secrets by name + - kinds: ["*"] # catch-all for all other kinds + labelSelector: + matchLabels: + backup: "true" # back up by label +``` + +**Result:** +- `Deployment` resources: only `api-server` and `worker` are backed up (name filter; the catch-all does not apply). +- `Secret` resources: only `db-credentials` and `tls-cert` are backed up (name filter; the catch-all does not apply). +- All other resource types (ConfigMap, StatefulSet, Service, etc.): backed up only if they carry `backup=true`. + +This pattern is useful when certain high-value resources need precise name-based selection, while the rest of the namespace is covered by a label convention. + +### Glob Namespace Patterns and Ordering + +Apply filters to namespaces matching patterns. **Critical: Order patterns from most specific to least specific:** + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: team-filter-policy + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + # More specific patterns first + - namespaces: + - "team-frontend-prod" # Most specific (exact match) + resourceFilters: + - kinds: [Deployment, Service, ConfigMap, Secret, PVC] + - namespaces: + - "team-frontend-*" # Less specific (pattern match) + resourceFilters: + - kinds: [Deployment, Service, ConfigMap] + - namespaces: + - "team-*" # Least specific (broad pattern) + resourceFilters: + - kinds: [Deployment, Service] +``` + +**Pattern Matching Results:** +- `team-frontend-prod` → Uses exact match policy (backs up 5 resource types) +- `team-frontend-dev` → Uses `team-frontend-*` policy (backs up 3 resource types) +- `team-backend-test` → Uses `team-*` policy (backs up 2 resource types) +- `app-namespace` → No match, uses global filters + +### Combined with Volume Policies + +Both volume policies and namespace-scoped filters in the same ConfigMap: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: combined-policy + namespace: velero +data: + policy: | + version: v1 + volumePolicies: + - conditions: + capacity: "0,10Gi" + storageClass: + - standard + action: + type: fs-backup + - conditions: + capacity: "10Gi,100Gi" + action: + type: snapshot + namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [Deployment] + names: [workload-1, workload-2] + - kinds: [StatefulSet] + labelSelector: + matchLabels: + app: my-app + - kinds: [ConfigMap, Secret] + names: ["app-*"] + excludedNames: ["*-tmp", "*-debug"] +``` + +### Backup CR — No ResourcePolicy (backward compatible) + +Existing backups continue to work exactly as before: + +```yaml +apiVersion: velero.io/v1 +kind: Backup +metadata: + name: full-backup + namespace: velero +spec: + includedNamespaces: + - "*" + includedResources: + - "*" + labelSelector: + matchLabels: + backup: "true" + storageLocation: default +``` + +## CLI + +### `velero backup describe` + +The output displays the referenced ResourcePolicy ConfigMap name when configured on the backup. It intentionally avoids resolving and displaying the live ConfigMap contents, because the ConfigMap content in the cluster may be modified or deleted after the backup execution, which could lead to displaying out-of-sync or inaccurate information: + +``` +Name: selective-backup +Namespace: velero +Labels: +Annotations: + +Phase: Completed + +Errors: 0 +Warnings: 0 + +Namespaces: + Included: ns-a, ns-b + Excluded: + +Resources: + Included: * + Excluded: + Cluster-scoped: auto + +Label selector: + +Resource policies: + Type: configmap + Name: backup-filter-policy + +Storage Location: default + +... +``` + +### `velero backup create` + +No new CLI flags are added. The namespace-scoped filter policies are specified in the ResourcePolicy ConfigMap, which is already referenced via the existing `--resource-policies-configmap` flag: + +```bash +velero backup create selective-backup \ + --include-namespaces ns-a,ns-b \ + --resource-policies-configmap backup-filter-policy +``` + +The `--help` output for `velero backup create` is updated to clarify the interaction between global and namespace-scoped filters: + +``` +Backup Filtering Options: + --include-namespaces stringArray namespaces to include in the backup (use '*' for all namespaces) + --exclude-namespaces stringArray namespaces to exclude from the backup + --include-resources stringArray resources to include in the backup, formatted as resource.group + --exclude-resources stringArray resources to exclude from the backup, formatted as resource.group + --include-cluster-resources optionalBool[=true] include cluster-scoped resources + --exclude-cluster-resources exclude cluster-scoped resources + --selector labelSelector only back up resources matching this label selector + --or-selector labelSelector back up resources matching any of the label selectors (can be repeated) + --resource-policies-configmap string reference to a configmap containing resource policies for volume snapshots and namespace-scoped filtering + +Notes: +- Global filters (--include-resources, --selector, etc.) apply to all included namespaces +- Namespace-scoped filters defined in --resource-policies-configmap override global filters for matching namespaces +- Fine-grained global filter policies defined in --resource-policies-configmap override global filters for cluster-scoped resources +- Use 'velero backup describe' to view the referenced ResourcePolicy ConfigMap name after backup creation +``` + +### CLI Integration Points + +**Backup Creation Workflow:** +1. User creates ResourcePolicy ConfigMap with `namespacedFilterPolicies` +2. User references ConfigMap via `--resource-policies-configmap` flag +3. Backup controller validates policies during backup initialization +4. Validation errors are reported immediately with specific line/field references + +**Help and Discovery:** +- `velero backup create --help` includes updated filtering documentation +- `velero backup describe` shows the referenced ResourcePolicy ConfigMap name +- Validation errors include ConfigMap field references for easy debugging + +**Configuration Discovery:** +- `velero backup create --help` includes namespace-scoped filtering documentation +- `velero backup describe` shows the referenced ResourcePolicy ConfigMap name for verification + +## User Perspective + +This design provides fine-grained, per-namespace, per-kind control over backup filtering. Key user-facing aspects: + +- **For users not using namespace-scoped filter policies**: Zero changes. All existing backups and workflows continue to work identically. The new YAML key is optional. +- **For users adopting namespace-scoped filter policies**: Create a ConfigMap with the `namespacedFilterPolicies` section and reference it via `BackupSpec.ResourcePolicy` (or the existing `--resource-policies-configmap` flag). The backup will selectively include/exclude resources per namespace based on the filter rules. +- **For users already using ResourcePolicy for volume policies**: Add the `namespacedFilterPolicies` section to the same ConfigMap. Both volume policies and namespace-scoped filters coexist. +- **For restore from a namespace-filtered backup**: No changes to restore workflow. Restore processes whatever is in the archive. Users can use existing `RestoreSpec.IncludedNamespaces` for additional filtering at restore time. +- **`velero backup describe` output**: Displays the referenced ResourcePolicy ConfigMap name when configured on the backup. +- **Validation errors**: Reported at backup start when the ResourcePolicy ConfigMap contains invalid `namespacedFilterPolicies` configurations. Consistent with how volume policy validation errors are reported today. + +## Alternatives Considered + +1. **CRD-Based `NamespacedFilters` Field**: Add `NamespacedFilters []NamespaceFilter` directly to `BackupSpec`. Rejected for this iteration due to heavy CRD change overhead. The ResourcePolicy approach achieves the same functionality with less API surface change. + +2. **Flat Fields on NamespacedFilterPolicy (No Per-Kind Selectors)**: Use flat fields (`includedResources`, `labelSelector`, `includedResourceNames`) shared across all kinds within a namespace. Rejected because it cannot express per-kind label selectors or per-kind name lists — a critical requirement for workloads where different resource types have different labels or naming conventions. + +3. **Scoped Label Selectors Only**: Augment existing label selectors with an optional namespace scope. Rejected because it only addresses label-scoped filtering and does not support per-namespace resource type filtering or name filtering. + +4. **Global Name Filter Only**: Add only global name filter fields. Rejected because it only addresses name filtering globally and does not address namespace-scoped or kind-scoped filtering. + +5. **Separate ConfigMap for Namespace Filters**: Use a new `BackupSpec` field pointing to a different ConfigMap (separate from volume policies). Rejected because it adds a new CRD field (which has similar reasons with #1) and splits configuration across multiple ConfigMaps. diff --git a/design/global-backup-volume-policies.md b/design/global-backup-volume-policies.md new file mode 100644 index 0000000000..3f0ae37220 --- /dev/null +++ b/design/global-backup-volume-policies.md @@ -0,0 +1,162 @@ +# Global Backup Volume Policies for Velero + +## Background + +Velero supports [resource policies](./Implemented/handle-backup-of-volumes-by-resources-filters.md) (commonly referred to as "volume policies") that let a user control how volumes are handled during a backup — for example, whether a volume is skipped, backed up via file-system backup (`fs-backup`), snapshotted, or handled by a custom plugin. + +Today these policies are defined per-backup: + +1. A user creates a ConfigMap in the Velero install namespace whose single data key holds a `ResourcePolicies` YAML document (`volumePolicies` and the related include/exclude and fine-grained filter policies). +2. The user opts a specific backup into that ConfigMap with the CLI flag `--resource-policies-configmap`, which sets `Backup.Spec.ResourcePolicy` as a reference to the ConfigMap. +3. When the backup is processed, velero loads the referenced ConfigMap, unmarshals the YAML, builds a `Policies` object, and applies it when performing the backup. + +The limitation today is that volume policies are strictly opt-in **per backup**. An administrator, who usually has the best knowledge of the environment, may want a baseline behavior to apply to *every* backup in the cluster (for example, "always skip volumes from the `gp2` storage class", or "always use `fs-backup` for NFS volumes"). However, today they must remember to attach the same ConfigMap to every backup and every schedule. There is no way to express a cluster-wide default volume policy that is enforced regardless of what an individual backup requests. + +## Goals + +- Introduce "global backup volume policies" that an administrator configures once when the Velero server starts. +- Expose it as a Velero server CLI parameter that points to a ConfigMap in the Velero install namespace. +- When a backup runs, merge the global backup volume policies with the backup's own resource policies ConfigMap (if any) and use the merged result as the effective resource policies for that backup. +- Keep the existing per-backup `--resource-policies-configmap` behavior fully backward compatible when no global policy is configured. + +## Non Goals + +- Changing the schema of the `ResourcePolicies`/`volumePolicies` YAML itself. +- Defining global defaults for anything other than resource policies (e.g. it does not introduce new global backup spec defaults). +- Supporting per-namespace or per-schedule global policy overrides. The "global policies" is a single, server-wide configuration. +- Hot-reloading the global policies ConfigMap without a server restart is out of scope for the initial implementation. +- Support setting other filters in "resource policies" (e.g. include/exclude or fine-grained filters) in the global policy is out of scope for the initial implementation. Only `volumePolicies` will be supported in the global policy for now. + +## Design + +A new Velero server flag, `--global-backup-volume-policies-configmap`, accepts the name of a ConfigMap that lives in the Velero install namespace. The ConfigMap has the exact same format as an existing per-backup resource policies ConfigMap (a single data key holding a `ResourcePolicies` YAML document). + +The flag value is plumbed from the server `Config` into the `backupReconciler`. During `prepareBackupRequest`, in addition to loading the backup's own resource policy (referenced by `Backup.Spec.ResourcePolicy`), Velero loads the global policy ConfigMap. The two `ResourcePolicies` documents are then **merged** into a single effective `ResourcePolicies`, which is compiled into a `Policies` object, validated, and stored on `request.ResPolicies` exactly as today. The rest of the backup pipeline is unchanged because it only consumes `request.ResPolicies`. + +``` + server flag --global-backup-volume-policies-configmap + | + v + Backup.Spec.ResourcePolicy global policies ConfigMap (install ns) + | | + v v + backup-level ResourcePolicies global ResourcePolicies + \ / + \ / + v v + merge() -> effective ResourcePolicies + | + v + Policies (compiled + validated) + | + v + request.ResPolicies (unchanged consumers) +``` + +### Volume Policy only + +The resource policies ConfigMap schema includes both volume policies and include/exclude/fine-grained filter policies. The global backup volume policy only applies to the `volumePolicies` section of the schema. If the global ConfigMap includes any include/exclude/fine-grained filter policies, they are ignored and not merged into the effective policy. In this case, a warning message will be printed in the Velero server logs. +This is a design choice because only the volume policies are more tied to the environment where velero runs, and are more likely to be something an administrator would want to enforce globally. The include/exclude/fine-grained filter policies are more tied to the specific backup use case, and it would be less intuitive for an administrator to have those apply globally across all backups. + +### Validation + +Velero will validate the global backup volume policies ConfigMap at server startup. If the ConfigMap is missing or invalid, the server will fail to start and log an error. This ensures any mistakes in configuration will be caught early. +It should also make sure the validation happens for each backup, because the ConfigMap could be updated or removed after the server starts. If the global policies ConfigMap is missing or invalid at backup time, the backup CR will be put into "FailedValidation" phase, with an appropriate error message in the logs. + +### Merge semantics + +The merge combines two `ResourcePolicies` documents: the global policy (`G`) and the backup-level policy (`B`). The guiding principle is that the global policy provides a baseline, and the backup-level policy is layered with it. + +- **`volumePolicies`**: `volumePolicies` is an ordered list where the *first* matching policy wins (per the existing `Policies.match` logic). The merged list is the concatenation of the backup-level policies followed by the global policies: + + ``` + merged.volumePolicies = B.volumePolicies ++ G.volumePolicies + ``` + + This gives a backup the ability to override the global baseline for a specific volume (because its policy is evaluated first), while still inheriting all global rules that the backup does not override. + +When only the global policy is configured (the backup does not reference a resource policy), the effective policy is the global policy alone. When only the backup policy exists (no global policy configured), behavior is identical to today. + +#### Example + +Global policy ConfigMap (set on the server with `--global-backup-volume-policies-configmap=global-volume-policy`): + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: global-volume-policy + namespace: velero +data: + policies.yaml: | + version: v1 + volumePolicies: + - conditions: + storageClass: + - gp2 + action: + type: skip +``` + +Backup-level policy ConfigMap (referenced with `velero backup create --resource-policies-configmap backup01`): + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: backup01 + namespace: velero +data: + policies.yaml: | + version: v1 + volumePolicies: + - conditions: + nfs: {} + action: + type: fs-backup +``` + +Effective (merged) volume policies used for the backup — backup rules first, then global: + +```yaml +version: v1 +volumePolicies: + - conditions: + nfs: {} + action: + type: fs-backup + - conditions: + storageClass: + - gp2 + action: + type: skip +``` + +### Output of `velero backup describe` + +Currently, the `velero backup describe` command shows the backup-level resource policy. We should update the CLI to make sure the global volume policies are also shown in the output, so that user will not need to check the parameter of velero server. + +## Implementation + +- **Server flag and config.** Add a new field (e.g. `GlobalBackupVolumePoliciesConfigMap`) to the server `Config` struct in `pkg/cmd/server/config/config.go`, register the `--global-backup-volume-policies-configmap` flag in `Config.BindFlags`, and leave its default empty in `GetDefaultConfig` so the feature stays opt-in. +- **Plumb the value into the reconciler.** In `pkg/cmd/server/server.go`, pass the configured ConfigMap name (along with the Velero install namespace) into `controller.NewBackupReconciler`. Add a corresponding parameter and store it as a field on the `backupReconciler` struct in `pkg/controller/backup_controller.go`. +- **Load and merge the policies.** In `internal/resourcepolicies/resource_policies.go`, add a new function (e.g. `GetResourcePoliciesFromBackupWithGlobal`) that, in addition to loading the backup-referenced ConfigMap as `GetResourcePoliciesFromBackup` does today, also loads the global ConfigMap from the install namespace via the existing `getResourcePoliciesFromConfig` helper. After that the function merges the two `ResourcePolicies` documents according to the semantics described above. +- **Call site.** Update `prepareBackupRequest` in `pkg/controller/backup_controller.go` (currently calling `GetResourcePoliciesFromBackup`) to apply the merged policies from the new function. The rest of the backup pipeline remains unchanged. +- **CLI describe output.** Update `DescribeResourcePolicies` in `pkg/cmd/util/output/backup_describer.go` and `DescribeResourcePoliciesInSF` in `pkg/cmd/util/output/backup_structured_describer.go` to also surface the global volume policy ConfigMap that contributed to the backup. + +## Security Considerations + +The Global Backup Volume Policy is read from a ConfigMap in the Velero install namespace, the same trust boundary as existing resource policy ConfigMaps and Velero's own configuration. Setting it requires the ability to pass server flags / edit the Velero deployment, which is already an administrative privilege. No new data is exposed and no new external access patterns are introduced. + +## Compatibility + +- The feature is fully opt-in. If `--global-backup-volume-policies-configmap` is not set (the default), behavior is byte-for-byte identical to today. +- Existing per-backup `--resource-policies-configmap` usage is unchanged; it is simply merged with the global baseline when one is configured. +- Backups created before this feature, and backups that reference no resource policy, transparently start honoring the global policy once it is configured. This is the intended behavior of a "global" policy, but operators should be aware that introducing a global policy changes the effective behavior of backups that previously had no resource policy. +- The behavior of scheduled backup may change when a global backup volume policy is introduced, because the scheduled backup will start honoring the global volume policies. This is an expected change, but administrators should be aware of this when introducing a global policy to an existing velero instance with scheduled backups. +- The merged policy is computed at backup time and is reflected wherever `request.ResPolicies` is consumed. `velero backup describe` should be updated to indicate when a global policy contributed to a backup. + +## Alternatives Considered + +- **Global policies applied only when a backup has no policy of its own.** Simpler, but it makes the global policy a fallback default rather than an enforced baseline, and it cannot express "always do X in addition to whatever the backup wants". Merging is more expressive. +- **Global precedence over backup-level policies** (global volume policies evaluated first). Rejected as the default because it would prevent backups from overriding the baseline for specific volumes. diff --git a/design/restore-filter-enhancement/fine-grained-restore-filters-design.md b/design/restore-filter-enhancement/fine-grained-restore-filters-design.md new file mode 100644 index 0000000000..fd3069b683 --- /dev/null +++ b/design/restore-filter-enhancement/fine-grained-restore-filters-design.md @@ -0,0 +1,878 @@ +# Fine Grained Restore Filters via Resource Policies + +This is a continuation of the work done for backup filters enhancement introduced by [PR 9783](https://github.com/velero-io/velero/pull/9783), referred to as Phase 1 throughout this design. + +## Glossary & Abbreviation + +**Restore Filter**: The mechanism in Velero that determines which resources from a backup archive are restored into the target cluster. Restore filters currently operate on four dimensions: namespace, resource type, label, and cluster scope. +**Global Filter**: A filter that applies uniformly across all namespaces in a restore. All existing Velero restore filters are global filters. +**Namespace-Scoped Filter**: A filter that applies only within specific namespaces, overriding the global filter for those namespaces. This is the capability introduced by this design. +**ClusterScopedFilterPolicy**: A global filter for cluster-scoped resources that allows per-kind label selectors and name patterns, functioning similarly to `NamespacedFilterPolicy` but applied to cluster-scoped resources globally. Mirrors the backup-side concept of the same name. +**Resource Filter**: A filter rule that pairs one or more resource kinds with their own label selector and/or name patterns. Multiple resource filters within a namespace-scoped policy allow different filtering criteria for different resource types. +**Resource Name Filter**: A filter that matches individual resource instances by their metadata.name, using glob patterns. This filter dimension was introduced in Phase 1 (backup-side) and is extended to restore in this design. +**Resource Policy**: An existing Velero mechanism where backup behavior rules are defined in a ConfigMap and referenced from `BackupSpec.ResourcePolicy`. Phase 1 extended this with `namespacedFilterPolicies` and `clusterScopedFilterPolicy` for backup. This design adds an analogous `RestoreSpec.ResourcePolicy` for restore, reusing the same ConfigMap format. + +## Background + +### Why Restore-Side Filters? + +Phase 1 enables selective backup — for example, backing up only Deployments and ConfigMaps from `ns-a` while backing up everything from `ns-b`. However, backup-time filtering alone is insufficient for several real-world restore scenarios: + +**Scenario 1 — Selective restore from a full backup.** An organization performs full-cluster backups (all namespaces, all resource types) for disaster recovery. When a specific application needs recovery, the administrator wants to restore only the application's resources (specific resource types, specific names) from a single namespace — without restoring monitoring, logging, or infrastructure resources that exist in the same namespace. Today, `RestoreSpec.IncludedResources` applies globally, so filtering out ConfigMaps means filtering them out of *every* namespace being restored. + +**Scenario 2 — Cross-environment migration with selective resources.** When migrating workloads between clusters, different namespaces may need different resource types restored. A database namespace needs StatefulSets and PVCs but not Deployments; a frontend namespace needs Deployments and Services but not PVCs. The current global filter cannot express this. + +**Scenario 3 — Restore with name-based selection.** A backup contains many ConfigMaps and Secrets in a namespace (e.g., `app-config`, `app-secret`, `monitoring-config`, `monitoring-secret`). The user wants to restore only the `app-*` resources. Without name-based filtering at restore time, this requires either pre-filtering at backup time (which may not have been done) or post-restore manual cleanup. + +**Scenario 4 — Restore-time override of backup-time filters.** A backup was produced with `namespacedFilterPolicies` that included specific resources per namespace. At restore time, the operator may want to apply *different* per-namespace filters — for example, restoring only a subset of what was backed up, or applying different label selectors to handle environment differences. + +### Existing Restore Filter Mechanisms + +The restore pipeline currently supports: + +| Filter | Scope | Where Applied | +|---|---|---| +| `RestoreSpec.IncludedNamespaces` / `ExcludedNamespaces` | Global | `getOrderedResourceCollection()` | +| `RestoreSpec.IncludedResources` / `ExcludedResources` | Global | `getOrderedResourceCollection()`, `restoreItem()` | +| `RestoreSpec.LabelSelector` / `OrLabelSelectors` | Global | `getSelectedRestoreableItems()` | +| `RestoreSpec.IncludeClusterResources` | Global | `getOrderedResourceCollection()` | +| `RestoreSpec.NamespaceMapping` | Per-namespace | `getSelectedRestoreableItems()` | + +All resource-type, label, and name filters are global. There is no per-namespace override capability. + +### Design Approach: New `RestoreSpec.ResourcePolicy` Field + +Phase 1 avoided CRD changes for backup by reusing the existing `BackupSpec.ResourcePolicy` ConfigMap reference. For restore, no equivalent field exists — `RestoreSpec` has no `ResourcePolicy` field today. + +Two approaches were evaluated: + +**Option A — Reuse the backup's ResourcePolicy ConfigMap.** The restore pipeline could read the `namespacedFilterPolicies` from the backup's ConfigMap. This is rejected because: +- Restore should be able to apply *different* filters than backup +- The backup's ConfigMap may no longer exist at restore time +- The backup's ConfigMap is semantically about backup behavior, not restore +- The ConfigMap may have been updated since the backup was taken +- The ConfigMap may not exist on the target cluster, because it's maybe on a different velero instance. + +**Option B — Add `RestoreSpec.ResourcePolicy` (minimal CRD change).** Add a single `TypedLocalObjectReference` field to `RestoreSpec`, mirroring the existing `BackupSpec.ResourcePolicy` and `RestoreSpec.ResourceModifier` patterns. This is a small, focused CRD change that follows an established pattern in the codebase. + +This design uses **Option B**. The rationale: + +| Consideration | Assessment | +|---|---| +| CRD change size | **Minimal** — one `TypedLocalObjectReference` field, identical pattern to `ResourceModifier` | +| Precedent | `RestoreSpec.ResourceModifier` already uses the exact same pattern (ConfigMap ref loaded in `validateAndComplete()`) | +| Independence from backup | Restore filters are decoupled from backup filters — different ConfigMap, different lifecycle | +| Reuse | The `NamespacedFilterPolicy` and `ClusterScopedFilterPolicy` types from Phase 1 (`internal/resourcepolicies/`) are reused unchanged | + +### Why Not Just Reuse `BackupSpec.ResourcePolicy` Semantics? + +The backup-side `ResourcePolicy` ConfigMap contains multiple policy types (`volumePolicies`, `includeExcludePolicy`, `namespacedFilterPolicies`, `clusterScopedFilterPolicy`). Rather than forcing users to create a ConfigMap with backup-specific sections just to specify restore filters, this design introduces a restore-specific ConfigMap format that contains only `namespacedFilterPolicies` and `clusterScopedFilterPolicy` (and potentially other restore-specific policies in the future). + +The restore-side ConfigMap uses the **same YAML structure** for both sections. The `NamespacedFilterPolicy` and `ClusterScopedFilterPolicy` types are reused without modification. This means: +- Users who already understand the backup-side format can immediately use the restore-side one +- The `internal/resourcepolicies/` validation code is reused +- A single ConfigMap can be used for both backup and restore if the user wants (by specifying it in both `BackupSpec.ResourcePolicy` and `RestoreSpec.ResourcePolicy`) + +## Goals + +- Add a `ResourcePolicy` field to `RestoreSpec` pointing to a ConfigMap with `namespacedFilterPolicies` and/or `clusterScopedFilterPolicy` +- Reuse the `NamespacedFilterPolicy`, `ClusterScopedFilterPolicy`, and `ResourceFilter` types from Phase 1 unchanged +- Apply per-namespace resource type filters, label selectors, and resource name patterns during restore +- Apply per-kind label selectors and name patterns for cluster-scoped resources during restore +- Maintain full backward compatibility — existing restores without `ResourcePolicy` behave exactly as they do today +- Define clear precedence rules for how per-namespace filters interact with global restore filters +- Add corresponding validation in the restore controller +- Update `velero restore describe` output to display per-namespace and cluster-scoped filter information when present +- Ensure restore-side filters work correctly with both filtered and unfiltered backups + +## Non-Goals + +- Modifying the existing `NamespacedFilterPolicy`, `ClusterScopedFilterPolicy`, or `ResourceFilter` types or the `internal/resourcepolicies/` package structure (reused as-is from Phase 1) +- Adding volume policies or include/exclude policies to the restore-side ResourcePolicy ConfigMap +- Supporting regex patterns for resource names (glob patterns only, consistent with Phase 1) +- Modifying the restore plugin `ResourceSelector` system (`AppliesTo()` / `resolvedAction.ShouldUse()`) +- CLI flags for inline specification of namespace-scoped restore filters (configuration is in ConfigMap YAML) + +## Architecture of Restore-Side Filters + +### Configuration Model + +The restore-side filters are defined in a ConfigMap referenced by a new `RestoreSpec.ResourcePolicy` field. The ConfigMap YAML format reuses the `namespacedFilterPolicies` and `clusterScopedFilterPolicy` sections from Phase 1, with the same `resourceFilters` model: + +```yaml +version: v1 +clusterScopedFilterPolicy: + # NEW: global overrides for cluster-scoped resources during restore + resourceFilters: + - kinds: [ClusterRole, ClusterRoleBinding] + names: ["my-app-*"] + - kinds: [CustomResourceDefinition] + labelSelector: + matchLabels: + app: my-app +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret, Deployment] + labelSelector: + matchLabels: + app: my-app + - namespaces: + - ns-b + resourceFilters: + - kinds: [Deployment] + names: [app-1, app-2] + - kinds: [ConfigMap] + labelSelector: + matchLabels: + app: my-service +``` + +The restore-side ConfigMap does **not** require `volumePolicies` or `includeExcludePolicy` sections. Those are backup-specific. The YAML parser will ignore unknown fields gracefully, so a user can technically point to the same ConfigMap used for backup — the restore pipeline will only read `namespacedFilterPolicies` and `clusterScopedFilterPolicy`. + +### The `resourceFilters` Model + +Each `namespacedFilterPolicies` entry targets one or more namespaces and contains a `resourceFilters` array. Each entry in `resourceFilters` pairs one or more resource kinds with their own label selector and name patterns: + +```yaml +namespacedFilterPolicies: + - namespaces: [ns-a] + resourceFilters: + - kinds: [ConfigMap, Secret] # these kinds share a selector + labelSelector: + matchLabels: + app: my-app + names: ["app-*"] + - kinds: [Deployment] # this kind has its own selector + names: [workload-1, workload-2] + - kinds: [StatefulSet] # this kind has no extra filtering +``` + +Only resource kinds listed in `resourceFilters` entries are restored for the matched namespaces; unlisted kinds are implicitly excluded (globally excluded kinds cannot be re-included — see precedence model). + +#### Label selectors (`matchLabels` / `matchExpressions`) + +`labelSelector` and each entry of `orLabelSelectors` use the standard Kubernetes selector shape (same as `RestoreSpec.labelSelector`): + +```yaml +labelSelector: + matchLabels: + app: my-app + matchExpressions: + - key: environment + operator: In + values: [prod, staging] + - key: do-not-restore + operator: DoesNotExist +``` + +Supported `matchExpressions` operators: `In`, `NotIn`, `Exists`, `DoesNotExist`. Prefer `In` for value-OR on one key; use `orLabelSelectors` for OR across independent multi-key groups. `labelSelector` and `orLabelSelectors` cannot co-exist in the same `resourceFilters` entry. + +```yaml +orLabelSelectors: + - matchLabels: + tier: frontend + matchExpressions: + - key: track + operator: In + values: [canary] + - matchLabels: + tier: backend +``` + +#### Peek-and-Map Fallback for Unresolved Kinds + +The `kinds` field accepts both plural resource names (e.g., `configmaps`, `mycustomkinds.mygroup.io`) and singular `Kind` names (e.g., `ConfigMap`, `MyCustomKind`). + +To ensure consistent case-insensitive behavior across all code paths, Velero normalizes all input `kinds` to lowercase *before* attempting discovery or fallback matching. + +During a restore, Velero attempts to resolve `Kind` names to fully-qualified plural resource names using the cluster's discovery helper. However, for Custom Resources (CRDs), the CRD might not exist in the cluster yet when the restore begins. + +To handle this, Velero implements a **peek-and-map fallback**: +1. If a normalized `Kind` cannot be resolved via the discovery helper at the start of the restore, Velero stores the normalized string as provided in the policy. +2. Later, when iterating through the backup tarball, if Velero encounters a resource type (e.g., `mycustomkinds.mygroup.io`) that doesn't match any resolved filters, it peeks at the `Kind` of the first item in the tarball for that resource type. +3. It then checks if this actual `Kind` (case-insensitively) matches any of the unresolved normalized strings in the user's policy. +4. If a match is found, the filter is applied and cached for subsequent lookups. + +This ensures that users can intuitively write `kinds: [MyCustomKind]` and it will work reliably, even if the CRD hasn't been restored yet. This logic applies to both `namespacedFilterPolicies` and `clusterScopedFilterPolicy`. + +#### Catch-All Resource Filter (Empty `kinds` or `["*"]`) + +A `ResourceFilter` entry with an empty (or omitted) `kinds` field, or a field explicitly set to `["*"]`, acts as a **catch-all**. Its `labelSelector` or `orLabelSelectors` (if provided) is applied to **all resource types in the namespace that are not already matched by a kind-specific filter entry**. If no selectors are provided, all unlisted resources are included. Using `["*"]` is highly recommended as it makes the catch-all intention explicit and self-documenting. + +**Rules for catch-all entries:** +- At most **one** catch-all entry is allowed per `NamespacedFilterPolicy`. +- `names` and `excludedNames` are **not** supported on catch-all entries. Name patterns are kind-specific by nature and cannot be applied across arbitrary kinds; use kind-specific entries for name-based filtering. +- The catch-all applies to kinds that are **not listed in any other `resourceFilters` entry** in the same policy. Kind-specific entries take precedence over the catch-all. +- A catch-all entry **does not inherit or fall back to `RestoreSpec.LabelSelector`**. If a catch-all entry has no `labelSelector`/`orLabelSelectors`, all unlisted resource kinds in the namespace are included with **no label filtering** — the global label selector is not applied. +- **Catch-all is a `namespacedFilterPolicies`-only feature**. `clusterScopedFilterPolicy` does **not** support catch-all entries (empty or `["*"]` kinds). This is because `clusterScopedFilterPolicy` is a refinement overlay — unlisted cluster-scoped kinds already fall back to global filters by default. A catch-all would conflict with that fallback semantics. Validation rejects catch-all entries in `clusterScopedFilterPolicy`. + +**Evaluation order within a namespace filter policy:** +1. For each resource kind encountered during restore, the system first checks whether a kind-specific `resourceFilters` entry exists for that kind. +2. If a kind-specific entry exists, it is used exclusively (label selectors, name patterns from that entry). +3. If no kind-specific entry exists but a catch-all entry is present, the catch-all's `labelSelector`/`orLabelSelectors` is applied to that kind. +4. If neither a kind-specific entry nor a catch-all entry exists, the kind is excluded from the restore for that namespace. + +### Filter Precedence Model + +The restore-side namespace-scoped filter system layers on top of the existing global restore filter system. The evaluation order is: + +1. **Global namespace filter** (`RestoreSpec.IncludedNamespaces`/`ExcludedNamespaces`) is checked first. A namespace must pass this filter to be considered at all. `namespacedFilterPolicies` cannot override namespace exclusion — if a namespace is excluded globally, no filter policy entry can bring it back. + +2. **Global resource type filter** (`RestoreSpec.IncludedResources`/`ExcludedResources`) is checked next. A resource type must pass the global filter to be considered. Per-namespace filters can further narrow the set of resource types within a namespace, but cannot include a resource type that is globally excluded. + +3. **Per-namespace filter lookup.** For each namespace that passes the global filters, the system checks whether any `namespacedFilterPolicies` entry matches (by namespace name or glob pattern). If a match is found, the `resourceFilters` array determines what gets restored for that namespace: + - Only resource kinds listed in `resourceFilters[].kinds` are restored (globally excluded kinds cannot be re-included by a per-namespace policy) + - Each kind uses its own `labelSelector`/`orLabelSelectors` from its `ResourceFilter` entry, **replacing** the global label selector for that kind + - Each kind uses its own `names`/`excludedNames` patterns from its `ResourceFilter` entry + +4. **Namespaces without a matching filter policy** continue to use the global filters (`RestoreSpec.IncludedResources`, `RestoreSpec.LabelSelector`, etc.) exactly as they do today. + +5. **If multiple filter policy entries could match the same namespace** (e.g., `team-*` and `team-frontend-*` both matching `team-frontend-prod`), the **first matching policy in the list** is used. **Important: Place more specific patterns before broader patterns** to achieve the intended filtering behavior. + +6. **Namespace mapping** is applied after filter lookup. If `RestoreSpec.NamespaceMapping` maps `ns-a` to `ns-a-restored`, the filter policy lookup uses the *original* namespace name (`ns-a`), since the ConfigMap was authored against the backup's namespace structure. + +**For Cluster-Scoped Resources:** + +1. If `clusterScopedFilterPolicy` is present, it acts as a **refinement overlay** over the existing global filters for cluster-scoped resources. It is NOT an exclusive allowlist. + - If a cluster-scoped kind is listed in its `resourceFilters`, its specific `labelSelector`/`orLabelSelectors` and `names`/`excludedNames` patterns are applied. + - If a cluster-scoped kind is **not listed**, it falls back to the standard global filters (`RestoreSpec.LabelSelector`, etc.). + +2. If `clusterScopedFilterPolicy` is absent, Velero falls back to the existing global filters (`IncludedResources`, `LabelSelector`, etc.) for cluster-scoped resources. + +3. **The `velero.io/exclude-from-backup=true` label** always takes precedence over all filters. Although named for backup, this label is set on resources at backup time and remains present on items in the archive. The restore pipeline honors it: any item carrying this label is skipped regardless of whether it matches global or per-namespace restore filters. + +```mermaid +flowchart TD + A["RestoreSpec Global
IncludedNamespaces / ExcludedNamespaces"] + B{Namespace passes
global filter?} + C[Namespace excluded
from restore] + D{"Resource type passes
IncludedResources / ExcludedResources?"} + E[Resource type excluded
from restore] + G{namespacedFilterPolicies
lookup by original namespace} + H{"For each resource kind:
is kind in resourceFilters?"} + I["Apply namespace kind-specific filters:
- labelSelector / orLabelSelectors
- names / excludedNames"] + J[Kind skipped for
this namespace] + K["Use global filters:
- RestoreSpec LabelSelector
- RestoreSpec OrLabelSelectors"] + L{"Is resource
cluster-scoped?"} + M{"Is clusterScopedFilterPolicy
present?"} + N{"Is kind in clusterScopedFilterPolicy
resourceFilters?"} + O["Apply cluster kind-specific filters:
- labelSelector / orLabelSelectors
- names / excludedNames"] + + L -- Yes --> M + M -- Yes --> N + N -- Yes --> O + N -- No --> K + M -- No --> K + L -- No --> A + A --> B + B -- No --> C + B -- Yes --> D + D -- No --> E + D -- Yes --> G + G -- Match found --> H + H -- Yes --> I + H -- No --> J + G -- No match found --> K +``` + +### Key Difference from Backup-Side Precedence + +Both sides enforce the same fundamental rule: **a per-namespace filter policy cannot re-include a resource kind that has been globally excluded**. The difference lies in which global gate enforces this constraint and how unlisted kinds are handled for namespaces *without* a matching filter policy: + +- **Backup side**: The global exclusion gate is `includeExcludePolicy` (in the ResourcePolicy ConfigMap). It runs first at the resource-type level before any per-namespace lookup occurs. For a namespace that *has* a matching `namespacedFilterPolicies` entry, the per-namespace kind list acts as an exclusive allowlist — only listed kinds are collected, and no fallback to `BackupSpec.IncludedResources` occurs. However, any kind that `includeExcludePolicy` globally excludes remains excluded even if it appears in the per-namespace `resourceFilters`. For a namespace *without* a matching entry, the standard global filters (`BackupSpec.IncludedResources`, `BackupSpec.LabelSelector`, `includeExcludePolicy`) apply as before. See point 6 in the backup design's Filter Precedence Model (`fine-grained-backup-filters-design.md`) for the full treatment, including the warning log emitted when a per-namespace entry lists a globally excluded kind. +- **Restore side**: The global exclusion gate is `RestoreSpec.IncludedResources`/`ExcludedResources` directly on the RestoreSpec. It runs first, globally. For a namespace that *has* a matching `namespacedFilterPolicies` entry, the per-namespace kind list acts as an exclusive allowlist within what the global gate permits — a kind must pass the global filter and be listed in `resourceFilters` to be restored. No fallback to `RestoreSpec.IncludedResources` for additional kinds occurs. For a namespace *without* a matching entry, the standard global filters apply as before. See the "Interaction with Global `IncludedResources`/`ExcludedResources`" entry in the Edge Cases section below for a detailed example. + +In both cases, per-namespace policies are an **allowlist that operates within globally established bounds** — the label selector for a matched kind is fully replaced by the per-namespace one on both sides. + +For label selectors, **replacement** semantics are used on both sides, because label selectors are typically workload-specific and a per-namespace selector is a complete override of the filtering intent for that namespace. + +| | Backup | Restore | +|---|---|---| +| **Data source** | Live cluster — items are listed from Kubernetes API | Backup archive — items are read from tarball | +| **Operator intent** | "What should go into the archive for this namespace?" | "Of what's in the archive, what should I restore for this namespace?" | +| **Global exclusion gate** | `includeExcludePolicy` in ResourcePolicy ConfigMap | `RestoreSpec.IncludedResources` / `ExcludedResources` | +| **Namespaces without a matching policy** | Fall back to `BackupSpec.IncludedResources` + `includeExcludePolicy` | Fall back to `RestoreSpec.IncludedResources` / `ExcludedResources` | +| **Per-namespace label selector** | Replaces global label selector for that kind | Replaces global label selector for that kind | +| **clusterScopedFilterPolicy behavior** | Refinement overlay (unlisted kinds fall back to global) | Refinement overlay (unlisted kinds fall back to global) | + +### Data Flow in the Restore Pipeline + +The restore pipeline has two phases: resource selection and item restore. Namespace-scoped filters are applied in both: + +**Phase A — Resource Selection (`getOrderedResourceCollection()` + `getSelectedRestoreableItems()`)** + +Resources are enumerated from the backup archive (not from the live cluster — this is a key difference from backup). + +- **Resource type check** in `getOrderedResourceCollection()`: The global resource type check still applies. Within the namespace iteration, a per-namespace resource type check is added. If a filter policy matches the current namespace, only kinds listed in `resourceFilters[].kinds` (or matched by a catch-all) are restored — unlisted kinds are skipped for that namespace. Globally excluded kinds cannot be re-included by a per-namespace policy. +- **Label selector** in `getSelectedRestoreableItems()`: The function looks up the filter policy for the current namespace and retrieves the `ResourceFilter` entry for the current resource kind. If found, it uses that entry's `labelSelector`/`orLabelSelectors` instead of the global ones. If not found, the global selectors are used as before. +- **Name pattern check** in `getSelectedRestoreableItems()`: After the label selector check, the item's name is checked against the `ResourceFilter` entry's `names`/`excludedNames` glob patterns for the current kind. + +**Phase B — Item Restore (`restoreItem()`)** + +The `restoreItem()` function is called for each selected item and also for "additional items" requested by restore plugins. + +**Important:** Like the backup-side Stage 2 which is permissive for unlisted kinds requested by plugins, the restore-side Phase B is permissive for AdditionalItems requested by plugins regarding kind, name, and label selectors. This means if a plugin requests an AdditionalItem, it bypasses the fine-grained `namespacedFilterPolicies` and `clusterScopedFilterPolicy` checks, though it must still pass global resource/namespace exclusions. This is intentional to ensure that semantic dependencies (like a PV needed by a PVC) are successfully restored even if their specific resource kind or name pattern wasn't explicitly allowed in the user's namespace-scoped filter policy. + +### Interaction with NamespaceMapping + +When `RestoreSpec.NamespaceMapping` remaps namespaces (e.g., `ns-a` -> `ns-a-staging`), the filter policy lookup uses the **original** (backup-side) namespace name. This is because: + +- The filter ConfigMap is authored against the backup's namespace structure +- The archive directory structure uses the original namespace names +- The `getSelectedRestoreableItems()` function receives `originalNamespace` and applies mapping afterward + +The `getNamespaceFilter()` method on `restoreContext` takes the original namespace name as input. + +### Interaction with Existing Restore Features + +| Feature | Interaction | +|---|---| +| `RestoreSpec.RestorePVs` | Orthogonal — controls PV snapshot restoration, not resource inclusion | +| `RestoreSpec.ExistingResourcePolicy` | Orthogonal — controls overwrite behavior for resources that pass all filters | +| `RestoreSpec.RestoreStatus` | Orthogonal — controls status field restoration for resources that pass all filters | +| `RestoreSpec.Hooks` | Applied to resources that pass all filters. Hooks run regardless of how the item was selected | +| `RestoreSpec.ResourceModifier` | Applied to resources that pass all filters. Modifiers run on resources after filter selection | +| `RestoreSpec.PreserveNodePorts` | Orthogonal — applies to Services that pass all filters | +| Restore Item Actions (plugins) | Plugins may request "additional items." These go through `restoreItem()` which permits them, bypassing the fine-grained filter checks (similar to backup side Stage 2). | + +### Edge Cases and Behavior Documentation + +**Plugin Additional Items (Restore-Side):** +Like the backup side — which is permissive at Stage 2 to allow CSI plugin-injected resources through — the restore side is permissive for AdditionalItems in `restoreItem()`. If a restore plugin requests an additional item, it is allowed to bypass the fine-grained `namespacedFilterPolicies` and `clusterScopedFilterPolicy` kind, name, and label selector checks. This allows plugins to successfully restore dependencies (like a PV needed by a PVC, or a specific Secret) without the user having to explicitly authorize every single dependent resource type in their configuration. Note that these additional items must still pass global resource/namespace exclusions. + +**Exact Namespace Match Priority:** +If a namespace matches both an exact name pattern and a glob pattern across different `namespacedFilterPolicies` entries, the exact match always takes precedence, regardless of list order. This aligns with the backup pipeline behavior and ensures specific overrides are always honored. + +**Multiple Glob Patterns Matching Same Namespace (Incorrect Order):** +```yaml +namespacedFilterPolicies: + - namespaces: ["team-*"] # Broader pattern listed first + resourceFilters: + - kinds: [Deployment, Service] + - namespaces: ["team-frontend-*"] # More specific pattern listed second + resourceFilters: + - kinds: [ConfigMap, Secret, Deployment, Service] +``` +**Behavior:** For namespace `team-frontend-prod`, the broader `team-*` pattern matches first, so only `Deployment` and `Service` are restored. The more specific `team-frontend-*` rule is never reached. + +**Multiple Glob Patterns Matching Same Namespace (Correct Order):** +```yaml +namespacedFilterPolicies: + - namespaces: ["team-frontend-*"] # More specific pattern listed first + resourceFilters: + - kinds: [ConfigMap, Secret, Deployment, Service] + - namespaces: ["team-*"] # Broader pattern listed second + resourceFilters: + - kinds: [Deployment, Service] +``` +**Behavior:** For namespace `team-frontend-prod`, the specific `team-frontend-*` pattern matches first, restoring all specified resources. For `team-backend-dev`, the broader `team-*` pattern matches, restoring only `Deployment` and `Service`. This achieves the intended behavior. + +**Namespace Included Globally But No Matching Filter Policy:** +```yaml +# RestoreSpec includes "production" namespace +# ResourcePolicy has no namespacedFilterPolicies entry for "production" +``` +**Behavior:** The namespace uses global filters exactly as it does today. This is the backward compatibility behavior. + +**Empty ResourceFilters Array:** +```yaml +namespacedFilterPolicies: + - namespaces: ["test-namespace"] + resourceFilters: [] # empty array +``` +**Behavior:** Validation error during restore creation: +``` +namespacedFilterPolicies[0]: at least one resourceFilter must be specified +``` + +**Namespace Pattern with No Matches:** +```yaml +namespacedFilterPolicies: + - namespaces: ["nonexistent-*"] + resourceFilters: [...] +``` +**Behavior:** No error. The filter policy is loaded but never applied since no namespaces match the pattern. + +**Resource Kind Not Present in Target Namespaces:** +```yaml +resourceFilters: + - kinds: ["StatefulSet"] # namespace has no StatefulSets in the backup archive + names: ["workload-1"] +``` +**Behavior:** No error. The filter is applied but finds no matching resources. Empty result set is valid. + +**Conflicting Name Patterns:** +```yaml +resourceFilters: + - kinds: ["ConfigMap"] + names: ["app-*"] + excludedNames: ["app-config"] # conflicts with names pattern +``` +**Behavior:** The `excludedNames` takes precedence. Resources matching `app-*` are included, then `app-config` is excluded. Net result: includes `app-secret`, `app-data`, etc., but excludes `app-config`. + +**Invalid Label Selector Syntax:** +```yaml +resourceFilters: + - kinds: ["Deployment"] + labelSelector: + matchLabels: + "invalid label key!": "value" # invalid key syntax +``` +**Behavior:** Validation error during restore creation when `metav1.LabelSelectorAsSelector()` fails: +``` +namespacedFilterPolicies[0].resourceFilters[0]: invalid label selector: "invalid label key!" is not a valid label key +``` + +**Out-of-Scope Kinds in Filter Entries:** +A user may accidentally list a cluster-scoped kind (e.g., `ClusterRole`) inside a `namespacedFilterPolicies` entry, or a namespace-scoped kind (e.g., `ConfigMap`) inside `clusterScopedFilterPolicy`. The system silently ignores such entries at the archive traversal level: namespace-scoped items are never in the cluster-scope portion of the archive, and vice versa. A warning is logged at restore start so the user can detect the misconfiguration: + +``` +WARN kind "ClusterRole" in namespacedFilterPolicies[0].resourceFilters[1] is a cluster-scoped resource; it will never match in a namespace-scoped filter — did you mean clusterScopedFilterPolicy? +``` + +**Discovery Helper Unavailable:** +If the discovery helper is unavailable during restore initialization, the restore fails with: +``` +failed to resolve namespace filter policies: discovery client unavailable +``` + +**Interaction with Global `IncludedResources`/`ExcludedResources`:** + +`namespacedFilterPolicies` operates within the bounds already established by the global resource type filter — it is a refinement, not a replacement. `RestoreSpec.IncludedResources`/`ExcludedResources` is applied first at the resource-type level, before any per-namespace filter policy is consulted. A namespace-scoped filter policy cannot re-include a resource kind that has been globally excluded. + +Two separate gates are applied in order: +1. **`RestoreSpec.IncludedResources`/`ExcludedResources` runs first**, globally, across all namespaces. It decides which resource types are eligible at all. +2. **`namespacedFilterPolicies` runs second**, within the bounds established by step 1. It can only further restrict kinds that survived the global gate — it cannot widen it. + +```yaml +# RestoreSpec +excludedResources: [secrets] # global — Secrets excluded from all namespaces + +# ResourcePolicy ConfigMap +namespacedFilterPolicies: + - namespaces: [ns-a] + resourceFilters: + - kinds: [ConfigMap, Secret] # Secret listed here is ineffective — globally excluded + labelSelector: + matchLabels: + app: my-app + - kinds: [Deployment] +``` + +**What gets restored from `ns-a`:** +- `ConfigMap` with label `app=my-app` — restored (listed in per-namespace policy, not globally excluded) +- `Secret` with label `app=my-app` — **not restored** (globally excluded by `ExcludedResources`, even though listed in the per-namespace policy) +- `Deployment` — restored (listed in per-namespace policy, not globally excluded) + +The "no fallback to `RestoreSpec.IncludedResources`" rule means that for a namespace *with* a matching policy, only the kinds listed in `resourceFilters` are candidates for restore — `RestoreSpec.IncludedResources` is not consulted to add additional kinds. The global `ExcludedResources` exclusions, however, still apply because they are enforced at an earlier, separate stage. + +To restore `Secret` in specific namespaces, users must remove `secrets` from `ExcludedResources` globally, or restructure their policy. + +A warning is logged at restore start when a `namespacedFilterPolicies` entry lists a kind that is globally excluded: +``` +level=warn msg="namespacedFilterPolicies entry lists a kind that is globally excluded by RestoreSpec.ExcludedResources; the per-namespace filter entry has no effect" kind="secrets" namespacePattern="ns-a" +``` + +> **See also:** The backup-side design's "Interaction with `includeExcludePolicy`" (point 6 in the Filter Precedence Model of `fine-grained-backup-filters-design.md`) documents the structurally identical behavior for backup. The only difference is the global gate: on the backup side it is `includeExcludePolicy` (in the ResourcePolicy ConfigMap); on the restore side it is `RestoreSpec.IncludedResources`/`ExcludedResources` (on the RestoreSpec directly). + +# Detailed Design + +## Workflow + +### Restore Workflow + +The restore workflow is preserved with the following additions. The modules in the existing restore path remain unchanged when `ResourcePolicy` is absent from `RestoreSpec`. + +**Step 1 — Load and parse policies (in `restore_controller.go`, `validateAndComplete()`)** + +The restore controller loads the ConfigMap, similar to how `ResourceModifier` is loaded today: + +The loaded policies are passed through to `runValidatedRestore()` and stored on the `restore.Request`. + +**Step 2 — Resolve namespace and cluster-scoped filter maps (in `restore.go`, `RestoreWithResolvers()`)** + +After existing filter setup, the filter policies are resolved into the runtime maps: + +The `resolveRestoreNamespacedFilterPolicies` function: +- For each `NamespacedFilterPolicy`, iterates its `ResourceFilters` entries +- Resolves kind names to fully-qualified group-resource strings using the discovery helper +- Converts `labelSelector` into a `labels.Selector` via `ToMetaV1LabelSelector` + `metav1.LabelSelectorAsSelector()` +- Converts `orLabelSelectors` into `[]labels.Selector` the same way +- Creates `IncludesExcludes` instances for `names`/`excludedNames` patterns +- Identifies catch-all entries (empty or `["*"]` kinds) and stores them in `catchAllFilter` +- Builds a `resourceFilterMap` keyed by the resolved group-resource string +- Returns both the map and an ordered `namespacedFilterPatterns` slice for first-match traversal + +**Step 3 — Per-namespace resource type check (in `restore.go`, `getOrderedResourceCollection()`)** + +Inside the namespace iteration, after the global namespace check and global resource type check, and before calling `getSelectedRestoreableItems()`: + +**Step 4 — Label selector and name filter (in `restore.go`, `getSelectedRestoreableItems()`)** + +Before the items loop, resolve the effective `ResourceFilter` (hoisted for performance). The function handles three cases in order: + +1. **Namespace-scoped item with a matching `namespacedFilterPolicies` entry** — resolve the effective `ResourceFilter` by checking the kind-specific entry first, then falling back to the catch-all +2. **Cluster-scoped item with the kind listed in `clusterScopedFilterPolicy`** — apply that kind's label/name filters (refinement overlay; unlisted cluster-scoped kinds fall through to global) +3. **All other cases** — fall back to the existing global label selector logic + +**Note on cluster-scoped resources:** There is no separate kind-level skip step in `getOrderedResourceCollection()` for cluster-scoped resources analogous to Step 3. `clusterScopedFilterPolicy` is a refinement overlay — unlisted cluster-scoped kinds are not skipped; they fall through to existing global filter handling. Behavior changes only when the kind is explicitly listed in `clusterScopedFilterMap`, and only in `getSelectedRestoreableItems()` (above). + +### Backup Workflow + +No changes. The backup pipeline is unaffected by this design. + +### Delete Workflow + +No changes. Restore deletion removes the restore metadata. The backup archive is unaffected. + +## Validation + +The following validation is added in `restore_controller.go`'s `validateAndComplete()`: + +1. **ConfigMap existence and format**: Handled by `GetResourcePoliciesFromRestore()`, which returns validation errors if the ConfigMap is missing, malformed, or fails `Policies.Validate()`. + +2. **`ResourcePolicy.Kind` must be `"configmap"`** (case-insensitive): Consistent with `BackupSpec.ResourcePolicy` and `RestoreSpec.ResourceModifier`. + +3. **Namespace filter policy validation** (delegated to `Policies.Validate()`): + - Each filter policy must specify at least one namespace + - Each filter policy must specify at least one resource filter + - Each resource filter without kinds can only be defined once (at most one catch-all), and cannot specify `names`/`excludedNames` + - No duplicate kinds across resource filter entries within the same namespace filter + - `labelSelector` and `orLabelSelectors` cannot co-exist within each resource filter + - No duplicate exact namespace patterns across filter policies (overlapping glob patterns are allowed — first-match semantics handle them at runtime) + - Name/excludedNames patterns must be valid globs + +4. **`clusterScopedFilterPolicy` validation** (delegated to `Policies.Validate()`): + - At least one resourceFilter must be specified + - Each resource filter must specify at least one kind — **catch-all (empty `kinds` or `["*"]`) is NOT permitted in `clusterScopedFilterPolicy`** since it is a refinement overlay rather than an allowlist + - No duplicate kinds across resource filters + - `labelSelector` and `orLabelSelectors` mutual exclusion + - Resource name patterns must be valid globs + +5. **Mutual exclusion with global `OrLabelSelectors`/`LabelSelector`**: If `namespacedFilterPolicies` are present and the `RestoreSpec` also has both `LabelSelector` and `OrLabelSelectors`, the existing validation catches this. No additional validation needed for the interaction — per-namespace selectors simply override the global ones for matching namespaces. + +## ConfigMap Examples + +### Restore-Specific ResourcePolicy ConfigMap + +Restore only Deployments and ConfigMaps (labeled `app=my-app`) from `ns-a`, but everything from `ns-b`: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: restore-filter-policy + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [Deployment, ConfigMap] + labelSelector: + matchLabels: + app: my-app + # ns-b has no filter policy entry, so global filters apply (restore everything) +``` + +Restore CR: + +```yaml +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: selective-restore + namespace: velero +spec: + backupName: full-backup + includedNamespaces: + - ns-a + - ns-b + resourcePolicy: + kind: configmap + name: restore-filter-policy +``` + +### Restore with Name Pattern Filtering + +Restore only `app-*` ConfigMaps and Secrets from `production`: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: app-restore-filter + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [ConfigMap, Secret] + names: ["app-*"] + excludedNames: ["*-tmp", "*-debug"] +``` + +### Catch-All with No Label Selector (Override-Only) + +A user may want to use the global configuration for 99% of resources in a namespace, but only apply a specific name filter to a single kind. A catch-all filter without a label selector achieves this: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: override-only-restore-policy + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [Secret] + names: [my-secret] # Specific override for Secrets + - kinds: ["*"] # Catch-all: NO label selector + # Restores all other kinds unconditionally +``` + +**Result:** +- `Secret` resources: only `my-secret` is restored. +- All other resource types: restored unconditionally (acting like a global fallback). + +### Catch-All with Per-Kind Name Overrides + +Use exact names for specific kinds, and fall back to a label selector for all remaining kinds: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: mixed-restore-filter-policy + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [Deployment] + names: [api-server, worker] # these exact Deployments by name + - kinds: [Secret] + names: [db-credentials, tls-cert] # these exact Secrets by name + - kinds: ["*"] # catch-all for all other kinds + labelSelector: + matchLabels: + backup: "true" # restore by label +``` + +**Result:** +- `Deployment` resources: only `api-server` and `worker` are restored. +- `Secret` resources: only `db-credentials` and `tls-cert` are restored. +- All other resource types: restored only if they carry `backup=true`. + +### Cluster-Scoped Filter Policy + +Restore only specific ClusterRoles and CRDs matching a label: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-restore-filter + namespace: velero +data: + policy: | + version: v1 + clusterScopedFilterPolicy: + resourceFilters: + - kinds: [ClusterRole, ClusterRoleBinding] + names: ["my-app-*"] + - kinds: [CustomResourceDefinition] + labelSelector: + matchLabels: + app: my-app + namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [Deployment, ConfigMap, Secret, StatefulSet, PersistentVolumeClaim] +``` + +### Restore with Glob Namespace Patterns + +Apply the same filter to all namespaces matching a pattern. **Note on Precedence:** Exact namespace matches always take precedence regardless of where they are listed. However, if multiple glob patterns could match a namespace, they are evaluated in the order they appear. Always list specific globs before broad globs. + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: team-restore-filter + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + # Globs must be ordered specific-to-broad + - namespaces: + - "team-frontend-*" # specific pattern match + resourceFilters: + - kinds: [Deployment, Service, ConfigMap] + - namespaces: + - "team-*" # broad pattern + resourceFilters: + - kinds: [Deployment, Service] + + # Exact matches always win, even if placed at the bottom + - namespaces: + - "team-frontend-prod" # exact match + resourceFilters: + - kinds: [Deployment, Service, ConfigMap, Secret, PersistentVolumeClaim] +``` + +**Pattern Matching Results:** +- `team-frontend-prod` → Uses exact match policy (restores 5 resource types) +- `team-frontend-dev` → Uses `team-frontend-*` policy (restores 3 resource types) +- `team-backend-test` → Uses `team-*` policy (restores 2 resource types) +- `app-namespace` → No match, uses global filters + +### Same ConfigMap for Backup and Restore + +A single ConfigMap can be referenced by both `BackupSpec.ResourcePolicy` and `RestoreSpec.ResourcePolicy`. The backup pipeline uses `volumePolicies`, `includeExcludePolicy`, `namespacedFilterPolicies`, and `clusterScopedFilterPolicy`. The restore pipeline uses only `namespacedFilterPolicies` and `clusterScopedFilterPolicy`: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: shared-policy + namespace: velero +data: + policy: | + version: v1 + volumePolicies: + - conditions: + capacity: "0,10Gi" + action: + type: fs-backup + clusterScopedFilterPolicy: + resourceFilters: + - kinds: [ClusterRole, ClusterRoleBinding] + names: ["my-app-*"] + namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [Deployment, ConfigMap, Secret, StatefulSet, PersistentVolumeClaim] +``` + +### Restore CR — No ResourcePolicy (backward compatible) + +Existing restores continue to work exactly as before: + +```yaml +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: full-restore + namespace: velero +spec: + backupName: my-backup + includedNamespaces: + - "*" +``` + +## CLI + +### `velero restore describe` + +The output is extended to display resource policy configmap name when present: + +``` +Name: selective-restore +Namespace: velero +Labels: +Annotations: + +Phase: Completed + +Errors: 0 +Warnings: 0 + +Backup: full-backup + +Namespaces: + Included: ns-a, ns-b + Excluded: + +Resources: + Included: * + Excluded: + Cluster-scoped: auto + +Namespace Mapping: + +Label Selector: + +Resource Policy: restore-filter-policy + +Restore PVs: auto + +... +``` + +### `velero restore create` + +A new `--resource-policies-configmap` flag is added to `velero restore create`, mirroring the existing backup-side flag: + +```bash +velero restore create selective-restore \ + --from-backup full-backup \ + --include-namespaces ns-a,ns-b \ + --resource-policies-configmap restore-filter-policy +``` + +The `--help` output for `velero restore create` is updated to clarify the interaction between global and namespace-scoped filters: + +``` +Restore Filtering Options: + --include-namespaces stringArray namespaces to include in the restore (use '*' for all namespaces) + --exclude-namespaces stringArray namespaces to exclude from the restore + --include-resources stringArray resources to include in the restore, formatted as resource.group + --exclude-resources stringArray resources to exclude from the restore, formatted as resource.group + --include-cluster-resources optionalBool[=true] include cluster-scoped resources + --selector labelSelector only restore resources matching this label selector + --or-selector labelSelector restore resources matching any of the label selectors (can be repeated) + --resource-policies-configmap string reference to a configmap containing resource policies for namespace-scoped and cluster-scoped filtering + +Notes: +- Global filters (--include-resources, --selector, etc.) apply to all included namespaces +- Namespace-scoped filters defined in --resource-policies-configmap refine global filters for matching namespaces (globally excluded kinds cannot be re-included) +- Fine-grained global filter policies defined in --resource-policies-configmap refine global filters for cluster-scoped resources +- Use 'velero restore describe' to view resolved filter policies after restore creation +``` + +## User Perspective + +- **For users not using restore-side filter policies**: Zero changes. All existing restores work identically. +- **For users adopting restore-side filter policies**: Create a ConfigMap with the `namespacedFilterPolicies` and/or `clusterScopedFilterPolicy` sections and reference it via `RestoreSpec.ResourcePolicy` (or `--resource-policies-configmap` CLI flag). The restore will selectively include/exclude resources per namespace. +- **For users already using backup-side filter policies**: Restore-side policies are independent. A backup-side ConfigMap can be reused for restore (both `BackupSpec.ResourcePolicy` and `RestoreSpec.ResourcePolicy` can point to the same ConfigMap), or a different ConfigMap can be used. +- **Interaction with NamespaceMapping**: Filter policies use the original (backup-side) namespace names. If `NamespaceMapping` remaps `ns-a` to `ns-b`, the filter ConfigMap should reference `ns-a`. +- **`velero restore describe`**: Shows per-namespace and cluster-scoped filter details when `ResourcePolicy` is present. +- **Validation errors**: Reported at restore start when the ConfigMap is invalid. + +## Alternatives Considered + +1. **Reuse Backup's ResourcePolicy ConfigMap**: Automatically apply the backup's `namespacedFilterPolicies` during restore without requiring restore-side configuration. Rejected because restore should be independently configurable from backup, and the backup's ConfigMap may not exist at restore time or may have been modified. + +2. **No CRD Change — Annotation-Based Reference**: Use a Velero annotation on the Restore CR to point to the ConfigMap instead of a CRD field. Rejected because annotations are not validated, not documented via `kubectl explain`, and are inconsistent with how the backup side works. + +3. **Embed Filter Policies in RestoreSpec (Full CRD Approach)**: Add `NamespacedFilters []NamespaceFilter` directly to `RestoreSpec`. Rejected because it requires complex nested CRD types, doesn't reuse the Phase 1 ConfigMap infrastructure, and is a drift from backup side design. + +4. **CLI-Only (No CRD Change)**: Express restore filters entirely via CLI flags that get stored as annotations. Rejected because it doesn't support the declarative Restore CR workflow and is not auditable. diff --git a/design/ria-must-include-addtional-items-design.md b/design/ria-must-include-addtional-items-design.md new file mode 100644 index 0000000000..95f6863fda --- /dev/null +++ b/design/ria-must-include-addtional-items-design.md @@ -0,0 +1,357 @@ +# RestoreItemAction Must-Include Additional Items + +## Abstract + +Backup Item Actions (BIAs) can already mark additional items as must-include via `backup.velero.io/must-include-additional-items`, so Velero bypasses resource and namespace exclusion filters when backing those dependencies up. +This proposal adds the same plugin-controlled escape hatch on restore: `restore.velero.io/must-include-additional-items`, so Restore Item Actions (RIAs) can force-restore declared `AdditionalItems` even when they would otherwise be dropped by global restore filters. + +## Glossary & Abbreviation + +**Additional Item**: A resource identifier returned by a Backup/Restore Item Action's `Execute()` result that Velero should process as a dependency of the current item. +**BIA**: Backup Item Action plugin. +**RIA**: Restore Item Action plugin. +**Must-Include**: A plugin-set annotation on the action's `UpdatedItem` that tells Velero to bypass global include/exclude filters for that action's `AdditionalItems`. +**Global Restore Filter**: `RestoreSpec` filters applied uniformly — `IncludedNamespaces`/`ExcludedNamespaces`, `IncludedResources`/`ExcludedResources`, `IncludeClusterResources`, and label selectors. +**Fine-Grained Restore Filter**: Per-namespace / cluster-scoped policies from `RestoreSpec.ResourcePolicy` (`namespacedFilterPolicies`, `clusterScopedFilterPolicy`), as described in [Fine Grained Restore Filters via Resource Policies](https://github.com/velero-io/velero/blob/main/design/restore-filter-enhancement/fine-grained-restore-filters-design.md). +**`resourceMustHave`**: A small hardcoded server-side set of resource types that bypass resource and namespace I/E checks inside `restoreItem()` today (but not `IncludeClusterResources=false`). + +## Background + +### Backup-side precedent + +On backup, a BIA may set `backup.velero.io/must-include-additional-items: "true"` on the returned `UpdatedItem`. +Velero strips that annotation (it is an internal signal, not intended to land on the live object) and passes `mustInclude=true` into recursive `backupItem` calls for that action's `AdditionalItems`. +When `mustInclude` is true, `itemInclusionChecks` skips namespace/resource exclusion checks (and related exclusion labels / fine-grained name filters) so plugin-declared dependencies are not dropped by the user's backup filters. +In-tree CSI BIAs already rely on this for VolumeSnapshot / VolumeSnapshotContent / VolumeSnapshotClass style dependency chains. + +### Restore-side gap + +On restore, RIAs can return `AdditionalItems`, and Velero recursively calls `restoreItem()` for each of them. +That path already bypasses fine-grained restore filters and global label selectors, because those are evaluated earlier in `getOrderedResourceCollection` / `getSelectedRestoreableItems`. +However, `restoreItem()` still enforces global resource includes/excludes, namespace includes/excludes, and `IncludeClusterResources=false`. + +The fine-grained restore filters design explicitly documents this remaining floor: + +> Note that these additional items must still pass global resource/namespace exclusions. + +There is no restore-side equivalent of the BIA must-include annotation. +Plugins that need a hard dependency restored despite a selective restore configuration have no opt-in way to express that, short of relying on the server-side `resourceMustHave` list (which is global, not plugin-scoped, and does not bypass `IncludeClusterResources=false`). + +### Motivating scenario + +Consider a selective restore that includes only application namespaces and excludes storage/snapshot resource types, while a plugin knows that restoring a PVC correctly requires a related cluster-scoped or cross-namespace dependency that exists in the backup archive. +Today the RIA can request that dependency as an `AdditionalItem`, but Velero will skip it at the global exclusion checks inside `restoreItem()`. +With a restore must-include annotation, the plugin can declare the dependency as required and Velero will restore it (provided the object is present in the backup tarball). + +## Goals + +- Add `restore.velero.io/must-include-additional-items` with the same parent-annotation contract as the backup-side must-include annotation. +- When an RIA sets the annotation on `UpdatedItem`, bypass global resource I/E, namespace I/E, and `IncludeClusterResources=false` for that RIA's `AdditionalItems`. +- Keep the change opt-in and backward compatible: restores and plugins that do not set the annotation behave exactly as today. +- Document the trust model, precedence rules, and interaction with existing restore gates for plugin authors and operators. + +## Non-Goals + +- Changing the plugin protobuf / `RestoreItemAction` interface shape (no new RPC fields). +- Changing CRDs or adding CLI flags. +- Changing the `resourceMustHave` list (including any narrowing related to VolumeSnapshotContent). +- Updating in-tree RIAs (CSI or otherwise) to set the new annotation as part of this change. +- Per-additional-item granularity (the annotation applies blanket to all `AdditionalItems` from that RIA invocation, matching BIA). +- Materializing items that were never backed up. + +## High-Level Design + +Mirror the backup workflow: + +1. Introduce annotation constant `restore.velero.io/must-include-additional-items`. +2. After each RIA `Execute()`, if `UpdatedItem` carries the annotation with value `"true"`, strip it and set `mustIncludeAdditionalItems=true`. +3. Pass that boolean into recursive `restoreItem(..., mustInclude)` calls for the action's `AdditionalItems`. +4. When `mustInclude` is true, skip the global resource/namespace/`IncludeClusterResources` exclusion checks inside `restoreItem()`. +5. Keep all non-filter gates unchanged (tarball presence, already-restored, completed Jobs, API errors, wait-for-additional-items, etc.). + +Top-level items from the archive continue to be restored with `mustInclude=false`, so user filters still apply to the primary restore set. + +```mermaid +flowchart TD + startRestore[Start Restore] --> readTarball[Read Item from Backup Tarball] + readTarball --> topLevelRestoreItem["restoreItem(..., mustInclude=false)"] + + topLevelRestoreItem --> checkMustInclude{"mustInclude == true?"} + + checkMustInclude -- No --> checkFilters{"Pass Global Resource/Namespace Filters?"} + checkFilters -- No --> skipItem[Skip Restore] + checkFilters -- Yes --> nonFilterGates["Other gates: isCompleted, already-restored, ..."] + + checkMustInclude -- Yes --> nonFilterGates + + nonFilterGates --> executeRIA[Execute RestoreItemAction] + + executeRIA --> checkSkip{"SkipRestore?"} + checkSkip -- Yes --> skipItem + checkSkip -- No --> checkAnnotation{"Has must-include annotation?"} + + checkAnnotation -- Yes --> stripAnnotation[Strip Annotation] + stripAnnotation --> setFlagTrue["mustIncludeAdditionalItems = true"] + + checkAnnotation -- No --> setFlagFalse["mustIncludeAdditionalItems = false"] + + setFlagTrue --> loopAdditionalItems[Loop over AdditionalItems] + setFlagFalse --> loopAdditionalItems + + loopAdditionalItems --> existsInBackup{"Item file in tarball?"} + existsInBackup -- No --> warnSkip[Warn and skip] + existsInBackup -- Yes --> recursiveRestoreItem["restoreItem(..., mustInclude=mustIncludeAdditionalItems)"] + recursiveRestoreItem --> checkMustInclude +``` + +> The edge `recursiveRestoreItem --> checkMustInclude` is a recursive call (new `restoreItem` stack frame), not a same-frame loop. + +## Detailed Design + +### Annotation constant + +In `pkg/apis/velero/v1/labels_annotations.go`, next to the existing backup constant: + +```go +// Velero checks this annotation to determine whether to skip resource excluding check. +MustIncludeAdditionalItemAnnotation = "backup.velero.io/must-include-additional-items" + +// MustIncludeAdditionalItemRestoreAnnotation is set by RestoreItemActions on the UpdatedItem +// to tell Velero to bypass global resource/namespace exclusion checks (and IncludeClusterResources=false) +// for that action's AdditionalItems. Value must be "true". The annotation is stripped before +// the item is applied to the cluster. +// +// Notice: SkipRestore on the Execute output takes precedence. If SkipRestore is true, the +// annotation is never inspected and AdditionalItems are not processed. +MustIncludeAdditionalItemRestoreAnnotation = "restore.velero.io/must-include-additional-items" +``` + +Only the string value `"true"` enables the bypass (same as backup). + +### `restoreItem` signature + +```go +func (ctx *restoreContext) restoreItem( + obj *unstructured.Unstructured, + groupResource schema.GroupResource, + namespace string, + mustInclude bool, +) (results.Result, results.Result, bool) +``` + +Call sites: + +| Site | `mustInclude` value | +|---|---| +| Top-level restore loop | `false` | +| Recursive additional-item restore after an RIA | derived from that RIA's `UpdatedItem` annotation | + +### Bypass exclusion checks; keep namespace creation + +Today, namespace exclusion and `EnsureNamespaceExistsAndIsReady` share one `if namespace != ""` block in `restoreItem()`. +If must-include only skipped the exclusion check without refactoring, an additional item targeting an excluded namespace would fail because its target namespace was never ensured. + +Required structure: + +```go +if mustInclude { + restoreLogger.Info("Skipping the resource/namespace exclusion checks because the item is marked as must-include") +} else { + if !ctx.resourceIncludesExcludes.ShouldInclude(groupResource.String()) && !ctx.resourceMustHave.Has(groupResource.String()) { + restoreLogger.Info("Not restoring item because resource is excluded") + return warnings, errs, itemExists + } + + if namespace != "" { + if !ctx.namespaceIncludesExcludes.ShouldInclude(obj.GetNamespace()) && !ctx.resourceMustHave.Has(groupResource.String()) { + restoreLogger.Info("Not restoring item because namespace is excluded") + return warnings, errs, itemExists + } + } else { + if boolptr.IsSetToFalse(ctx.restore.Spec.IncludeClusterResources) { + restoreLogger.Info("Not restoring item because it's cluster-scoped") + return warnings, errs, itemExists + } + } +} + +// Namespace creation runs regardless of mustInclude. +if namespace != "" { + nsToEnsure := getNamespace(restoreLogger, archive.GetItemFilePath(ctx.restoreDir, "namespaces", "", obj.GetNamespace()), namespace) + _, nsCreated, err := kube.EnsureNamespaceExistsAndIsReady(nsToEnsure, ctx.namespaceClient, ctx.resourceTerminatingTimeout, ctx.resourceDeletionStatusTracker) + // ... existing error handling and restoredItems bookkeeping ... +} +``` + +Namespace remapping is unchanged: exclusion checks use the original namespace (`obj.GetNamespace()`); namespace creation uses the remapped target `namespace` parameter. + +### Process the annotation after each RIA + +Inside the applicable-actions loop in `restoreItem()`, after `SkipRestore` handling and type-asserting `UpdatedItem`: + +```go +obj = unstructuredObj + +mustIncludeAdditionalItems := false +if annotations := obj.GetAnnotations(); annotations != nil && + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] == "true" { + mustIncludeAdditionalItems = true + restoreLogger.Info("RestoreItemAction marked additional items as must-include; bypassing resource/namespace exclusion checks for them") + delete(annotations, velerov1api.MustIncludeAdditionalItemRestoreAnnotation) + obj.SetAnnotations(annotations) +} + +for _, additionalItem := range executeOutput.AdditionalItems { + // existing tarball stat / unmarshal / namespace mapping ... + w, e, additionalItemExists := ctx.restoreItem( + additionalObj, + additionalItem.GroupResource, + additionalItemNamespace, + mustIncludeAdditionalItems, + ) + // existing merge / filteredAdditionalItems bookkeeping ... +} +``` + +### Filter bypass matrix + +| Gate | Plain AdditionalItem | `resourceMustHave` | RIA `mustInclude=true` | BIA `mustInclude=true` (parity target) | +|---|---|---|---|---| +| Fine-grained policies (kind/name/label) | Bypass (never enter selection Phase B filters) | N/A in `restoreItem` | Bypass (same) | Bypass | +| Global label selectors | Bypass (never re-enter selection) | N/A in `restoreItem` | Bypass (same) | Bypass | +| Global resource I/E | Honored | Bypass | Bypass | Bypass | +| Global namespace I/E | Honored | Bypass | Bypass | Bypass | +| `IncludeClusterResources=false` | Honored | Honored (not bypassed) | Bypass | Bypass | +| Item must exist in backup tarball | Required | Required | Required | N/A (fetched from cluster) | +| `isCompleted` / already-restored / API errors | Still apply | Still apply | Still apply | `DeletionTimestamp` still applies on backup | + +RIA must-include is intentionally a **stronger** override than `resourceMustHave` because it also bypasses `IncludeClusterResources=false`. +That matches BIA must-include semantics (plugin-trusted hard dependencies), rather than widening the hardcoded server list. + +### Interaction with fine-grained restore filters + +Per [Fine Grained Restore Filters via Resource Policies](../restore-filter-enhancement/fine-grained-restore-filters-design.md), plugin additional items already bypass `namespacedFilterPolicies` / `clusterScopedFilterPolicy` kind, name, and label checks. +Those filters live in the selection phases; additional items enter `restoreItem()` directly. + +This proposal only changes the remaining global gates inside `restoreItem()`. +With must-include set, an additional item effectively bypasses **all** restore filters (fine-grained and global). +Without the annotation, behavior is unchanged: fine-grained filters are still bypassed, global exclusions still apply. + +### Interaction with existing restore gates + +#### `SkipRestore` precedence + +If `Execute()` returns `SkipRestore: true`, `restoreItem()` returns before inspecting the annotation, and no `AdditionalItems` are processed. +This mirrors backup-side precedence where `velero.io/skip-from-backup` outranks must-include. + +#### Multi-RIA semantics + +Annotation handling is per RIA invocation inside the actions loop: + +1. RIA N executes → inspect/strip annotation on that `UpdatedItem` → restore that RIA's `AdditionalItems` with the derived flag. +2. RIA N+1 sees the already-stripped object unless it sets the annotation again. + +A later RIA does not inherit an earlier RIA's must-include decision. + +#### Transitive propagation + +The parent's `mustInclude` flag admits the child additional item through filters. +It does **not** automatically force-include grandchildren. +Each RIA level that needs the escape hatch must set the annotation on its own `UpdatedItem`, matching BIA behavior. + +#### Non-filter gates that still apply + +Even when `mustInclude=true`: + +- Missing archive file → warn and skip (existing behavior). +- `isCompleted` resources (e.g. completed Jobs) → skip. +- Already present in `ctx.restoredItems` → skip. +- Create/update API failures → errors as today. +- `WaitForAdditionalItems` / `AreAdditionalItemsReady` polling after the additional-item loop → unchanged. + +### Relationship to `resourceMustHave` + +| Mechanism | Who decides | Bypasses resource/ns I/E | Bypasses `IncludeClusterResources=false` | +|---|---|---|---| +| `resourceMustHave` | Velero server (hardcoded) | Yes | No | +| RIA must-include | Plugin author (annotation) | Yes | Yes | + +The two mechanisms coexist. +This proposal does not migrate in-tree CSI (or other) RIAs onto the annotation. +Doing so would be a separate behavior change: it could force-restore types users explicitly excluded, and would newly restore cluster-scoped dependencies even when `IncludeClusterResources=false`. + +### Plugin usage sketch + +```go +func (p *myRestoreAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: schema.GroupResource{Group: "example.io", Resource: "dependencies"}, Namespace: "dep-ns", Name: "dep-1"}, + }, + }, nil +} +``` + +Plugin authors must ensure the additional item was actually captured in the backup (typically via the corresponding BIA also using `backup.velero.io/must-include-additional-items`). + +### Tests + +Extend restore coverage (existing `TestRestoreActionAdditionalItems` patterns / focused cases) for: + +1. Resource exclusion bypass with annotation; still skipped without annotation. +2. Namespace exclusion bypass **and** target namespace creation. +3. `IncludeClusterResources=false` bypass for cluster-scoped additional items. +4. Annotation stripped from the object applied to the cluster. +5. `SkipRestore: true` prevents additional-item processing even if the annotation is set. +6. Missing tarball entry still warns and skips. +7. Transitive case: child RIA must re-set the annotation for grandchildren. +8. Top-level restore path still passes `mustInclude=false` and honors filters. + +### Documentation + +- Constant doc comment (including `SkipRestore` precedence). +- Plugin-author docs for Restore Item Actions: annotation key/value, blanket scope, filter-bypass matrix, namespace-creation side effect, tarball requirement. + +## Security Considerations + +Installing an RIA that sets this annotation grants that plugin authority to restore dependencies outside the operator's restore filters, including: + +- resources in namespaces the restore excluded (and creation of those target namespaces if needed); +- resource types the restore excluded; +- cluster-scoped resources even when `IncludeClusterResources=false`. + +This matches the existing BIA trust model: item-action plugins are already privileged components of the Velero deployment. +Operators should treat RIA installation as a trust decision. +The annotation is stripped before apply so it does not persist as attacker-controlled cluster state from the backup archive alone; a matching RIA must run and return `AdditionalItems` for the bypass to take effect. + +## Compatibility + +- No CRD or plugin interface changes. +- Existing restores unchanged when no RIA sets the annotation. +- Existing tests that assert additional items are dropped under namespace filters / `IncludeClusterResources=false` remain valid for the no-annotation path. +- Compatible with fine-grained restore filters: additional items already bypass those filters; this proposal only addresses the documented global-exclusion floor. + +## Alternatives Considered + +### Per-item must-include on each `ResourceIdentifier` + +Pros: selective control within one `AdditionalItems` list. +Cons: requires API changes to `ResourceIdentifier` or a parallel structure; diverges from BIA; plugins that need selectivity can already split across actions or omit non-required items. + +Rejected for this proposal; may be revisited later if plugin authors demonstrate a concrete need. + +### Widen `resourceMustHave` instead of a plugin annotation + +Pros: no plugin contract change. +Cons: server-forced, global, not scoped to a plugin call; does not give third-party plugins a general tool; does not match BIA; conflicts with efforts to keep hardcoded force-include lists narrow. + +Rejected — wrong trust model for a general plugin escape hatch. diff --git a/design/volume-policy-pvc-volume-mode-access-modes.md b/design/volume-policy-pvc-volume-mode-access-modes.md new file mode 100644 index 0000000000..1c5abba945 --- /dev/null +++ b/design/volume-policy-pvc-volume-mode-access-modes.md @@ -0,0 +1,359 @@ +# Add PVC VolumeMode and AccessModes as Criteria for Volume Policy + +## Abstract +This proposal extends Velero VolumePolicy conditions with two PVC-based criteria, `pvcVolumeMode` and `pvcAccessModes`. +These conditions allow users to select volumes according to the `volumeMode` and `accessModes` of the associated PersistentVolumeClaim (PVC), enabling backup behavior such as skipping block-mode PVCs or choosing a specific backup method for volumes with selected access modes. + +## Background +Velero VolumePolicy already supports selecting volumes by attributes such as capacity, storage class, volume source, volume type, PVC labels, and PVC phase. +PVC metadata and spec fields are often the most direct way for users to express the intended storage semantics of a workload. + +Kubernetes PVCs include a `spec.volumeMode` field that describes whether the volume is exposed as a filesystem or as a raw block device. +The field supports values such as `Filesystem` and `Block`. + +Kubernetes PVCs also include a `spec.accessModes` field that describes how the volume can be mounted. +Common values are `ReadWriteOnce`, `ReadOnlyMany`, `ReadWriteMany`, and `ReadWriteOncePod`. +For resource policies, `pvcAccessModes` uses an exact set match against the PVC's `spec.accessModes`, so a policy does not match PVCs that have missing or additional access modes. + +## Goals +- Add a `pvcVolumeMode` VolumePolicy condition to match volumes by a single `spec.volumeMode` value of their associated PVC. +- Add a `pvcAccessModes` VolumePolicy condition to match volumes whose associated PVC has exactly the configured `spec.accessModes` values, regardless of order. +- Keep the new conditions consistent with existing VolumePolicy behavior, where all conditions in a policy must match and the first matching policy wins. + +## Non-Goals +- This proposal does not add new VolumePolicy actions. +- This proposal does not change how PVCs are discovered or passed into the resource policy matching code. +- This proposal does not add set-based or negative matching operators such as `NotIn`, `Exists`, or `DoesNotExist`. +- This proposal does not change Kubernetes PVC semantics or validate storage provider capabilities. + +## Use-cases/Scenarios + +### Skip block-mode PVCs +A user wants to skip volumes whose associated PVC is configured with raw block volume mode. + +```yaml +version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Block + action: + type: skip +``` + +### Snapshot filesystem PVCs +A user wants to use snapshots only for volumes whose associated PVC has filesystem mode. + +```yaml +version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Filesystem + action: + type: snapshot +``` + +### Match PVCs by access mode +A user wants to apply a policy only to PVCs whose `spec.accessModes` is exactly `ReadWriteOnce`. + +```yaml +version: v1 +volumePolicies: +- conditions: + pvcAccessModes: + - ReadWriteOnce + action: + type: skip +``` + +### Match an exact access mode set +A user wants to match volumes whose associated PVC access modes are exactly `ReadOnlyMany` and `ReadWriteMany`. +A PVC that includes only one of these modes, or includes additional modes, does not match. + +```yaml +version: v1 +volumePolicies: +- conditions: + pvcAccessModes: + - ReadOnlyMany + - ReadWriteMany + action: + type: snapshot +``` + +### Combine PVC spec criteria +A user wants to select block-mode PVCs whose access modes are exactly `ReadWriteOnce`. +Because VolumePolicy conditions are conjunctive, the volume must satisfy both conditions. + +```yaml +version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Block + pvcAccessModes: + - ReadWriteOnce + action: + type: snapshot +``` + +## High-Level Design +The VolumePolicy condition schema is extended with two optional fields, `pvcVolumeMode` and `pvcAccessModes`. +`pvcVolumeMode` is represented as a single string value in the resource policy YAML. +`pvcAccessModes` is represented as a string list in the resource policy YAML. + +The internal `structuredVolume` representation is extended to store the associated PVC's volume mode and access modes. +The existing PVC parsing path populates these fields when a PVC is available in `VolumeFilterData`. + +The policy builder creates a `pvcVolumeModeCondition` when `pvcVolumeMode` is specified and creates a `pvcAccessModesCondition` when `pvcAccessModes` is specified. +The existing matching flow remains unchanged: each condition implements the `volumeCondition` interface, all conditions in a policy must match, and the first matching policy's action is returned. + +## Detailed Design + +### Resource policy YAML schema +Two new fields are added under `volumePolicies[].conditions`. + +```yaml +version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Block + pvcAccessModes: + - ReadWriteOnce + - ReadWriteMany + action: + type: snapshot +``` + +`pvcVolumeMode` is a string. +The intended values are Kubernetes PVC volume mode values, including `Filesystem` and `Block`. +The condition matches only when the PVC volume mode value observed by Velero exactly equals the configured value. +Matching is case-sensitive, so `block` does not match `Block`. + +`pvcAccessModes` is a list of strings. +The intended values are Kubernetes PVC access mode values, including `ReadWriteOnce`, `ReadOnlyMany`, `ReadWriteMany`, and `ReadWriteOncePod`. +The condition matches only when the configured access modes exactly equal the PVC's `spec.accessModes`, ignoring order. +Matching is case-sensitive, so `readwriteonce` does not match `ReadWriteOnce`. + +The implementation validates that `pvcVolumeMode`, when present, is a string. +The implementation validates that `pvcAccessModes`, when present, is a list of strings. +The implementation does not strictly reject unknown string values so that the condition format remains tolerant of Kubernetes additions or storage-provider-specific behavior. +Unknown `pvcVolumeMode` values match only when the PVC has the same string value, and unknown `pvcAccessModes` values match only as part of the same exact access-mode set. + +### Volume condition struct +The parsed condition struct is extended as follows. + +```go +type volumeConditions struct { + Capacity string `yaml:"capacity,omitempty"` + StorageClass []string `yaml:"storageClass,omitempty"` + NFS *nFSVolumeSource `yaml:"nfs,omitempty"` + CSI *csiVolumeSource `yaml:"csi,omitempty"` + VolumeTypes []SupportedVolume `yaml:"volumeTypes,omitempty"` + PVCLabels map[string]string `yaml:"pvcLabels,omitempty"` + PVCPhase []string `yaml:"pvcPhase,omitempty"` + PVCVolumeMode string `yaml:"pvcVolumeMode,omitempty"` + PVCAccessModes []string `yaml:"pvcAccessModes,omitempty"` +} +``` + +### Structured volume data +The internal `structuredVolume` is extended with `pvcVolumeMode` and `pvcAccessModes`. + +```go +type structuredVolume struct { + capacity resource.Quantity + storageClass string + nfs *nFSVolumeSource + csi *csiVolumeSource + volumeType SupportedVolume + pvcLabels map[string]string + pvcPhase string + pvcVolumeMode string + pvcAccessModes []string +} +``` + +When a PVC is available, `parsePVC` extracts PVC attributes into `structuredVolume` for later condition evaluation. +This parsing step does not create or imply a `pvcVolumeMode` policy condition; `pvcVolumeMode` only constrains matching when the user explicitly configures `conditions.pvcVolumeMode` in the VolumePolicy. +Velero uses `pvc.Spec.VolumeMode` as-is when it is present. +If `pvc.Spec.VolumeMode` is nil, `pvcVolumeMode` remains empty and does not match any non-empty `pvcVolumeMode` condition. +If `pvc.Spec.AccessModes` is empty, `pvcAccessModes` remains empty and does not match any non-empty `pvcAccessModes` condition. + +```go +func (s *structuredVolume) parsePVC(pvc *corev1api.PersistentVolumeClaim) { + if pvc != nil { + if len(pvc.GetLabels()) > 0 { + s.pvcLabels = pvc.Labels + } + s.pvcPhase = string(pvc.Status.Phase) + if pvc.Spec.VolumeMode != nil { + s.pvcVolumeMode = string(*pvc.Spec.VolumeMode) + } + if len(pvc.Spec.AccessModes) > 0 { + s.pvcAccessModes = make([]string, 0, len(pvc.Spec.AccessModes)) + for _, accessMode := range pvc.Spec.AccessModes { + s.pvcAccessModes = append(s.pvcAccessModes, string(accessMode)) + } + } + } +} +``` + +### PVC volume mode condition +`pvcVolumeModeCondition` matches when the associated PVC's parsed volume mode exactly equals the configured value. +The comparison is case-sensitive and does not normalize values. +An empty configured value is treated as no constraint and always matches, consistent with other VolumePolicy conditions. +A non-empty configured value does not match if no PVC volume mode is available. + +```go +type pvcVolumeModeCondition struct { + volumeMode string +} + +func (c *pvcVolumeModeCondition) match(v *structuredVolume) bool { + if c.volumeMode == "" { + return true + } + if v.pvcVolumeMode == "" { + return false + } + return v.pvcVolumeMode == c.volumeMode +} +``` + +### PVC access modes condition +`pvcAccessModesCondition` matches when the configured access modes exactly equal the associated PVC's access modes, ignoring order. +The comparison is case-sensitive and does not normalize values. +An empty configured list is treated as no constraint and always matches. +A non-empty configured list does not match if the structured volume has no PVC access modes, has a different number of access modes, or has a different access-mode set. + +```go +type pvcAccessModesCondition struct { + accessModes []string +} + +func (c *pvcAccessModesCondition) match(v *structuredVolume) bool { + if len(c.accessModes) == 0 { + return true + } + if len(v.pvcAccessModes) == 0 || len(v.pvcAccessModes) != len(c.accessModes) { + return false + } + + return sets.New(c.accessModes...).Equal(sets.New(v.pvcAccessModes...)) +} +``` + +### Condition validation +Both `pvcVolumeModeCondition` and `pvcAccessModesCondition` implement the `validate()` method required by the `volumeCondition` interface. +The `validate()` method returns nil for both conditions. + +```go +func (c *pvcVolumeModeCondition) validate() error { + return nil +} + +func (c *pvcAccessModesCondition) validate() error { + return nil +} +``` + +YAML shape validation is handled when resource policy conditions are unmarshaled. +`pvcVolumeMode` must be a string, and `pvcAccessModes` must be a list of strings. +Condition-level validation intentionally does not reject unknown string values. +This keeps the policy format forward-compatible with future Kubernetes values and consistent with other string-based VolumePolicy conditions. +Unknown values simply do not match normal PVCs unless the evaluated PVC has the same exact value or access-mode set. + +### Policy builder integration +The policy builder appends the new conditions only when the corresponding YAML fields are present. + +```go +func (p *Policies) BuildPolicy(resPolicies *ResourcePolicies) error { + for _, vp := range resPolicies.VolumePolicies { + con, err := unmarshalVolConditions(vp.Conditions) + if err != nil { + return errors.WithStack(err) + } + + // Existing conditions are appended here. + + if con.PVCVolumeMode != "" { + volP.conditions = append(volP.conditions, &pvcVolumeModeCondition{volumeMode: con.PVCVolumeMode}) + } + if len(con.PVCAccessModes) > 0 { + volP.conditions = append(volP.conditions, &pvcAccessModesCondition{accessModes: con.PVCAccessModes}) + } + } + return nil +} +``` + +### Matching behavior with other conditions +The new conditions follow the existing VolumePolicy matching behavior. +Within a single policy, every configured condition must match. +If `pvcVolumeMode` is omitted from a policy, Velero does not add a volume mode condition and the policy does not restrict volume mode. +`pvcVolumeMode` and `pvcAccessModes` are PVC-specific conditions and only match when the volume policy evaluation has associated PVC data. +For non-PVC volumes such as `emptyDir`, `configMap`, or inline volumes without an associated PVC, the parsed PVC fields are empty and policies requiring `pvcVolumeMode` or `pvcAccessModes` do not match. +Across multiple policies, the first matching policy wins. + +For example, this policy matches only PVC-backed volumes that are both `Block` mode and have exactly `ReadWriteOnce` as their access modes. + +```yaml +version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Block + pvcAccessModes: + - ReadWriteOnce + action: + type: snapshot +``` + +## Alternatives Considered + +### A single `pvcSpec` condition object +One alternative is to add a nested object such as `pvcSpec.volumeMode` and `pvcSpec.accessModes`. +This was not chosen because existing PVC-based VolumePolicy conditions use flat field names such as `pvcLabels` and `pvcPhase`. +Flat names keep the YAML concise and consistent with existing conditions. + +### List-based `pvcVolumeMode` +One alternative is to make `pvcVolumeMode` a list, similar to `pvcPhase`. +This was not chosen because Kubernetes PVC `spec.volumeMode` is a single value and the policy condition is intended to describe an exact match against that value. +Using a string avoids implying that multiple volume modes can apply to one PVC. + +### Contains-based access mode matching +Another alternative is to make `pvcAccessModes` match when any or all configured access modes are present on the PVC. +This was not chosen because contains-based matching would also select PVCs with additional access modes. +Using exact set matching keeps `pvcAccessModes` consistent with `pvcVolumeMode`'s exact-match behavior and avoids matching PVCs whose access mode set differs from the policy. + +### Strict validation of allowed Kubernetes values +Another alternative is to reject `pvcVolumeMode` or `pvcAccessModes` values that are not currently known Kubernetes constants. +This was not chosen because accepting strings is more forward-compatible and keeps behavior consistent with other string-based resource policy conditions. +Invalid or unknown values naturally fail to match unless a PVC has the same value. + +## Security Considerations +This proposal does not introduce new privileges or access to additional Kubernetes resources. +It only uses PVC data already available to the volume policy matching path. + +The new conditions can cause Velero to skip or choose different backup actions for matched volumes. +Users should review policy configuration carefully because an overly broad policy can exclude data from backup or select an unintended backup method. + +## Compatibility +The new fields are optional and do not affect existing resource policy files. +Existing VolumePolicy behavior remains unchanged when `pvcVolumeMode` and `pvcAccessModes` are not configured. + +PVCs without a parsed `spec.volumeMode` value do not match non-empty `pvcVolumeMode` conditions. +PVCs without `spec.accessModes` do not match non-empty `pvcAccessModes` conditions. + +Unknown `pvcVolumeMode` or `pvcAccessModes` string values in a policy are accepted as strings but will not match normal Kubernetes PVCs unless the evaluated PVC has the same exact value or access-mode set. + +## Implementation +Implementation requires changes in the resource policies package and documentation. + +- Extend `volumeConditions` with `PVCVolumeMode string` and `PVCAccessModes []string`. +- Extend `structuredVolume` with `pvcVolumeMode string` and `pvcAccessModes []string`. +- Update `parsePVC` to populate the new fields from the PVC spec. +- Add `pvcVolumeModeCondition` and `pvcAccessModesCondition` implementations. +- Update `Policies.BuildPolicy` to append the new conditions. +- Add YAML type validation to ensure `pvcVolumeMode` is a string and `pvcAccessModes` is a string list. +- Add unit tests for parsing, validation, condition matching, and end-to-end `GetMatchAction` behavior. +- Update user documentation in `site/content/docs/main/resource-filtering.md`. diff --git a/go.mod b/go.mod index a9ae53ba4b..43d4da471e 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 github.com/bombsimon/logrusr/v3 v3.1.0 + github.com/cockroachdb/errors v1.13.0 github.com/evanphx/json-patch/v5 v5.9.11 github.com/fatih/color v1.18.0 github.com/gobwas/glob v0.2.3 @@ -24,31 +25,29 @@ require ( github.com/google/uuid v1.6.0 github.com/hashicorp/go-hclog v1.6.3 github.com/hashicorp/go-plugin v1.7.0 - github.com/joho/godotenv v1.3.0 github.com/kopia/kopia v0.16.0 github.com/kubernetes-csi/external-snapshotter/client/v8 v8.4.0 + github.com/netresearch/go-cron v0.15.0 github.com/onsi/ginkgo/v2 v2.28.3 github.com/onsi/gomega v1.40.0 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 - github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_model v0.6.2 - github.com/robfig/cron/v3 v3.0.1 github.com/sirupsen/logrus v1.9.4 github.com/spf13/afero v1.15.0 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 github.com/vmware-tanzu/crash-diagnostics v0.4.3 - go.uber.org/zap v1.27.1 - golang.org/x/mod v0.36.0 + go.uber.org/zap v1.28.0 + go.yaml.in/yaml/v3 v3.0.4 + golang.org/x/mod v0.40.0 golang.org/x/oauth2 v0.36.0 - golang.org/x/sys v0.46.0 - golang.org/x/text v0.38.0 + golang.org/x/sys v0.47.0 + golang.org/x/text v0.41.0 google.golang.org/api v0.283.0 - google.golang.org/grpc v1.81.1 + google.golang.org/grpc v1.82.1 google.golang.org/protobuf v1.36.11 - gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.33.12 k8s.io/apiextensions-apiserver v0.33.12 k8s.io/apimachinery v0.33.12 @@ -74,7 +73,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect @@ -96,6 +95,8 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chmduquesne/rollinghash v4.0.0+incompatible // indirect github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/redact v1.1.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/edsrzf/mmap-go v1.2.0 // indirect @@ -105,6 +106,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/getsentry/sentry-go v0.46.0 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect @@ -140,6 +142,8 @@ require ( github.com/klauspost/crc32 v1.3.0 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/klauspost/reedsolomon v1.12.6 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -160,10 +164,12 @@ require ( github.com/philhofer/fwd v1.2.0 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.67.4 // indirect github.com/prometheus/procfs v0.16.1 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/rs/xid v1.6.0 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/stretchr/objx v0.5.2 // indirect @@ -172,7 +178,7 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/zeebo/blake3 v0.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.42.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.43.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect go.opentelemetry.io/otel v1.44.0 // indirect @@ -182,21 +188,21 @@ require ( go.opentelemetry.io/otel/trace v1.44.0 // indirect go.starlark.net v0.0.0-20241226192728-8dfa5b98479f // indirect go.uber.org/multierr v1.11.0 // indirect - go.yaml.in/yaml/v2 v2.4.3 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.53.0 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect + golang.org/x/crypto v0.55.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/net v0.55.0 // indirect - golang.org/x/sync v0.21.0 // indirect - golang.org/x/term v0.44.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/term v0.45.0 // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.45.0 // indirect + golang.org/x/tools v0.49.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect diff --git a/go.sum b/go.sum index 8bc7b9ebed..e55a34789e 100644 --- a/go.sum +++ b/go.sum @@ -50,8 +50,8 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgv github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 h1:IEjq88XO4PuBDcvmjQJcQGg+w+UaafSy8G5Kcb5tBhI= github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5/go.mod h1:exZ0C/1emQJAw5tHOaUDyY1ycttqBAPcxuzf7QbY6ec= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 h1:rIkQfkCOVKc1OiRCNcSDD8ml5RJlZbH/Xsq7lbpynwc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0/go.mod h1:RD2SsorTmYhF6HkTmDw7KmPYQk8OBYwTkuasChwv7R4= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0 h1:4LP6hvB4I5ouTbGgWtixJhgED6xdf67twf9PoY96Tbg= @@ -118,6 +118,12 @@ github.com/chmduquesne/rollinghash v4.0.0+incompatible h1:hnREQO+DXjqIw3rUTzWN7/ github.com/chmduquesne/rollinghash v4.0.0+incompatible/go.mod h1:Uc2I36RRfTAf7Dge82bi3RU0OQUmXT9iweIcPqvr8A0= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/cockroachdb/errors v1.13.0 h1:BoCcJeiP9hpBJDETkX19qi8Tb8So37srSsp3stTaDMQ= +github.com/cockroachdb/errors v1.13.0/go.mod h1:bjxt/4E5+OyuAnacpTIU9rn2mzPu1VlthvHP+xpROq0= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= @@ -157,12 +163,16 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/getsentry/sentry-go v0.46.0 h1:mbdDaarbUdOt9X+dx6kDdntkShLEX3/+KyOsVDTPDj0= +github.com/getsentry/sentry-go v0.46.0/go.mod h1:evVbw2qotNUdYG8KxXbAdjOQWWvWIwKxpjdZZIvcIPw= github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= @@ -248,8 +258,6 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= @@ -333,6 +341,8 @@ github.com/mxk/go-vss v1.2.0 h1:JpdOPc/P6B3XyRoddn0iMiG/ADBi3AuEsv8RlTb+JeE= github.com/mxk/go-vss v1.2.0/go.mod h1:ZQ4yFxCG54vqPnCd+p2IxAe5jwZdz56wSjbwzBXiFd8= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= +github.com/netresearch/go-cron v0.15.0 h1:pu+dhMZjBao9m5IpYe0o+zcNFlP94Z7TDHrORQk+/t0= +github.com/netresearch/go-cron v0.15.0/go.mod h1:79iktHfV90py3jcaFUtWcGSKbZXRev+WwoLMyV5eMvo= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= @@ -349,8 +359,11 @@ github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= @@ -366,8 +379,7 @@ github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+L github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= -github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= -github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= @@ -428,8 +440,8 @@ github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/detectors/gcp v1.42.0 h1:kpt2PEJuOuqYkPcktfJqWWDjTEd/FNgrxcniL7kQrXQ= -go.opentelemetry.io/contrib/detectors/gcp v1.42.0/go.mod h1:W9zQ439utxymRrXsUOzZbFX4JhLxXU4+ZnCt8GG7yA8= +go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU= +go.opentelemetry.io/contrib/detectors/gcp v1.43.0/go.mod h1:RyaZMFY7yi1kAs45S6mbFGz8O8rqB0dTY14uzvG4LCs= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= @@ -454,36 +466,36 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= -go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= -go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= -golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -495,22 +507,22 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= -golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= -golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -527,8 +539,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1: google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= -google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/hack/build-image/Dockerfile b/hack/build-image/Dockerfile index d12b382b8b..30ed17d58c 100644 --- a/hack/build-image/Dockerfile +++ b/hack/build-image/Dockerfile @@ -98,7 +98,7 @@ RUN ARCH=$(go env GOARCH) && \ # get golangci-lint # Use "go install" so the download goes through GOPROXY instead of the GitHub # release API/CDN, which has been returning intermittent/persistent HTTP 504s. -RUN go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.5.0 +RUN go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.13.1 # install kubectl RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/$(go env GOARCH)/kubectl diff --git a/hack/fix_restic_cve.txt b/hack/fix_restic_cve.txt index 12d5d7ab0f..884b8a742e 100644 --- a/hack/fix_restic_cve.txt +++ b/hack/fix_restic_cve.txt @@ -1,5 +1,5 @@ diff --git a/go.mod b/go.mod -index 5f939c481..891dbd4e7 100644 +index 5f939c4..2bd75e5 100644 --- a/go.mod +++ b/go.mod @@ -1,15 +1,15 @@ @@ -33,13 +33,13 @@ index 5f939c481..891dbd4e7 100644 - golang.org/x/term v0.4.0 - golang.org/x/text v0.6.0 - google.golang.org/api v0.106.0 -+ golang.org/x/crypto v0.52.0 -+ golang.org/x/net v0.55.0 ++ golang.org/x/crypto v0.53.0 ++ golang.org/x/net v0.56.0 + golang.org/x/oauth2 v0.36.0 -+ golang.org/x/sync v0.20.0 -+ golang.org/x/sys v0.45.0 -+ golang.org/x/term v0.43.0 -+ golang.org/x/text v0.37.0 ++ golang.org/x/sync v0.21.0 ++ golang.org/x/sys v0.46.0 ++ golang.org/x/term v0.44.0 ++ golang.org/x/text v0.39.0 + google.golang.org/api v0.283.0 ) @@ -56,7 +56,7 @@ index 5f939c481..891dbd4e7 100644 + cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/monitoring v1.24.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect -+ github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect ++ github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect @@ -99,7 +99,7 @@ index 5f939c481..891dbd4e7 100644 - google.golang.org/protobuf v1.28.1 // indirect + github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect -+ go.opentelemetry.io/contrib/detectors/gcp v1.42.0 // indirect ++ go.opentelemetry.io/contrib/detectors/gcp v1.43.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect @@ -109,9 +109,9 @@ index 5f939c481..891dbd4e7 100644 + go.opentelemetry.io/otel/trace v1.43.0 // indirect + golang.org/x/time v0.15.0 // indirect + google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect -+ google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7 // indirect ++ google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260523011958-0a33c5d7ca68 // indirect -+ google.golang.org/grpc v1.81.1 // indirect ++ google.golang.org/grpc v1.82.1 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -120,7 +120,7 @@ index 5f939c481..891dbd4e7 100644 -go 1.18 +go 1.25.8 diff --git a/go.sum b/go.sum -index 026e1d2fa..94d984253 100644 +index 026e1d2..738e46e 100644 --- a/go.sum +++ b/go.sum @@ -1,42 +1,61 @@ @@ -169,8 +169,8 @@ index 026e1d2fa..94d984253 100644 github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= -+github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ= -+github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= ++github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 h1:rIkQfkCOVKc1OiRCNcSDD8ml5RJlZbH/Xsq7lbpynwc= ++github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0/go.mod h1:RD2SsorTmYhF6HkTmDw7KmPYQk8OBYwTkuasChwv7R4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0 h1:4LP6hvB4I5ouTbGgWtixJhgED6xdf67twf9PoY96Tbg= @@ -358,8 +358,8 @@ index 026e1d2fa..94d984253 100644 -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -+go.opentelemetry.io/contrib/detectors/gcp v1.42.0 h1:kpt2PEJuOuqYkPcktfJqWWDjTEd/FNgrxcniL7kQrXQ= -+go.opentelemetry.io/contrib/detectors/gcp v1.42.0/go.mod h1:W9zQ439utxymRrXsUOzZbFX4JhLxXU4+ZnCt8GG7yA8= ++go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU= ++go.opentelemetry.io/contrib/detectors/gcp v1.43.0/go.mod h1:RyaZMFY7yi1kAs45S6mbFGz8O8rqB0dTY14uzvG4LCs= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= @@ -386,8 +386,8 @@ index 026e1d2fa..94d984253 100644 -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -+golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= -+golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= ++golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= ++golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -406,8 +406,8 @@ index 026e1d2fa..94d984253 100644 -golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -+golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -+golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= ++golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= ++golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -415,8 +415,8 @@ index 026e1d2fa..94d984253 100644 -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= ++golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= ++golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -431,21 +431,21 @@ index 026e1d2fa..94d984253 100644 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -+golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -+golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= ++golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= ++golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= -+golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -+golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= ++golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= ++golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -+golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -+golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= ++golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= ++golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -498,12 +498,12 @@ index 026e1d2fa..94d984253 100644 +google.golang.org/api v0.283.0/go.mod h1:6Wssta4c5n9qHq5CBhmlai5h/PUa1djdDAIhYEHyvcM= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= -+google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7 h1:41r6JMbpzBMen0R/4TZeeAmGXSJC7DftGINUodzTkPI= -+google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y= ++google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= ++google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260523011958-0a33c5d7ca68 h1:PvEgGJf9C/1u5CHkInMg7UFYYUoiaQmW2LbtH0pjB78= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260523011958-0a33c5d7ca68/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -+google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= -+google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= ++google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= ++google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/credentials/file_store.go b/internal/credentials/file_store.go index d1f1fb10a5..e21418b564 100644 --- a/internal/credentials/file_store.go +++ b/internal/credentials/file_store.go @@ -21,7 +21,7 @@ import ( "os" "path/filepath" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" kbclient "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/internal/credentials/secret_store.go b/internal/credentials/secret_store.go index f4d2111a57..c03dfe73b9 100644 --- a/internal/credentials/secret_store.go +++ b/internal/credentials/secret_store.go @@ -17,7 +17,7 @@ limitations under the License. package credentials import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" kbclient "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/internal/delete/actions/csi/volumesnapshotcontent_action.go b/internal/delete/actions/csi/volumesnapshotcontent_action.go index 456127471c..67ebbfa6ef 100644 --- a/internal/delete/actions/csi/volumesnapshotcontent_action.go +++ b/internal/delete/actions/csi/volumesnapshotcontent_action.go @@ -20,9 +20,9 @@ import ( "context" "time" + "github.com/cockroachdb/errors" "github.com/google/uuid" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/internal/delete/actions/csi/volumesnapshotcontent_action_test.go b/internal/delete/actions/csi/volumesnapshotcontent_action_test.go index 6cf32adec6..1046341d59 100644 --- a/internal/delete/actions/csi/volumesnapshotcontent_action_test.go +++ b/internal/delete/actions/csi/volumesnapshotcontent_action_test.go @@ -22,8 +22,8 @@ import ( "testing" "time" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/internal/delete/delete_item_action_handler.go b/internal/delete/delete_item_action_handler.go index ba242c0ca0..4837d02432 100644 --- a/internal/delete/delete_item_action_handler.go +++ b/internal/delete/delete_item_action_handler.go @@ -21,7 +21,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/plugin/framework" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/internal/hook/item_hook_handler.go b/internal/hook/item_hook_handler.go index 52dd815aba..cd0d339820 100644 --- a/internal/hook/item_hook_handler.go +++ b/internal/hook/item_hook_handler.go @@ -23,8 +23,8 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" "github.com/google/uuid" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -419,7 +419,7 @@ func getInitContainerFromAnnotation(podName string, annotations map[string]strin return nil } if command == "" { - log.Infof("RestoreHook init container for pod %s is using container's default entrypoint", podName, containerImage) + log.Infof("RestoreHook init container for pod %s is using the default entrypoint of image %s", podName, containerImage) } if containerName == "" { uid, err := uuid.NewRandom() diff --git a/internal/hook/item_hook_handler_test.go b/internal/hook/item_hook_handler_test.go index 37f1500a39..1f2df94696 100644 --- a/internal/hook/item_hook_handler_test.go +++ b/internal/hook/item_hook_handler_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" diff --git a/internal/hook/wait_exec_hook_handler_test.go b/internal/hook/wait_exec_hook_handler_test.go index fb102b16f5..bb0a7c8b17 100644 --- a/internal/hook/wait_exec_hook_handler_test.go +++ b/internal/hook/wait_exec_hook_handler_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" diff --git a/internal/resourcemodifiers/resource_modifiers.go b/internal/resourcemodifiers/resource_modifiers.go index cc780df032..e045108048 100644 --- a/internal/resourcemodifiers/resource_modifiers.go +++ b/internal/resourcemodifiers/resource_modifiers.go @@ -19,9 +19,9 @@ import ( "fmt" "regexp" + "github.com/cockroachdb/errors" jsonpatch "github.com/evanphx/json-patch/v5" "github.com/gobwas/glob" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/internal/resourcepolicies/resource_policies.go b/internal/resourcepolicies/resource_policies.go index 6b5046e57b..9b0024d93a 100644 --- a/internal/resourcepolicies/resource_policies.go +++ b/internal/resourcepolicies/resource_policies.go @@ -21,14 +21,17 @@ import ( "fmt" "strings" - "k8s.io/apimachinery/pkg/util/sets" - - "github.com/pkg/errors" + "github.com/cockroachdb/errors" + "github.com/gobwas/glob" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" crclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/util/wildcard" ) type VolumeActionType string @@ -54,6 +57,84 @@ type Action struct { Parameters map[string]any `yaml:"parameters,omitempty"` } +// PolicyLabelSelector mirrors metav1.LabelSelector with yaml tags for ConfigMap decode. +// metav1.LabelSelector only has json tags, which do not populate under go.yaml.in/yaml/v3. +type PolicyLabelSelector struct { + MatchLabels map[string]string `yaml:"matchLabels,omitempty"` + MatchExpressions []PolicyLabelSelectorRequirement `yaml:"matchExpressions,omitempty"` +} + +// PolicyLabelSelectorRequirement mirrors metav1.LabelSelectorRequirement with yaml tags. +type PolicyLabelSelectorRequirement struct { + Key string `yaml:"key"` + Operator string `yaml:"operator"` + Values []string `yaml:"values,omitempty"` +} + +// IsPresentLabelSelector reports whether s defines any label constraints. +// Empty {} (nil MatchLabels and empty MatchExpressions) is treated as absent. +func IsPresentLabelSelector(s *PolicyLabelSelector) bool { + return s != nil && (len(s.MatchLabels) > 0 || len(s.MatchExpressions) > 0) +} + +// ToMetaV1LabelSelector converts the YAML mirror type to metav1.LabelSelector. +// Conversion itself is infallible; call LabelSelectorAsSelector (or +// SelectorFromPolicyLabelSelector) to validate operators and values. +func ToMetaV1LabelSelector(s *PolicyLabelSelector) *metav1.LabelSelector { + if s == nil { + return nil + } + ls := &metav1.LabelSelector{MatchLabels: s.MatchLabels} + for _, expr := range s.MatchExpressions { + ls.MatchExpressions = append(ls.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: expr.Key, + Operator: metav1.LabelSelectorOperator(expr.Operator), + Values: expr.Values, + }) + } + return ls +} + +// SelectorFromPolicyLabelSelector converts a present policy label selector to a +// runtime labels.Selector. Returns (nil, nil) when s defines no constraints. +func SelectorFromPolicyLabelSelector(s *PolicyLabelSelector) (labels.Selector, error) { + if !IsPresentLabelSelector(s) { + return nil, nil + } + return metav1.LabelSelectorAsSelector(ToMetaV1LabelSelector(s)) +} + +// validatePolicyLabelSelector converts and validates a policy label selector. +func validatePolicyLabelSelector(s *PolicyLabelSelector) error { + _, err := SelectorFromPolicyLabelSelector(s) + return err +} + +// ResourceFilter defines a filter for specific resource kinds. +type ResourceFilter struct { + Kinds []string `yaml:"kinds"` + LabelSelector *PolicyLabelSelector `yaml:"labelSelector,omitempty"` + OrLabelSelectors []*PolicyLabelSelector `yaml:"orLabelSelectors,omitempty"` + Names []string `yaml:"names,omitempty"` + ExcludedNames []string `yaml:"excludedNames,omitempty"` +} + +// IsCatchAll returns true if the filter is a catch-all entry (empty kinds or ["*"]) +func (rf *ResourceFilter) IsCatchAll() bool { + return len(rf.Kinds) == 0 || (len(rf.Kinds) == 1 && rf.Kinds[0] == "*") +} + +// ClusterScopedFilterPolicy defines backup filters scoped globally to cluster-scoped resources. +type ClusterScopedFilterPolicy struct { + ResourceFilters []ResourceFilter `yaml:"resourceFilters"` +} + +// NamespacedFilterPolicy defines backup filters scoped to specific namespaces. +type NamespacedFilterPolicy struct { + Namespaces []string `yaml:"namespaces"` + ResourceFilters []ResourceFilter `yaml:"resourceFilters"` +} + // IncludeExcludePolicy defined policy to include or exclude resources based on the names type IncludeExcludePolicy struct { // The following fields have the same semantics as those from the spec of backup. @@ -95,17 +176,21 @@ type VolumePolicy struct { // ResourcePolicies currently defined slice of volume policies to handle backup type ResourcePolicies struct { - Version string `yaml:"version"` - VolumePolicies []VolumePolicy `yaml:"volumePolicies"` - IncludeExcludePolicy *IncludeExcludePolicy `yaml:"includeExcludePolicy"` + Version string `yaml:"version"` + VolumePolicies []VolumePolicy `yaml:"volumePolicies"` + IncludeExcludePolicy *IncludeExcludePolicy `yaml:"includeExcludePolicy"` + ClusterScopedFilterPolicy *ClusterScopedFilterPolicy `yaml:"clusterScopedFilterPolicy,omitempty"` + NamespacedFilterPolicies []NamespacedFilterPolicy `yaml:"namespacedFilterPolicies,omitempty"` // we may support other resource policies in the future, and they could be added separately // OtherResourcePolicies []OtherResourcePolicy } type Policies struct { - version string - volumePolicies []volPolicy - includeExcludePolicy *IncludeExcludePolicy + version string + volumePolicies []volPolicy + includeExcludePolicy *IncludeExcludePolicy + clusterScopedFilterPolicy *ClusterScopedFilterPolicy + namespacedFilterPolicies []NamespacedFilterPolicy // OtherPolicies } @@ -124,10 +209,35 @@ func unmarshalResourcePolicies(yamlData *string) (*ResourcePolicies, error) { return nil, fmt.Errorf("pvcLabels must be a map of string to string, got %T", raw) } } + if raw, ok := vp.Conditions["pvcVolumeMode"]; ok { + if _, ok := raw.(string); !ok { + return nil, fmt.Errorf("pvcVolumeMode must be a string, got %T", raw) + } + } + if raw, ok := vp.Conditions["pvcAccessModes"]; ok { + if err := validateStringSliceCondition("pvcAccessModes", raw); err != nil { + return nil, err + } + } } return resPolicies, nil } +func validateStringSliceCondition(name string, raw any) error { + switch values := raw.(type) { + case []any: + for _, value := range values { + if _, ok := value.(string); !ok { + return fmt.Errorf("%s must be a list of strings, got element %T", name, value) + } + } + case []string: + default: + return fmt.Errorf("%s must be a list of strings, got %T", name, raw) + } + return nil +} + func (p *Policies) BuildPolicy(resPolicies *ResourcePolicies) error { for _, vp := range resPolicies.VolumePolicies { con, err := unmarshalVolConditions(vp.Conditions) @@ -151,6 +261,12 @@ func (p *Policies) BuildPolicy(resPolicies *ResourcePolicies) error { if len(con.PVCPhase) > 0 { volP.conditions = append(volP.conditions, &pvcPhaseCondition{phases: con.PVCPhase}) } + if con.PVCVolumeMode != "" { + volP.conditions = append(volP.conditions, &pvcVolumeModeCondition{volumeMode: con.PVCVolumeMode}) + } + if len(con.PVCAccessModes) > 0 { + volP.conditions = append(volP.conditions, &pvcAccessModesCondition{accessModes: con.PVCAccessModes}) + } p.volumePolicies = append(p.volumePolicies, volP) } @@ -158,6 +274,8 @@ func (p *Policies) BuildPolicy(resPolicies *ResourcePolicies) error { p.version = resPolicies.Version p.includeExcludePolicy = resPolicies.IncludeExcludePolicy + p.clusterScopedFilterPolicy = resPolicies.ClusterScopedFilterPolicy + p.namespacedFilterPolicies = resPolicies.NamespacedFilterPolicies return nil } @@ -228,6 +346,38 @@ func (p *Policies) Validate() error { } } + if err := p.validateClusterScopedFilterPolicy(); err != nil { + return errors.WithStack(err) + } + + if err := p.validateNamespacedFilterPolicies(); err != nil { + return errors.WithStack(err) + } + + return nil +} + +func (p *Policies) ValidateForRestore() error { + if p.version != currentSupportDataVersion { + return fmt.Errorf("incompatible version number %s with supported version %s", p.version, currentSupportDataVersion) + } + + if len(p.volumePolicies) > 0 { + return fmt.Errorf("volumePolicies are not supported for restore") + } + + if p.GetIncludeExcludePolicy() != nil { + return fmt.Errorf("includeExcludePolicy is not supported for restore") + } + + if err := p.validateClusterScopedFilterPolicy(); err != nil { + return errors.WithStack(err) + } + + if err := p.validateNamespacedFilterPolicies(); err != nil { + return errors.WithStack(err) + } + return nil } @@ -235,6 +385,14 @@ func (p *Policies) GetIncludeExcludePolicy() *IncludeExcludePolicy { return p.includeExcludePolicy } +func (p *Policies) GetClusterScopedFilterPolicy() *ClusterScopedFilterPolicy { + return p.clusterScopedFilterPolicy +} + +func (p *Policies) GetNamespacedFilterPolicies() []NamespacedFilterPolicy { + return p.namespacedFilterPolicies +} + func GetResourcePoliciesFromBackup( backup velerov1api.Backup, client crclient.Client, @@ -251,26 +409,143 @@ func GetResourcePoliciesFromBackup( if err != nil { logger.Errorf("Fail to get ResourcePolicies %s ConfigMap with error %s.", backup.Namespace+"/"+backup.Spec.ResourcePolicy.Name, err.Error()) - return nil, fmt.Errorf("fail to get ResourcePolicies %s ConfigMap with error %s", - backup.Namespace+"/"+backup.Spec.ResourcePolicy.Name, err.Error()) + return nil, fmt.Errorf("fail to get ResourcePolicies %s ConfigMap: %w", + backup.Namespace+"/"+backup.Spec.ResourcePolicy.Name, err) } resourcePolicies, err = getResourcePoliciesFromConfig(policiesConfigMap) if err != nil { logger.Errorf("Fail to read ResourcePolicies from ConfigMap %s with error %s.", backup.Namespace+"/"+backup.Name, err.Error()) - return nil, fmt.Errorf("fail to read the ResourcePolicies from ConfigMap %s with error %s", - backup.Namespace+"/"+backup.Name, err.Error()) + return nil, fmt.Errorf("fail to read the ResourcePolicies from ConfigMap %s: %w", + backup.Namespace+"/"+backup.Name, err) } else if err = resourcePolicies.Validate(); err != nil { logger.Errorf("Fail to validate ResourcePolicies in ConfigMap %s with error %s.", backup.Namespace+"/"+backup.Name, err.Error()) - return nil, fmt.Errorf("fail to validate ResourcePolicies in ConfigMap %s with error %s", - backup.Namespace+"/"+backup.Name, err.Error()) + return nil, fmt.Errorf("fail to validate ResourcePolicies in ConfigMap %s: %w", + backup.Namespace+"/"+backup.Name, err) } } return resourcePolicies, nil } +// GetGlobalResourcePolicies loads and validates the cluster-wide global backup volume +// policies from a ConfigMap in the Velero install namespace. Only the volumePolicies +// section is honored globally; any include/exclude or fine-grained filter policies are +// ignored (a warning is logged), as those are tied to a specific backup use case. +func GetGlobalResourcePolicies( + client crclient.Client, + namespace string, + configMapName string, + logger logrus.FieldLogger, +) (*Policies, error) { + cm := &corev1api.ConfigMap{} + if err := client.Get(context.Background(), crclient.ObjectKey{Namespace: namespace, Name: configMapName}, cm); err != nil { + return nil, fmt.Errorf("fail to get global backup volume policies ConfigMap %s/%s: %w", namespace, configMapName, err) + } + + policies, err := getResourcePoliciesFromConfig(cm) + if err != nil { + return nil, fmt.Errorf("fail to read global backup volume policies from ConfigMap %s/%s: %w", namespace, configMapName, err) + } + if err := policies.Validate(); err != nil { + return nil, fmt.Errorf("fail to validate global backup volume policies in ConfigMap %s/%s: %w", namespace, configMapName, err) + } + + // Only volumePolicies apply globally; warn about any other filter policies that will be ignored. + if policies.includeExcludePolicy != nil || + policies.clusterScopedFilterPolicy != nil || + len(policies.namespacedFilterPolicies) > 0 { + logger.Warnf("Global backup volume policies ConfigMap %s/%s contains include/exclude or fine-grained "+ + "filter policies; these are ignored, only volumePolicies apply globally.", namespace, configMapName) + } + + // Return a fresh Policies carrying only the globally-applicable fields. Using an allowlist here + // (rather than nil-ing out the ignored fields) means any filter field added to Policies in the + // future is excluded from the global policies by default, without needing to update this code. + return &Policies{ + version: policies.version, + volumePolicies: policies.volumePolicies, + }, nil +} + +// GetResourcePoliciesFromBackupWithGlobal builds the effective resource policies for a backup +// by merging the backup-referenced resource policies with the global backup volume policies +// (when globalConfigMapName is set). The merged volumePolicies list is the backup-level +// policies followed by the global ones, so the first match wins and a backup can override the +// global baseline for a specific volume while still inheriting the rest of the global rules. +func GetResourcePoliciesFromBackupWithGlobal( + backup velerov1api.Backup, + client crclient.Client, + globalConfigMapName string, + installNamespace string, + logger logrus.FieldLogger, +) (*Policies, error) { + backupPolicies, err := GetResourcePoliciesFromBackup(backup, client, logger) + if err != nil { + return nil, err + } + + if globalConfigMapName == "" { + return backupPolicies, nil + } + + globalPolicies, err := GetGlobalResourcePolicies(client, installNamespace, globalConfigMapName, logger) + if err != nil { + return nil, err + } + + if backupPolicies == nil { + return globalPolicies, nil + } + // Backup-level policies first, then global, so backups can override the global baseline. + backupPolicies.volumePolicies = append(backupPolicies.volumePolicies, globalPolicies.volumePolicies...) + return backupPolicies, nil +} + +// GetResourcePoliciesFromRestore retrieves the resource policies from the ConfigMap referenced in the Restore spec. +func GetResourcePoliciesFromRestore( + ctx context.Context, + restore *velerov1api.Restore, + client crclient.Client, + logger logrus.FieldLogger, +) (resourcePolicies *Policies, err error) { + if restore.Spec.ResourcePolicy != nil { + if !strings.EqualFold(restore.Spec.ResourcePolicy.Kind, ConfigmapRefType) { + return nil, fmt.Errorf("invalid ResourcePolicy kind %q, only %q is supported", + restore.Spec.ResourcePolicy.Kind, ConfigmapRefType) + } + policiesConfigMap := &corev1api.ConfigMap{} + err = client.Get( + ctx, + crclient.ObjectKey{ + Namespace: restore.Namespace, + Name: restore.Spec.ResourcePolicy.Name, + }, + policiesConfigMap, + ) + if err != nil { + logger.Errorf("Fail to get ResourcePolicies %s ConfigMap with error %s.", + restore.Namespace+"/"+restore.Spec.ResourcePolicy.Name, err.Error()) + return nil, fmt.Errorf("fail to get ResourcePolicies %s ConfigMap: %w", + restore.Namespace+"/"+restore.Spec.ResourcePolicy.Name, err) + } + resourcePolicies, err = getResourcePoliciesFromConfig(policiesConfigMap) + if err != nil { + logger.Errorf("Fail to read ResourcePolicies from ConfigMap %s with error %s.", + restore.Namespace+"/"+restore.Spec.ResourcePolicy.Name, err.Error()) + return nil, fmt.Errorf("fail to read the ResourcePolicies from ConfigMap %s: %w", + restore.Namespace+"/"+restore.Spec.ResourcePolicy.Name, err) + } else if err = resourcePolicies.ValidateForRestore(); err != nil { + logger.Errorf("Fail to validate ResourcePolicies in ConfigMap %s with error %s.", + restore.Namespace+"/"+restore.Spec.ResourcePolicy.Name, err.Error()) + return nil, fmt.Errorf("fail to validate ResourcePolicies in ConfigMap %s: %w", + restore.Namespace+"/"+restore.Spec.ResourcePolicy.Name, err) + } + } + return resourcePolicies, nil +} + func getResourcePoliciesFromConfig(cm *corev1api.ConfigMap) (*Policies, error) { if cm == nil { return nil, fmt.Errorf("could not parse config from nil configmap") @@ -296,3 +571,133 @@ func getResourcePoliciesFromConfig(cm *corev1api.ConfigMap) (*Policies, error) { return policies, nil } + +func (p *Policies) validateNamespacedFilterPolicies() error { + seenPatterns := make(map[string][]int) // pattern -> list of policy indices + + // Rule 1-7: Basic validation rules + for i, nfp := range p.namespacedFilterPolicies { + if len(nfp.Namespaces) == 0 { + return fmt.Errorf("namespacedFilterPolicies[%d]: at least one namespace must be specified", i) + } + if len(nfp.ResourceFilters) == 0 { + return fmt.Errorf("namespacedFilterPolicies[%d]: at least one resourceFilter must be specified", i) + } + + // Rule 8 & 9: Validate glob patterns and collect namespace patterns for duplicate check + for j, pattern := range nfp.Namespaces { + if err := wildcard.ValidateNamespaceName(pattern); err != nil { + return fmt.Errorf("namespacedFilterPolicies[%d].namespaces[%d]: %w", i, j, err) + } + seenPatterns[pattern] = append(seenPatterns[pattern], i) + } + + seenKinds := make(map[string]int) + hasCatchAll := false + for j, rf := range nfp.ResourceFilters { + if rf.IsCatchAll() { + if hasCatchAll { + return fmt.Errorf("namespacedFilterPolicies[%d]: only one catch-all resource filter is allowed", i) + } + hasCatchAll = true + if len(rf.Names) > 0 || len(rf.ExcludedNames) > 0 { + return fmt.Errorf("namespacedFilterPolicies[%d].resourceFilters[%d]: names or excludedNames cannot be specified for catch-all filters", i, j) + } + } + + for _, kind := range rf.Kinds { + if kind == "*" { + continue // "*" is handled by IsCatchAll, no need to check duplicates against other kinds + } + if prevJ, ok := seenKinds[kind]; ok { + return fmt.Errorf("namespacedFilterPolicies[%d]: kind %q appears in both resourceFilters[%d] and resourceFilters[%d]", i, kind, prevJ, j) + } + seenKinds[kind] = j + } + + if IsPresentLabelSelector(rf.LabelSelector) && len(rf.OrLabelSelectors) > 0 { + return fmt.Errorf("namespacedFilterPolicies[%d].resourceFilters[%d]: labelSelector and orLabelSelectors cannot co-exist", i, j) + } + if err := validatePolicyLabelSelector(rf.LabelSelector); err != nil { + return fmt.Errorf("namespacedFilterPolicies[%d].resourceFilters[%d]: invalid label selector: %w", i, j, err) + } + for k, ols := range rf.OrLabelSelectors { + if err := validatePolicyLabelSelector(ols); err != nil { + return fmt.Errorf("namespacedFilterPolicies[%d].resourceFilters[%d].orLabelSelectors[%d]: invalid label selector: %w", i, j, k, err) + } + } + + // Validate glob patterns for names and excludedNames using gobwas/glob + for k, pattern := range rf.Names { + if _, err := glob.Compile(pattern); err != nil { + return fmt.Errorf("namespacedFilterPolicies[%d].resourceFilters[%d].names[%d]: invalid glob pattern %q: %v", i, j, k, pattern, err) + } + } + for k, pattern := range rf.ExcludedNames { + if _, err := glob.Compile(pattern); err != nil { + return fmt.Errorf("namespacedFilterPolicies[%d].resourceFilters[%d].excludedNames[%d]: invalid glob pattern %q: %v", i, j, k, pattern, err) + } + } + } + } + + // Rule 8: Report exact duplicates only + for pattern, policyIndices := range seenPatterns { + if len(policyIndices) > 1 { + return fmt.Errorf( + "namespacedFilterPolicies: duplicate namespace pattern '%s' found in policies %v", + pattern, policyIndices) + } + } + + return nil +} + +func (p *Policies) validateClusterScopedFilterPolicy() error { + if p.clusterScopedFilterPolicy == nil { + return nil + } + + if len(p.clusterScopedFilterPolicy.ResourceFilters) == 0 { + return fmt.Errorf("clusterScopedFilterPolicy: resourceFilters cannot be empty; remove the policy block entirely if it is not needed") + } + + seenKinds := make(map[string]int) + for j, rf := range p.clusterScopedFilterPolicy.ResourceFilters { + if rf.IsCatchAll() { + return fmt.Errorf("clusterScopedFilterPolicy.resourceFilters[%d]: kinds must be specified (catch-all is not supported)", j) + } + + for _, kind := range rf.Kinds { + if prevJ, ok := seenKinds[kind]; ok { + return fmt.Errorf("clusterScopedFilterPolicy: kind %q appears in both resourceFilters[%d] and resourceFilters[%d]", kind, prevJ, j) + } + seenKinds[kind] = j + } + + if IsPresentLabelSelector(rf.LabelSelector) && len(rf.OrLabelSelectors) > 0 { + return fmt.Errorf("clusterScopedFilterPolicy.resourceFilters[%d]: labelSelector and orLabelSelectors cannot co-exist", j) + } + if err := validatePolicyLabelSelector(rf.LabelSelector); err != nil { + return fmt.Errorf("clusterScopedFilterPolicy.resourceFilters[%d]: invalid label selector: %w", j, err) + } + for k, ols := range rf.OrLabelSelectors { + if err := validatePolicyLabelSelector(ols); err != nil { + return fmt.Errorf("clusterScopedFilterPolicy.resourceFilters[%d].orLabelSelectors[%d]: invalid label selector: %w", j, k, err) + } + } + + for k, pattern := range rf.Names { + if _, err := glob.Compile(pattern); err != nil { + return fmt.Errorf("clusterScopedFilterPolicy.resourceFilters[%d].names[%d]: invalid glob pattern %q: %v", j, k, pattern, err) + } + } + for k, pattern := range rf.ExcludedNames { + if _, err := glob.Compile(pattern); err != nil { + return fmt.Errorf("clusterScopedFilterPolicy.resourceFilters[%d].excludedNames[%d]: invalid glob pattern %q: %v", j, k, pattern, err) + } + } + } + + return nil +} diff --git a/internal/resourcepolicies/resource_policies_test.go b/internal/resourcepolicies/resource_policies_test.go index 06b1bea1cf..52c3bd6103 100644 --- a/internal/resourcepolicies/resource_policies_test.go +++ b/internal/resourcepolicies/resource_policies_test.go @@ -16,15 +16,27 @@ limitations under the License. package resourcepolicies import ( + "context" "testing" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerotest "github.com/vmware-tanzu/velero/pkg/test" ) +func pvcVolumeMode(mode corev1api.PersistentVolumeMode) *corev1api.PersistentVolumeMode { + return &mode +} + func TestLoadResourcePolicies(t *testing.T) { testCases := []struct { name string @@ -158,6 +170,64 @@ volumePolicies: `, wantErr: false, }, + { + name: "supported format pvcVolumeMode", + yamlData: `version: v1 +volumePolicies: + - conditions: + pvcVolumeMode: Block + action: + type: skip +`, + wantErr: false, + }, + { + name: "error format of pvcVolumeMode (not a string)", + yamlData: `version: v1 +volumePolicies: + - conditions: + pvcVolumeMode: + - Block + action: + type: skip +`, + wantErr: true, + }, + { + name: "supported format pvcAccessModes", + yamlData: `version: v1 +volumePolicies: + - conditions: + pvcAccessModes: + - ReadWriteOnce + action: + type: skip +`, + wantErr: false, + }, + { + name: "error format of pvcAccessModes (not a list)", + yamlData: `version: v1 +volumePolicies: + - conditions: + pvcAccessModes: ReadWriteOnce + action: + type: skip +`, + wantErr: true, + }, + { + name: "error format of pvcAccessModes (list with non-string)", + yamlData: `version: v1 +volumePolicies: + - conditions: + pvcAccessModes: + - 123 + action: + type: skip +`, + wantErr: true, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -353,14 +423,20 @@ func TestGetResourceMatchedAction(t *testing.T) { } func TestGetResourcePoliciesFromConfig(t *testing.T) { - // Create a test ConfigMap - cm := &corev1api.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-configmap", - Namespace: "test-namespace", - }, - Data: map[string]string{ - "test-data": `version: v1 + testCases := []struct { + name string + cm *corev1api.ConfigMap + expectedErr string + }{ + { + name: "valid configmap", + cm: &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "test-data": `version: v1 volumePolicies: - conditions: capacity: '0,10Gi' @@ -381,306 +457,700 @@ volumePolicies: action: type: skip `, - }, - } - - // Call the function and check for errors - resPolicies, err := getResourcePoliciesFromConfig(cm) - require.NoError(t, err) - - // Check that the returned resourcePolicies object contains the expected data - assert.Equal(t, "v1", resPolicies.version) - - assert.Len(t, resPolicies.volumePolicies, 3) - - policies := ResourcePolicies{ - Version: "v1", - VolumePolicies: []VolumePolicy{ - { - Conditions: map[string]any{ - "capacity": "0,10Gi", - "csi": map[string]any{ - "driver": "disks.csi.driver", - }, - }, - Action: Action{ - Type: Skip, - }, - }, - { - Conditions: map[string]any{ - "csi": map[string]any{ - "driver": "files.csi.driver", - "volumeAttributes": map[string]string{"protocol": "nfs"}, - }, - }, - Action: Action{ - Type: Skip, - }, - }, - { - Conditions: map[string]any{ - "pvcLabels": map[string]string{ - "environment": "production", - }, - }, - Action: Action{ - Type: Skip, }, }, + expectedErr: "", }, - } - - p := &Policies{} - err = p.BuildPolicy(&policies) - if err != nil { - t.Fatalf("failed to build policy: %v", err) - } - - assert.Equal(t, p, resPolicies) -} - -func TestGetMatchAction(t *testing.T) { - testCases := []struct { - name string - yamlData string - vol *corev1api.PersistentVolume - podVol *corev1api.Volume - pvc *corev1api.PersistentVolumeClaim - skip bool - }{ { - name: "empty csi", - yamlData: `version: v1 -volumePolicies: -- conditions: - csi: {} - action: - type: skip`, - vol: &corev1api.PersistentVolume{ - Spec: corev1api.PersistentVolumeSpec{ - PersistentVolumeSource: corev1api.PersistentVolumeSource{ - CSI: &corev1api.CSIPersistentVolumeSource{Driver: "ebs.csi.aws.com"}, - }}, - }, - skip: true, + name: "nil configmap", + cm: nil, + expectedErr: "could not parse config from nil configmap", }, { - name: "empty csi with pv no csi driver", - yamlData: `version: v1 -volumePolicies: -- conditions: - csi: {} - action: - type: skip`, - vol: &corev1api.PersistentVolume{ - Spec: corev1api.PersistentVolumeSpec{ - Capacity: corev1api.ResourceList{ - corev1api.ResourceStorage: resource.MustParse("1Gi"), - }}, + name: "empty data configmap", + cm: &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{}, }, - skip: false, + expectedErr: "illegal resource policies test-namespace/test-configmap configmap", }, { - name: "Skip AFS CSI condition with Disk volumes", - yamlData: `version: v1 -volumePolicies: - - conditions: - csi: - driver: files.csi.driver - action: - type: skip`, - vol: &corev1api.PersistentVolume{ - Spec: corev1api.PersistentVolumeSpec{ - PersistentVolumeSource: corev1api.PersistentVolumeSource{ - CSI: &corev1api.CSIPersistentVolumeSource{Driver: "disks.csi.driver"}, - }}, + name: "multiple data configmap", + cm: &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "data1": "value1", + "data2": "value2", + }, }, - skip: false, + expectedErr: "illegal resource policies test-namespace/test-configmap configmap", }, { - name: "Skip AFS CSI condition with AFS volumes", - yamlData: `version: v1 + name: "invalid yaml data", + cm: &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "test-data": `version: v1 volumePolicies: - conditions: + capacity: '0,10Gi' csi: - driver: files.csi.driver + driver: disks.csi.driver action: - type: skip`, - vol: &corev1api.PersistentVolume{ - Spec: corev1api.PersistentVolumeSpec{ - PersistentVolumeSource: corev1api.PersistentVolumeSource{ - CSI: &corev1api.CSIPersistentVolumeSource{Driver: "files.csi.driver"}, - }}, + type: skip + invalid-key: value +`, + }, }, - skip: true, + expectedErr: "failed to decode yaml data into resource policies", }, { - name: "Skip AFS NFS CSI condition with Disk volumes", - yamlData: `version: v1 + name: "build policy error", + cm: &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "test-data": `version: v1 volumePolicies: - conditions: + capacity: 'invalid-capacity' csi: - driver: files.csi.driver - volumeAttributes: - protocol: nfs + driver: disks.csi.driver action: type: skip `, - vol: &corev1api.PersistentVolume{ - Spec: corev1api.PersistentVolumeSpec{ - PersistentVolumeSource: corev1api.PersistentVolumeSource{ - CSI: &corev1api.CSIPersistentVolumeSource{Driver: "disks.csi.driver"}, - }}, + }, }, - skip: false, + expectedErr: "wrong format of Capacity invalid-capacity", }, - { - name: "Skip AFS NFS CSI condition with AFS SMB volumes", - yamlData: `version: v1 + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + resPolicies, err := getResourcePoliciesFromConfig(tc.cm) + if tc.expectedErr == "" { + require.NoError(t, err) + assert.Equal(t, "v1", resPolicies.version) + assert.Len(t, resPolicies.volumePolicies, 3) + } else { + require.ErrorContains(t, err, tc.expectedErr) + assert.Nil(t, resPolicies) + } + }) + } +} + +func TestGetResourcePoliciesFromBackup(t *testing.T) { + validCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "test-data": `version: v1 volumePolicies: - conditions: + capacity: '0,10Gi' csi: - driver: files.csi.driver - volumeAttributes: - protocol: nfs + driver: disks.csi.driver action: type: skip `, - vol: &corev1api.PersistentVolume{ - Spec: corev1api.PersistentVolumeSpec{ - PersistentVolumeSource: corev1api.PersistentVolumeSource{ - CSI: &corev1api.CSIPersistentVolumeSource{Driver: "files.csi.driver", VolumeAttributes: map[string]string{"key1": "val1"}}, - }}, - }, - skip: false, }, - { - name: "Skip AFS NFS CSI condition with AFS NFS volumes", - yamlData: `version: v1 + } + + invalidActionCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid-action-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "test-data": `version: v1 volumePolicies: - conditions: + capacity: '0,10Gi' csi: - driver: files.csi.driver - volumeAttributes: - protocol: nfs + driver: disks.csi.driver action: - type: skip + type: invalid-action `, - vol: &corev1api.PersistentVolume{ - Spec: corev1api.PersistentVolumeSpec{ - PersistentVolumeSource: corev1api.PersistentVolumeSource{ - CSI: &corev1api.CSIPersistentVolumeSource{Driver: "files.csi.driver", VolumeAttributes: map[string]string{"protocol": "nfs"}}, - }}, - }, - skip: true, }, - { - name: "Skip Disk and AFS NFS CSI condition with Disk volumes", - yamlData: `version: v1 + } + + invalidVersionCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid-version-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "test-data": `version: v2 volumePolicies: - conditions: + capacity: '0,10Gi' csi: driver: disks.csi.driver action: type: skip - - conditions: - csi: - driver: files.csi.driver - volumeAttributes: - protocol: nfs - action: - type: skip`, - vol: &corev1api.PersistentVolume{ - Spec: corev1api.PersistentVolumeSpec{ - PersistentVolumeSource: corev1api.PersistentVolumeSource{ - CSI: &corev1api.CSIPersistentVolumeSource{Driver: "disks.csi.driver", VolumeAttributes: map[string]string{"key1": "val1"}}, +`, + }, + } + + emptyCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "empty-configmap", + Namespace: "test-namespace", + }, + } + + client := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(validCM, invalidActionCM, invalidVersionCM, emptyCM).Build() + logger := logrus.New() + + testCases := []struct { + name string + backup velerov1api.Backup + expectedErr string + }{ + { + name: "valid configmap", + backup: velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-backup", + }, + Spec: velerov1api.BackupSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "test-configmap", + }, + }, + }, + expectedErr: "", + }, + { + name: "invalid kind", + backup: velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-backup", + }, + Spec: velerov1api.BackupSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: "Secret", + Name: "test-configmap", + }, + }, + }, + expectedErr: "", + }, + { + name: "configmap not found", + backup: velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-backup", + }, + Spec: velerov1api.BackupSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "non-existent-configmap", + }, + }, + }, + expectedErr: "fail to get ResourcePolicies test-namespace/non-existent-configmap ConfigMap", + }, + { + name: "invalid action configmap", + backup: velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-backup", + }, + Spec: velerov1api.BackupSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "invalid-action-configmap", + }, + }, + }, + expectedErr: "fail to validate ResourcePolicies in ConfigMap test-namespace/test-backup", + }, + { + name: "invalid version configmap", + backup: velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-backup", + }, + Spec: velerov1api.BackupSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "invalid-version-configmap", + }, + }, + }, + expectedErr: "fail to validate ResourcePolicies in ConfigMap test-namespace/test-backup", + }, + { + name: "empty configmap", + backup: velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-backup", + }, + Spec: velerov1api.BackupSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "empty-configmap", + }, + }, + }, + expectedErr: "fail to read the ResourcePolicies from ConfigMap test-namespace/test-backup", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + resPolicies, err := GetResourcePoliciesFromBackup(tc.backup, client, logger) + if tc.expectedErr == "" { + require.NoError(t, err) + if tc.backup.Spec.ResourcePolicy != nil && tc.backup.Spec.ResourcePolicy.Kind == ConfigmapRefType { + assert.NotNil(t, resPolicies) + } else { + assert.Nil(t, resPolicies) + } + } else { + require.ErrorContains(t, err, tc.expectedErr) + assert.Nil(t, resPolicies) + } + }) + } +} + +func TestGetResourcePoliciesFromRestore(t *testing.T) { + validCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "test-data": `version: v1 +namespacedFilterPolicies: + - namespaces: ["default"] + resourceFilters: + - kinds: ["Pod"] +`, + }, + } + + invalidNfpCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid-action-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "test-data": `version: v1 +namespacedFilterPolicies: + - namespaces: [] + resourceFilters: + - kinds: ["Pod"] +`, + }, + } + + invalidVersionCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid-version-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "test-data": `version: v2 +namespacedFilterPolicies: + - namespaces: ["default"] + resourceFilters: + - kinds: ["Pod"] +`, + }, + } + + emptyCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "empty-configmap", + Namespace: "test-namespace", + }, + } + + client := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(validCM, invalidNfpCM, invalidVersionCM, emptyCM).Build() + logger := logrus.New() + + testCases := []struct { + name string + restore *velerov1api.Restore + expectedErr string + }{ + { + name: "valid configmap", + restore: &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-restore", + }, + Spec: velerov1api.RestoreSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "test-configmap", + }, + }, + }, + expectedErr: "", + }, + { + name: "invalid kind", + restore: &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-restore", + }, + Spec: velerov1api.RestoreSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: "Secret", + Name: "test-configmap", + }, + }, + }, + expectedErr: "invalid ResourcePolicy kind \"Secret\", only \"configmap\" is supported", + }, + { + name: "configmap not found", + restore: &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-restore", + }, + Spec: velerov1api.RestoreSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "non-existent-configmap", + }, + }, + }, + expectedErr: "fail to get ResourcePolicies test-namespace/non-existent-configmap ConfigMap", + }, + { + name: "invalid action configmap", + restore: &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-restore", + }, + Spec: velerov1api.RestoreSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "invalid-action-configmap", + }, + }, + }, + expectedErr: "fail to validate ResourcePolicies in ConfigMap test-namespace/invalid-action-configmap", + }, + { + name: "invalid version configmap", + restore: &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-restore", + }, + Spec: velerov1api.RestoreSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "invalid-version-configmap", + }, + }, + }, + expectedErr: "fail to validate ResourcePolicies in ConfigMap test-namespace/invalid-version-configmap", + }, + { + name: "empty configmap", + restore: &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-restore", + }, + Spec: velerov1api.RestoreSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "empty-configmap", + }, + }, + }, + expectedErr: "fail to read the ResourcePolicies from ConfigMap test-namespace/empty-configmap", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + resPolicies, err := GetResourcePoliciesFromRestore(context.Background(), tc.restore, client, logger) + if tc.expectedErr == "" { + require.NoError(t, err) + assert.NotNil(t, resPolicies) + } else { + require.ErrorContains(t, err, tc.expectedErr) + assert.Nil(t, resPolicies) + } + }) + } +} + +func TestGetMatchAction(t *testing.T) { + testCases := []struct { + name string + yamlData string + vol *corev1api.PersistentVolume + podVol *corev1api.Volume + pvc *corev1api.PersistentVolumeClaim + skip bool + }{ + { + name: "empty csi", + yamlData: `version: v1 +volumePolicies: +- conditions: + csi: {} + action: + type: skip`, + vol: &corev1api.PersistentVolume{ + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeSource: corev1api.PersistentVolumeSource{ + CSI: &corev1api.CSIPersistentVolumeSource{Driver: "ebs.csi.aws.com"}, }}, }, skip: true, }, { - name: "Skip Disk and AFS NFS CSI condition with AFS SMB volumes", + name: "empty csi with pv no csi driver", + yamlData: `version: v1 +volumePolicies: +- conditions: + csi: {} + action: + type: skip`, + vol: &corev1api.PersistentVolume{ + Spec: corev1api.PersistentVolumeSpec{ + Capacity: corev1api.ResourceList{ + corev1api.ResourceStorage: resource.MustParse("1Gi"), + }}, + }, + skip: false, + }, + { + name: "Skip AFS CSI condition with Disk volumes", yamlData: `version: v1 volumePolicies: - - conditions: - csi: - driver: disks.csi.driver - action: - type: skip - conditions: csi: driver: files.csi.driver - volumeAttributes: - protocol: nfs action: type: skip`, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ - CSI: &corev1api.CSIPersistentVolumeSource{Driver: "files.csi.driver", VolumeAttributes: map[string]string{"key1": "val1"}}, + CSI: &corev1api.CSIPersistentVolumeSource{Driver: "disks.csi.driver"}, }}, }, skip: false, }, { - name: "Skip Disk and AFS NFS CSI condition with AFS NFS volumes", + name: "Skip AFS CSI condition with AFS volumes", yamlData: `version: v1 volumePolicies: - - conditions: - csi: - driver: disks.csi.driver - action: - type: skip - conditions: csi: driver: files.csi.driver - volumeAttributes: - protocol: nfs action: type: skip`, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ - CSI: &corev1api.CSIPersistentVolumeSource{Driver: "files.csi.driver", VolumeAttributes: map[string]string{"key1": "val1", "protocol": "nfs"}}, + CSI: &corev1api.CSIPersistentVolumeSource{Driver: "files.csi.driver"}, }}, }, skip: true, }, { - name: "csi not configured and testing capacity condition", + name: "Skip AFS NFS CSI condition with Disk volumes", yamlData: `version: v1 volumePolicies: -- conditions: - capacity: "0,100Gi" - action: - type: skip`, + - conditions: + csi: + driver: files.csi.driver + volumeAttributes: + protocol: nfs + action: + type: skip +`, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ - Capacity: corev1api.ResourceList{ - corev1api.ResourceStorage: resource.MustParse("1Gi"), - }, PersistentVolumeSource: corev1api.PersistentVolumeSource{ - CSI: &corev1api.CSIPersistentVolumeSource{Driver: "ebs.csi.aws.com"}, + CSI: &corev1api.CSIPersistentVolumeSource{Driver: "disks.csi.driver"}, }}, }, - skip: true, + skip: false, }, { - name: "empty nfs", + name: "Skip AFS NFS CSI condition with AFS SMB volumes", yamlData: `version: v1 volumePolicies: -- conditions: - nfs: {} - action: - type: skip`, + - conditions: + csi: + driver: files.csi.driver + volumeAttributes: + protocol: nfs + action: + type: skip +`, vol: &corev1api.PersistentVolume{ Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeSource: corev1api.PersistentVolumeSource{ - NFS: &corev1api.NFSVolumeSource{Server: "192.168.1.20"}, + CSI: &corev1api.CSIPersistentVolumeSource{Driver: "files.csi.driver", VolumeAttributes: map[string]string{"key1": "val1"}}, }}, }, - skip: true, + skip: false, + }, + { + name: "Skip AFS NFS CSI condition with AFS NFS volumes", + yamlData: `version: v1 +volumePolicies: + - conditions: + csi: + driver: files.csi.driver + volumeAttributes: + protocol: nfs + action: + type: skip +`, + vol: &corev1api.PersistentVolume{ + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeSource: corev1api.PersistentVolumeSource{ + CSI: &corev1api.CSIPersistentVolumeSource{Driver: "files.csi.driver", VolumeAttributes: map[string]string{"protocol": "nfs"}}, + }}, + }, + skip: true, + }, + { + name: "Skip Disk and AFS NFS CSI condition with Disk volumes", + yamlData: `version: v1 +volumePolicies: + - conditions: + csi: + driver: disks.csi.driver + action: + type: skip + - conditions: + csi: + driver: files.csi.driver + volumeAttributes: + protocol: nfs + action: + type: skip`, + vol: &corev1api.PersistentVolume{ + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeSource: corev1api.PersistentVolumeSource{ + CSI: &corev1api.CSIPersistentVolumeSource{Driver: "disks.csi.driver", VolumeAttributes: map[string]string{"key1": "val1"}}, + }}, + }, + skip: true, + }, + { + name: "Skip Disk and AFS NFS CSI condition with AFS SMB volumes", + yamlData: `version: v1 +volumePolicies: + - conditions: + csi: + driver: disks.csi.driver + action: + type: skip + - conditions: + csi: + driver: files.csi.driver + volumeAttributes: + protocol: nfs + action: + type: skip`, + vol: &corev1api.PersistentVolume{ + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeSource: corev1api.PersistentVolumeSource{ + CSI: &corev1api.CSIPersistentVolumeSource{Driver: "files.csi.driver", VolumeAttributes: map[string]string{"key1": "val1"}}, + }}, + }, + skip: false, + }, + { + name: "Skip Disk and AFS NFS CSI condition with AFS NFS volumes", + yamlData: `version: v1 +volumePolicies: + - conditions: + csi: + driver: disks.csi.driver + action: + type: skip + - conditions: + csi: + driver: files.csi.driver + volumeAttributes: + protocol: nfs + action: + type: skip`, + vol: &corev1api.PersistentVolume{ + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeSource: corev1api.PersistentVolumeSource{ + CSI: &corev1api.CSIPersistentVolumeSource{Driver: "files.csi.driver", VolumeAttributes: map[string]string{"key1": "val1", "protocol": "nfs"}}, + }}, + }, + skip: true, + }, + { + name: "csi not configured and testing capacity condition", + yamlData: `version: v1 +volumePolicies: +- conditions: + capacity: "0,100Gi" + action: + type: skip`, + vol: &corev1api.PersistentVolume{ + Spec: corev1api.PersistentVolumeSpec{ + Capacity: corev1api.ResourceList{ + corev1api.ResourceStorage: resource.MustParse("1Gi"), + }, + PersistentVolumeSource: corev1api.PersistentVolumeSource{ + CSI: &corev1api.CSIPersistentVolumeSource{Driver: "ebs.csi.aws.com"}, + }}, + }, + skip: true, + }, + { + name: "empty nfs", + yamlData: `version: v1 +volumePolicies: +- conditions: + nfs: {} + action: + type: skip`, + vol: &corev1api.PersistentVolume{ + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeSource: corev1api.PersistentVolumeSource{ + NFS: &corev1api.NFSVolumeSource{Server: "192.168.1.20"}, + }}, + }, + skip: true, }, { name: "nfs not configured", @@ -986,251 +1456,1346 @@ volumePolicies: { name: "PVC phase matching - Pending phase should skip", yamlData: `version: v1 -volumePolicies: -- conditions: - pvcPhase: ["Pending"] - action: - type: skip`, - vol: nil, - podVol: nil, - pvc: &corev1api.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "default", - Name: "pvc-pending", - }, - Status: corev1api.PersistentVolumeClaimStatus{ - Phase: corev1api.ClaimPending, - }, - }, - skip: true, +volumePolicies: +- conditions: + pvcPhase: ["Pending"] + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-pending", + }, + Status: corev1api.PersistentVolumeClaimStatus{ + Phase: corev1api.ClaimPending, + }, + }, + skip: true, + }, + { + name: "PVC phase matching - Bound phase should not skip", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcPhase: ["Pending"] + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-bound", + }, + Status: corev1api.PersistentVolumeClaimStatus{ + Phase: corev1api.ClaimBound, + }, + }, + skip: false, + }, + { + name: "PVC phase matching - Multiple phases (Pending, Lost)", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcPhase: ["Pending", "Lost"] + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-lost", + }, + Status: corev1api.PersistentVolumeClaimStatus{ + Phase: corev1api.ClaimLost, + }, + }, + skip: true, + }, + { + name: "PVC volume mode matching - Block volume mode should skip", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Block + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-block", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: pvcVolumeMode(corev1api.PersistentVolumeBlock), + }, + }, + skip: true, + }, + { + name: "PVC volume mode matching - Filesystem volume mode should not skip", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Block + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-filesystem", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: pvcVolumeMode(corev1api.PersistentVolumeFilesystem), + }, + }, + skip: false, + }, + { + name: "PVC volume mode matching - nil volume mode should not match Filesystem", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Filesystem + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-without-volume-mode", + }, + }, + skip: false, + }, + { + name: "PVC volume mode matching - unknown condition value should not match empty volume mode", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: foo + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-without-volume-mode", + }, + }, + skip: false, + }, + { + name: "PVC volume mode matching - omitted condition should not restrict volume mode", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcAccessModes: ["ReadWriteOnce"] + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-block-rwo", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: pvcVolumeMode(corev1api.PersistentVolumeBlock), + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce}, + }, + }, + skip: true, + }, + { + name: "PVC volume mode matching - non-PVC volume should not match", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Filesystem + action: + type: skip`, + vol: nil, + podVol: &corev1api.Volume{ + Name: "empty-dir-volume", + VolumeSource: corev1api.VolumeSource{ + EmptyDir: &corev1api.EmptyDirVolumeSource{}, + }, + }, + pvc: nil, + skip: false, + }, + { + name: "PVC access modes matching - non-PVC volume should not match", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcAccessModes: ["ReadWriteOnce"] + action: + type: skip`, + vol: nil, + podVol: &corev1api.Volume{ + Name: "configmap-volume", + VolumeSource: corev1api.VolumeSource{ + ConfigMap: &corev1api.ConfigMapVolumeSource{}, + }, + }, + pvc: nil, + skip: false, + }, + + { + name: "PVC access modes matching - ReadWriteOnce should skip", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcAccessModes: ["ReadWriteOnce"] + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-rwo", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce}, + }, + }, + skip: true, + }, + { + name: "PVC access modes matching - extra PVC access mode should not skip", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcAccessModes: ["ReadWriteOnce"] + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-rwo-rom", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce, corev1api.ReadOnlyMany}, + }, + }, + skip: false, + }, + { + name: "PVC access modes matching - ReadWriteMany should not skip", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcAccessModes: ["ReadWriteOnce"] + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-rwx", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteMany}, + }, + }, + skip: false, + }, + { + name: "PVC access modes matching - exact access mode set should match regardless of order", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcAccessModes: ["ReadWriteMany", "ReadOnlyMany"] + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-rom-rwx", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadOnlyMany, corev1api.ReadWriteMany}, + }, + }, + skip: true, + }, + { + name: "PVC access modes matching - missing one configured access mode should not skip", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcAccessModes: ["ReadOnlyMany", "ReadWriteMany"] + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-rwx", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteMany}, + }, + }, + skip: false, + }, + { + name: "PVC access modes matching - Combined with volume mode", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Block + pvcAccessModes: ["ReadWriteOnce"] + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-block-rwo", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: pvcVolumeMode(corev1api.PersistentVolumeBlock), + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce}, + }, + }, + skip: true, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + resPolicies, err := unmarshalResourcePolicies(&tc.yamlData) + if err != nil { + t.Fatalf("got error when get match action %v", err) + } + require.NoError(t, err) + policies := &Policies{} + err = policies.BuildPolicy(resPolicies) + require.NoError(t, err) + vfd := VolumeFilterData{} + if tc.pvc != nil { + vfd.PVC = tc.pvc + } + + if tc.vol != nil { + vfd.PersistentVolume = tc.vol + } + + if tc.podVol != nil { + vfd.PodVolume = tc.podVol + } + + action, err := policies.GetMatchAction(vfd) + require.NoError(t, err) + + if tc.skip { + if action.Type != Skip { + t.Fatalf("Expected action skip but is %v", action.Type) + } + } else if action != nil && action.Type == Skip { + t.Fatalf("Expected action not skip but is %v", action.Type) + } + }) + } +} + +func TestGetMatchAction_Errors(t *testing.T) { + p := &Policies{} + + testCases := []struct { + name string + input any + expectedErr string + }{ + { + name: "invalid input type", + input: "invalid input", + expectedErr: "failed to convert input to VolumeFilterData", + }, + { + name: "no volume provided", + input: VolumeFilterData{ + PersistentVolume: nil, + PodVolume: nil, + PVC: nil, + }, + expectedErr: "failed to convert object", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + action, err := p.GetMatchAction(tc.input) + require.ErrorContains(t, err, tc.expectedErr) + assert.Nil(t, action) + }) + } +} + +func TestParsePVC(t *testing.T) { + tests := []struct { + name string + pvc *corev1api.PersistentVolumeClaim + expectedLabels map[string]string + expectedPhase string + expectedVolumeMode string + expectedAccessModes []string + expectErr bool + }{ + { + name: "valid PVC with labels, Pending phase, Block volume mode, and access modes", + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"env": "prod"}, + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: pvcVolumeMode(corev1api.PersistentVolumeBlock), + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce, corev1api.ReadOnlyMany}, + }, + Status: corev1api.PersistentVolumeClaimStatus{ + Phase: corev1api.ClaimPending, + }, + }, + expectedLabels: map[string]string{"env": "prod"}, + expectedPhase: "Pending", + expectedVolumeMode: "Block", + expectedAccessModes: []string{"ReadWriteOnce", "ReadOnlyMany"}, + expectErr: false, + }, + { + name: "valid PVC with Bound phase and nil volume mode", + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{}, + }, + Status: corev1api.PersistentVolumeClaimStatus{ + Phase: corev1api.ClaimBound, + }, + }, + expectedLabels: nil, + expectedPhase: "Bound", + expectedVolumeMode: "", + expectedAccessModes: nil, + expectErr: false, + }, + { + name: "valid PVC with Lost phase and Filesystem volume mode", + pvc: &corev1api.PersistentVolumeClaim{ + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: pvcVolumeMode(corev1api.PersistentVolumeFilesystem), + }, + Status: corev1api.PersistentVolumeClaimStatus{ + Phase: corev1api.ClaimLost, + }, + }, + expectedLabels: nil, + expectedPhase: "Lost", + expectedVolumeMode: "Filesystem", + expectedAccessModes: nil, + expectErr: false, + }, + { + name: "valid PVC with unknown non-nil volume mode", + pvc: &corev1api.PersistentVolumeClaim{ + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: pvcVolumeMode(corev1api.PersistentVolumeMode("foo")), + }, + Status: corev1api.PersistentVolumeClaimStatus{ + Phase: corev1api.ClaimBound, + }, + }, + expectedLabels: nil, + expectedPhase: "Bound", + expectedVolumeMode: "foo", + expectedAccessModes: nil, + expectErr: false, + }, + { + name: "nil PVC pointer", + pvc: (*corev1api.PersistentVolumeClaim)(nil), + expectedLabels: nil, + expectedPhase: "", + expectedVolumeMode: "", + expectedAccessModes: nil, + expectErr: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s := &structuredVolume{} + s.parsePVC(tc.pvc) + + assert.Equal(t, tc.expectedLabels, s.pvcLabels) + assert.Equal(t, tc.expectedPhase, s.pvcPhase) + assert.Equal(t, tc.expectedVolumeMode, s.pvcVolumeMode) + assert.Equal(t, tc.expectedAccessModes, s.pvcAccessModes) + }) + } +} + +func TestPVCPhaseMatch(t *testing.T) { + tests := []struct { + name string + condition *pvcPhaseCondition + volume *structuredVolume + expectedMatch bool + }{ + { + name: "match Pending phase", + condition: &pvcPhaseCondition{phases: []string{"Pending"}}, + volume: &structuredVolume{pvcPhase: "Pending"}, + expectedMatch: true, + }, + { + name: "match multiple phases - Pending matches", + condition: &pvcPhaseCondition{phases: []string{"Pending", "Bound"}}, + volume: &structuredVolume{pvcPhase: "Pending"}, + expectedMatch: true, + }, + { + name: "match multiple phases - Bound matches", + condition: &pvcPhaseCondition{phases: []string{"Pending", "Bound"}}, + volume: &structuredVolume{pvcPhase: "Bound"}, + expectedMatch: true, + }, + { + name: "no match for different phase", + condition: &pvcPhaseCondition{phases: []string{"Pending"}}, + volume: &structuredVolume{pvcPhase: "Bound"}, + expectedMatch: false, + }, + { + name: "no match for empty phase", + condition: &pvcPhaseCondition{phases: []string{"Pending"}}, + volume: &structuredVolume{pvcPhase: ""}, + expectedMatch: false, + }, + { + name: "match with empty phases list (always match)", + condition: &pvcPhaseCondition{phases: []string{}}, + volume: &structuredVolume{pvcPhase: "Pending"}, + expectedMatch: true, + }, + { + name: "match with nil phases list (always match)", + condition: &pvcPhaseCondition{phases: nil}, + volume: &structuredVolume{pvcPhase: "Pending"}, + expectedMatch: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := tc.condition.match(tc.volume) + assert.Equal(t, tc.expectedMatch, result) + }) + } +} + +func TestNamespacedFilterPolicies(t *testing.T) { + testCases := []struct { + name string + yamlData string + wantErr bool + errMsg string + }{ + { + name: "valid namespacedFilterPolicies with multiple kinds", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["frontend", "backend"] + resourceFilters: + - kinds: ["Pod", "ConfigMap"] + labelSelector: + matchLabels: + app: web + names: ["app-*"] + - kinds: ["Secret"] + excludedNames: ["temp-*"]`, + wantErr: false, + }, + { + name: "valid namespacedFilterPolicies with glob patterns", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["team-*"] + resourceFilters: + - kinds: ["Pod"] + orLabelSelectors: + - matchLabels: + env: prod + - matchLabels: + env: staging`, + wantErr: false, + }, + { + name: "valid - overlapping patterns allowed (first-match semantics)", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["team-frontend-*"] + resourceFilters: + - kinds: ["Pod", "ConfigMap", "Secret"] +- namespaces: ["team-*"] + resourceFilters: + - kinds: ["Deployment", "Service"]`, + wantErr: false, + }, + { + name: "invalid - no namespaces", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: [] + resourceFilters: + - kinds: ["Pod"]`, + wantErr: true, + errMsg: "at least one namespace must be specified", + }, + { + name: "invalid - no resourceFilters", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: []`, + wantErr: true, + errMsg: "at least one resourceFilter must be specified", + }, + { + name: "valid - asterisk catch-all", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: ["*"] + labelSelector: + matchLabels: + app: web`, + wantErr: false, + }, + { + name: "invalid - multiple asterisk kinds", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: ["*"] + labelSelector: + matchLabels: + app: web + - kinds: ["*"] + labelSelector: + matchLabels: + app: db`, + wantErr: true, + errMsg: "only one catch-all resource filter is allowed", + }, + { + name: "invalid - empty and asterisk kinds", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: [] + labelSelector: + matchLabels: + app: web + - kinds: ["*"] + labelSelector: + matchLabels: + app: db`, + wantErr: true, + errMsg: "only one catch-all resource filter is allowed", + }, + { + name: "invalid - multiple empty kinds", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: [] + labelSelector: + matchLabels: + app: web + - kinds: [] + labelSelector: + matchLabels: + app: db`, + wantErr: true, + errMsg: "only one catch-all resource filter is allowed", + }, + { + name: "invalid - names with empty kinds", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: [] + names: ["app-*"] + labelSelector: + matchLabels: + app: web`, + wantErr: true, + errMsg: "names or excludedNames cannot be specified for catch-all filters", + }, + { + name: "invalid - excludedNames with empty kinds", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: [] + excludedNames: ["app-*"] + labelSelector: + matchLabels: + app: web`, + wantErr: true, + errMsg: "names or excludedNames cannot be specified for catch-all filters", + }, + { + name: "valid - no label selectors with catch-all", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: ["*"]`, + wantErr: false, + }, + { + name: "invalid - duplicate kinds", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: ["Pod"] + - kinds: ["Pod", "ConfigMap"]`, + wantErr: true, + errMsg: "kind \"Pod\" appears in both resourceFilters", + }, + { + name: "invalid - both labelSelector and orLabelSelectors", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: ["Pod"] + labelSelector: + matchLabels: + app: web + orLabelSelectors: + - matchLabels: + env: prod`, + wantErr: true, + errMsg: "labelSelector and orLabelSelectors cannot co-exist", + }, + { + name: "invalid - bad glob pattern in names", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: ["Pod"] + names: ["[invalid"]`, + wantErr: true, + errMsg: "invalid glob pattern", + }, + { + name: "invalid - bad glob pattern in excludedNames", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: ["Pod"] + excludedNames: ["[invalid"]`, + wantErr: true, + errMsg: "invalid glob pattern", + }, + { + name: "invalid - duplicate namespace pattern", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["production"] + resourceFilters: + - kinds: ["Pod"] +- namespaces: ["production"] + resourceFilters: + - kinds: ["ConfigMap"]`, + wantErr: true, + errMsg: "duplicate namespace pattern", + }, + { + name: "invalid - bad namespace pattern", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["prod**uction"] + resourceFilters: + - kinds: ["Pod"]`, + wantErr: true, + errMsg: "wildcard pattern contains consecutive asterisks", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + resPolicies, err := unmarshalResourcePolicies(&tc.yamlData) + require.NoError(t, err) // Unmarshal should always succeed for our test cases + + policies := &Policies{} + err = policies.BuildPolicy(resPolicies) + require.NoError(t, err) // BuildPolicy should always succeed for our test cases + + err = policies.Validate() + if tc.wantErr { + require.Error(t, err) + if tc.errMsg != "" { + assert.Contains(t, err.Error(), tc.errMsg) + } + } else { + require.NoError(t, err) + + // Verify that we can retrieve the policies + nfPolicies := policies.GetNamespacedFilterPolicies() + assert.GreaterOrEqual(t, len(nfPolicies), 1) // Valid test cases have at least 1 policy + } + }) + } +} + +func TestNamespacedFilterPoliciesAccessor(t *testing.T) { + yamlData := `version: v1 +namespacedFilterPolicies: +- namespaces: ["frontend"] + resourceFilters: + - kinds: ["Pod"] + labelSelector: + matchLabels: + app: web` + + resPolicies, err := unmarshalResourcePolicies(&yamlData) + require.NoError(t, err) + + policies := &Policies{} + err = policies.BuildPolicy(resPolicies) + require.NoError(t, err) + + nfPolicies := policies.GetNamespacedFilterPolicies() + require.Len(t, nfPolicies, 1) + + policy := nfPolicies[0] + assert.Equal(t, []string{"frontend"}, policy.Namespaces) + assert.Len(t, policy.ResourceFilters, 1) + + rf := policy.ResourceFilters[0] + assert.Equal(t, []string{"Pod"}, rf.Kinds) + assert.Equal(t, &PolicyLabelSelector{MatchLabels: map[string]string{"app": "web"}}, rf.LabelSelector) +} + +func TestPolicyLabelSelectorSetBased(t *testing.T) { + t.Run("yaml decode matchLabels and matchExpressions", func(t *testing.T) { + yamlData := `version: v1 +namespacedFilterPolicies: +- namespaces: ["ns1"] + resourceFilters: + - kinds: ["Pod"] + labelSelector: + matchLabels: + app: web + matchExpressions: + - key: environment + operator: In + values: [prod, staging] + - key: do-not-backup + operator: DoesNotExist` + + resPolicies, err := unmarshalResourcePolicies(&yamlData) + require.NoError(t, err) + + policies := &Policies{} + require.NoError(t, policies.BuildPolicy(resPolicies)) + require.NoError(t, policies.Validate()) + + rf := policies.GetNamespacedFilterPolicies()[0].ResourceFilters[0] + require.NotNil(t, rf.LabelSelector) + assert.Equal(t, map[string]string{"app": "web"}, rf.LabelSelector.MatchLabels) + require.Len(t, rf.LabelSelector.MatchExpressions, 2) + assert.Equal(t, "environment", rf.LabelSelector.MatchExpressions[0].Key) + assert.Equal(t, "In", rf.LabelSelector.MatchExpressions[0].Operator) + assert.Equal(t, []string{"prod", "staging"}, rf.LabelSelector.MatchExpressions[0].Values) + assert.Equal(t, "do-not-backup", rf.LabelSelector.MatchExpressions[1].Key) + assert.Equal(t, "DoesNotExist", rf.LabelSelector.MatchExpressions[1].Operator) + }) + + t.Run("empty labelSelector is no filter", func(t *testing.T) { + yamlData := `version: v1 +namespacedFilterPolicies: +- namespaces: ["ns1"] + resourceFilters: + - kinds: ["Pod"] + labelSelector: {}` + + resPolicies, err := unmarshalResourcePolicies(&yamlData) + require.NoError(t, err) + + policies := &Policies{} + require.NoError(t, policies.BuildPolicy(resPolicies)) + require.NoError(t, policies.Validate()) + + rf := policies.GetNamespacedFilterPolicies()[0].ResourceFilters[0] + assert.False(t, IsPresentLabelSelector(rf.LabelSelector)) + }) + + t.Run("invalid operator rejected", func(t *testing.T) { + yamlData := `version: v1 +namespacedFilterPolicies: +- namespaces: ["ns1"] + resourceFilters: + - kinds: ["Pod"] + labelSelector: + matchExpressions: + - key: environment + operator: Equals + values: [prod]` + + resPolicies, err := unmarshalResourcePolicies(&yamlData) + require.NoError(t, err) + + policies := &Policies{} + require.NoError(t, policies.BuildPolicy(resPolicies)) + err = policies.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid label selector") + }) + + t.Run("NotIn Exists operators validate", func(t *testing.T) { + yamlData := `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole"] + labelSelector: + matchExpressions: + - key: tier + operator: NotIn + values: [debug] + - key: managed-by + operator: Exists` + + resPolicies, err := unmarshalResourcePolicies(&yamlData) + require.NoError(t, err) + + policies := &Policies{} + require.NoError(t, policies.BuildPolicy(resPolicies)) + require.NoError(t, policies.Validate()) + }) + + t.Run("ToMetaV1LabelSelector and IsPresentLabelSelector", func(t *testing.T) { + assert.False(t, IsPresentLabelSelector(nil)) + assert.False(t, IsPresentLabelSelector(&PolicyLabelSelector{})) + assert.True(t, IsPresentLabelSelector(&PolicyLabelSelector{MatchLabels: map[string]string{"a": "b"}})) + + ls := ToMetaV1LabelSelector(&PolicyLabelSelector{ + MatchLabels: map[string]string{"app": "web"}, + MatchExpressions: []PolicyLabelSelectorRequirement{ + {Key: "env", Operator: "In", Values: []string{"prod"}}, + }, + }) + require.NotNil(t, ls) + assert.Equal(t, map[string]string{"app": "web"}, ls.MatchLabels) + require.Len(t, ls.MatchExpressions, 1) + assert.Equal(t, metav1.LabelSelectorOpIn, ls.MatchExpressions[0].Operator) + + assert.Nil(t, ToMetaV1LabelSelector(nil)) + + sel, err := SelectorFromPolicyLabelSelector(&PolicyLabelSelector{ + MatchLabels: map[string]string{"app": "web"}, + }) + require.NoError(t, err) + require.NotNil(t, sel) + assert.True(t, sel.Matches(labels.Set{"app": "web"})) + + emptySel, err := SelectorFromPolicyLabelSelector(&PolicyLabelSelector{}) + require.NoError(t, err) + assert.Nil(t, emptySel) + }) +} + +func TestClusterScopedFilterPoliciesAccessor(t *testing.T) { + yamlData := `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole"] + names: ["my-app-*"]` + + resPolicies, err := unmarshalResourcePolicies(&yamlData) + require.NoError(t, err) + + policies := &Policies{} + err = policies.BuildPolicy(resPolicies) + require.NoError(t, err) + + csfPolicy := policies.GetClusterScopedFilterPolicy() + require.NotNil(t, csfPolicy) + assert.Len(t, csfPolicy.ResourceFilters, 1) + + rf := csfPolicy.ResourceFilters[0] + assert.Equal(t, []string{"ClusterRole"}, rf.Kinds) + assert.Equal(t, []string{"my-app-*"}, rf.Names) +} + +func TestIncludeExcludePolicyAccessor(t *testing.T) { + yamlData := `version: v1 +includeExcludePolicy: + includedClusterScopedResources: + - ClusterRole + excludedClusterScopedResources: + - ClusterRoleBinding` + + resPolicies, err := unmarshalResourcePolicies(&yamlData) + require.NoError(t, err) + + policies := &Policies{} + err = policies.BuildPolicy(resPolicies) + require.NoError(t, err) + + iePolicy := policies.GetIncludeExcludePolicy() + require.NotNil(t, iePolicy) + assert.Equal(t, []string{"ClusterRole"}, iePolicy.IncludedClusterScopedResources) + assert.Equal(t, []string{"ClusterRoleBinding"}, iePolicy.ExcludedClusterScopedResources) +} + +func TestFirstMatchSemantics(t *testing.T) { + yamlData := `version: v1 +namespacedFilterPolicies: +- namespaces: ["team-frontend-*", "specific-ns"] + resourceFilters: + - kinds: ["Pod", "ConfigMap", "Secret"] +- namespaces: ["team-*", "another-pattern"] + resourceFilters: + - kinds: ["Deployment", "Service"]` + + resPolicies, err := unmarshalResourcePolicies(&yamlData) + require.NoError(t, err) + + policies := &Policies{} + err = policies.BuildPolicy(resPolicies) + require.NoError(t, err) + + err = policies.Validate() + require.NoError(t, err) + + nfPolicies := policies.GetNamespacedFilterPolicies() + require.Len(t, nfPolicies, 2) + + // Verify the first policy has the more specific patterns + policy1 := nfPolicies[0] + assert.Equal(t, []string{"team-frontend-*", "specific-ns"}, policy1.Namespaces) + assert.Equal(t, []string{"Pod", "ConfigMap", "Secret"}, policy1.ResourceFilters[0].Kinds) + + // Verify the second policy has the broader patterns + policy2 := nfPolicies[1] + assert.Equal(t, []string{"team-*", "another-pattern"}, policy2.Namespaces) + assert.Equal(t, []string{"Deployment", "Service"}, policy2.ResourceFilters[0].Kinds) +} + +func TestClusterScopedFilterPolicies(t *testing.T) { + testCases := []struct { + name string + yamlData string + wantErr bool + errMsg string + }{ + { + name: "valid - single kind with names", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole"] + names: ["my-app-*"]`, + wantErr: false, + }, + { + name: "valid - multi-kind with labelSelector", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole", "ClusterRoleBinding"] + labelSelector: + matchLabels: + app: my-app`, + wantErr: false, + }, + { + name: "valid - orLabelSelectors", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["CustomResourceDefinition"] + orLabelSelectors: + - matchLabels: + app: my-app + - matchLabels: + app: other-app`, + wantErr: false, + }, + { + name: "valid - excludedNames", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole"] + names: ["my-*"] + excludedNames: ["my-debug-*"]`, + wantErr: false, + }, + { + name: "invalid - empty resourceFilters", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: []`, + wantErr: true, + errMsg: "resourceFilters cannot be empty; remove the policy block entirely if it is not needed", + }, + { + name: "invalid - empty kinds in clusterScopedFilterPolicy", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: [] + names: ["my-app-*"]`, + wantErr: true, + errMsg: "kinds must be specified", + }, + { + name: "invalid - asterisk kinds (explicit catch-all) in clusterScopedFilterPolicy", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["*"] + labelSelector: + matchLabels: + app: my-app`, + wantErr: true, + errMsg: "kinds must be specified", + }, + { + name: "invalid - duplicate kinds across entries", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole"] + names: ["my-app-*"] + - kinds: ["ClusterRole"] + labelSelector: + matchLabels: + app: other`, + wantErr: true, + errMsg: `kind "ClusterRole" appears in both`, + }, + { + name: "invalid - labelSelector and orLabelSelectors co-exist", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole"] + labelSelector: + matchLabels: + app: my-app + orLabelSelectors: + - matchLabels: + app: other`, + wantErr: true, + errMsg: "labelSelector and orLabelSelectors cannot co-exist", }, { - name: "PVC phase matching - Bound phase should not skip", + name: "invalid - bad glob in names", yamlData: `version: v1 -volumePolicies: -- conditions: - pvcPhase: ["Pending"] - action: - type: skip`, - vol: nil, - podVol: nil, - pvc: &corev1api.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "default", - Name: "pvc-bound", - }, - Status: corev1api.PersistentVolumeClaimStatus{ - Phase: corev1api.ClaimBound, - }, - }, - skip: false, +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole"] + names: ["[invalid"]`, + wantErr: true, + errMsg: "invalid glob pattern", }, { - name: "PVC phase matching - Multiple phases (Pending, Lost)", + name: "invalid - bad glob in excludedNames", yamlData: `version: v1 -volumePolicies: -- conditions: - pvcPhase: ["Pending", "Lost"] - action: - type: skip`, - vol: nil, - podVol: nil, - pvc: &corev1api.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "default", - Name: "pvc-lost", - }, - Status: corev1api.PersistentVolumeClaimStatus{ - Phase: corev1api.ClaimLost, - }, - }, - skip: true, +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole"] + excludedNames: ["[bad"]`, + wantErr: true, + errMsg: "invalid glob pattern", }, } + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { resPolicies, err := unmarshalResourcePolicies(&tc.yamlData) - if err != nil { - t.Fatalf("got error when get match action %v", err) - } require.NoError(t, err) + policies := &Policies{} err = policies.BuildPolicy(resPolicies) require.NoError(t, err) - vfd := VolumeFilterData{} - if tc.pvc != nil { - vfd.PVC = tc.pvc - } - - if tc.vol != nil { - vfd.PersistentVolume = tc.vol - } - - if tc.podVol != nil { - vfd.PodVolume = tc.podVol - } - - action, err := policies.GetMatchAction(vfd) - require.NoError(t, err) - if tc.skip { - if action.Type != Skip { - t.Fatalf("Expected action skip but is %v", action.Type) - } - } else if action != nil && action.Type == Skip { - t.Fatalf("Expected action not skip but is %v", action.Type) + err = policies.Validate() + if tc.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errMsg) + } else { + require.NoError(t, err) } }) } } -func TestGetMatchAction_Errors(t *testing.T) { - p := &Policies{} - - testCases := []struct { - name string - input any - expectedErr string +func TestPVCVolumeModeMatch(t *testing.T) { + tests := []struct { + name string + condition *pvcVolumeModeCondition + volume *structuredVolume + expectedMatch bool }{ { - name: "invalid input type", - input: "invalid input", - expectedErr: "failed to convert input to VolumeFilterData", + name: "match Block volume mode", + condition: &pvcVolumeModeCondition{volumeMode: "Block"}, + volume: &structuredVolume{pvcVolumeMode: "Block"}, + expectedMatch: true, }, { - name: "no volume provided", - input: VolumeFilterData{ - PersistentVolume: nil, - PodVolume: nil, - PVC: nil, - }, - expectedErr: "failed to convert object", + name: "match Filesystem volume mode", + condition: &pvcVolumeModeCondition{volumeMode: "Filesystem"}, + volume: &structuredVolume{pvcVolumeMode: "Filesystem"}, + expectedMatch: true, }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - action, err := p.GetMatchAction(tc.input) - require.ErrorContains(t, err, tc.expectedErr) - assert.Nil(t, action) - }) - } -} - -func TestParsePVC(t *testing.T) { - tests := []struct { - name string - pvc *corev1api.PersistentVolumeClaim - expectedLabels map[string]string - expectedPhase string - expectErr bool - }{ { - name: "valid PVC with labels and Pending phase", - pvc: &corev1api.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"env": "prod"}, - }, - Status: corev1api.PersistentVolumeClaimStatus{ - Phase: corev1api.ClaimPending, - }, - }, - expectedLabels: map[string]string{"env": "prod"}, - expectedPhase: "Pending", - expectErr: false, + name: "no match for different volume mode", + condition: &pvcVolumeModeCondition{volumeMode: "Block"}, + volume: &structuredVolume{pvcVolumeMode: "Filesystem"}, + expectedMatch: false, }, { - name: "valid PVC with Bound phase", - pvc: &corev1api.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{}, - }, - Status: corev1api.PersistentVolumeClaimStatus{ - Phase: corev1api.ClaimBound, - }, - }, - expectedLabels: nil, - expectedPhase: "Bound", - expectErr: false, + name: "case-sensitive no match for lowercase volume mode", + condition: &pvcVolumeModeCondition{volumeMode: "block"}, + volume: &structuredVolume{pvcVolumeMode: "Block"}, + expectedMatch: false, }, { - name: "valid PVC with Lost phase", - pvc: &corev1api.PersistentVolumeClaim{ - Status: corev1api.PersistentVolumeClaimStatus{ - Phase: corev1api.ClaimLost, - }, - }, - expectedLabels: nil, - expectedPhase: "Lost", - expectErr: false, + name: "no match for unknown condition value against Filesystem", + condition: &pvcVolumeModeCondition{volumeMode: "foo"}, + volume: &structuredVolume{pvcVolumeMode: "Filesystem"}, + expectedMatch: false, }, { - name: "nil PVC pointer", - pvc: (*corev1api.PersistentVolumeClaim)(nil), - expectedLabels: nil, - expectedPhase: "", - expectErr: false, + name: "match unknown condition value only when volume has same value", + condition: &pvcVolumeModeCondition{volumeMode: "foo"}, + volume: &structuredVolume{pvcVolumeMode: "foo"}, + expectedMatch: true, + }, + { + name: "no match for empty volume mode", + condition: &pvcVolumeModeCondition{volumeMode: "Block"}, + volume: &structuredVolume{pvcVolumeMode: ""}, + expectedMatch: false, + }, + { + name: "match with empty volume mode condition (always match)", + condition: &pvcVolumeModeCondition{volumeMode: ""}, + volume: &structuredVolume{pvcVolumeMode: "Block"}, + expectedMatch: true, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - s := &structuredVolume{} - s.parsePVC(tc.pvc) - - assert.Equal(t, tc.expectedLabels, s.pvcLabels) - assert.Equal(t, tc.expectedPhase, s.pvcPhase) + result := tc.condition.match(tc.volume) + assert.Equal(t, tc.expectedMatch, result) }) } } -func TestPVCPhaseMatch(t *testing.T) { +func TestPVCAccessModesMatch(t *testing.T) { tests := []struct { name string - condition *pvcPhaseCondition + condition *pvcAccessModesCondition volume *structuredVolume expectedMatch bool }{ { - name: "match Pending phase", - condition: &pvcPhaseCondition{phases: []string{"Pending"}}, - volume: &structuredVolume{pvcPhase: "Pending"}, + name: "match ReadWriteOnce access mode", + condition: &pvcAccessModesCondition{accessModes: []string{"ReadWriteOnce"}}, + volume: &structuredVolume{pvcAccessModes: []string{"ReadWriteOnce"}}, expectedMatch: true, }, { - name: "match multiple phases - Pending matches", - condition: &pvcPhaseCondition{phases: []string{"Pending", "Bound"}}, - volume: &structuredVolume{pvcPhase: "Pending"}, + name: "match exact multiple access modes", + condition: &pvcAccessModesCondition{accessModes: []string{"ReadWriteOnce", "ReadOnlyMany"}}, + volume: &structuredVolume{pvcAccessModes: []string{"ReadWriteOnce", "ReadOnlyMany"}}, expectedMatch: true, }, { - name: "match multiple phases - Bound matches", - condition: &pvcPhaseCondition{phases: []string{"Pending", "Bound"}}, - volume: &structuredVolume{pvcPhase: "Bound"}, + name: "match exact multiple access modes regardless of order", + condition: &pvcAccessModesCondition{accessModes: []string{"ReadOnlyMany", "ReadWriteOnce"}}, + volume: &structuredVolume{pvcAccessModes: []string{"ReadWriteOnce", "ReadOnlyMany"}}, expectedMatch: true, }, { - name: "no match for different phase", - condition: &pvcPhaseCondition{phases: []string{"Pending"}}, - volume: &structuredVolume{pvcPhase: "Bound"}, + name: "no match when one of multiple access modes is missing", + condition: &pvcAccessModesCondition{accessModes: []string{"ReadWriteOnce", "ReadOnlyMany"}}, + volume: &structuredVolume{pvcAccessModes: []string{"ReadOnlyMany"}}, expectedMatch: false, }, { - name: "no match for empty phase", - condition: &pvcPhaseCondition{phases: []string{"Pending"}}, - volume: &structuredVolume{pvcPhase: ""}, + name: "no match when PVC has extra access modes", + condition: &pvcAccessModesCondition{accessModes: []string{"ReadWriteMany"}}, + volume: &structuredVolume{pvcAccessModes: []string{"ReadWriteOnce", "ReadWriteMany"}}, expectedMatch: false, }, { - name: "match with empty phases list (always match)", - condition: &pvcPhaseCondition{phases: []string{}}, - volume: &structuredVolume{pvcPhase: "Pending"}, + name: "no match for different access mode", + condition: &pvcAccessModesCondition{accessModes: []string{"ReadWriteOnce"}}, + volume: &structuredVolume{pvcAccessModes: []string{"ReadWriteMany"}}, + expectedMatch: false, + }, + { + name: "case-sensitive no match for lowercase access mode", + condition: &pvcAccessModesCondition{accessModes: []string{"readwriteonce"}}, + volume: &structuredVolume{pvcAccessModes: []string{"ReadWriteOnce"}}, + expectedMatch: false, + }, + { + name: "no match for empty PVC access modes", + condition: &pvcAccessModesCondition{accessModes: []string{"ReadWriteOnce"}}, + volume: &structuredVolume{pvcAccessModes: []string{}}, + expectedMatch: false, + }, + { + name: "match with empty access modes list (always match)", + condition: &pvcAccessModesCondition{accessModes: []string{}}, + volume: &structuredVolume{pvcAccessModes: []string{"ReadWriteOnce"}}, expectedMatch: true, }, { - name: "match with nil phases list (always match)", - condition: &pvcPhaseCondition{phases: nil}, - volume: &structuredVolume{pvcPhase: "Pending"}, + name: "match with nil access modes list (always match)", + condition: &pvcAccessModesCondition{accessModes: nil}, + volume: &structuredVolume{pvcAccessModes: []string{"ReadWriteOnce"}}, expectedMatch: true, }, } @@ -1242,3 +2807,192 @@ func TestPVCPhaseMatch(t *testing.T) { }) } } + +// ---- Global backup volume policies ---- + +func globalPolicyConfigMap(name, data string) *corev1api.ConfigMap { + return &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: "velero", Name: name}, + Data: map[string]string{"policies.yaml": data}, + } +} + +func backupWithPolicy(ref string) velerov1api.Backup { + b := velerov1api.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "velero", Name: "backup"}} + if ref != "" { + b.Spec.ResourcePolicy = &corev1api.TypedLocalObjectReference{Kind: ConfigmapRefType, Name: ref} + } + return b +} + +// firstActionFor returns the action type the policies select for a PV with the given storage +// class, or "" when nothing matches. It exercises the compiled match logic so the tests verify +// merge ordering rather than internal field layout. +func firstActionFor(p *Policies, storageClass string) VolumeActionType { + pv := &corev1api.PersistentVolume{Spec: corev1api.PersistentVolumeSpec{StorageClassName: storageClass}} + vol := &structuredVolume{} + vol.parsePV(pv) + if a := p.match(vol); a != nil { + return a.Type + } + return "" +} + +func TestGetResourcePoliciesFromBackupWithGlobal(t *testing.T) { + gp2Skip := `version: v1 +volumePolicies: + - conditions: + storageClass: + - gp2 + action: + type: skip +` + gp2Snapshot := `version: v1 +volumePolicies: + - conditions: + storageClass: + - gp2 + action: + type: snapshot +` + otherFsBackup := `version: v1 +volumePolicies: + - conditions: + storageClass: + - other + action: + type: fs-backup +` + + tests := []struct { + name string + backupCM *corev1api.ConfigMap + globalCMName string + globalCM *corev1api.ConfigMap + backupRef string + expectErr bool + expectedGp2Action VolumeActionType + expectedNumPolicies int + }{ + { + name: "no global, backup only - unchanged behavior", + backupRef: "backup01", + backupCM: globalPolicyConfigMap("backup01", gp2Snapshot), + expectedGp2Action: Snapshot, + expectedNumPolicies: 1, + }, + { + name: "global only, backup has no policy", + globalCMName: "global", + globalCM: globalPolicyConfigMap("global", gp2Skip), + expectedGp2Action: Skip, + expectedNumPolicies: 1, + }, + { + name: "no global configured and no backup policy", + expectedGp2Action: "", + expectedNumPolicies: 0, + }, + { + name: "merge - backup policy overrides global for gp2", + backupRef: "backup01", + backupCM: globalPolicyConfigMap("backup01", gp2Snapshot), + globalCMName: "global", + globalCM: globalPolicyConfigMap("global", gp2Skip), + expectedGp2Action: Snapshot, // backup-level wins (evaluated first) + expectedNumPolicies: 2, + }, + { + name: "merge - backup inherits non-overlapping global rule", + backupRef: "backup01", + backupCM: globalPolicyConfigMap("backup01", otherFsBackup), + globalCMName: "global", + globalCM: globalPolicyConfigMap("global", gp2Skip), + expectedGp2Action: Skip, // only global matches gp2 + expectedNumPolicies: 2, + }, + { + name: "global configmap missing - error", + globalCMName: "global", + expectErr: true, + }, + { + name: "global configmap invalid - error", + globalCMName: "global", + globalCM: globalPolicyConfigMap("global", "not: [valid"), + expectErr: true, + }, + { + // Parses cleanly but fails Policies.Validate() due to the unsupported version. + name: "global configmap fails validation - error", + globalCMName: "global", + globalCM: globalPolicyConfigMap("global", "version: v2\nvolumePolicies: []\n"), + expectErr: true, + }, + { + // Backup references a ResourcePolicy ConfigMap that does not exist, so resolving the + // backup-level policies fails before the global ones are consulted. + name: "backup configmap missing - error", + backupRef: "missing-backup-cm", + globalCMName: "global", + globalCM: globalPolicyConfigMap("global", gp2Skip), + expectErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + client := velerotest.NewFakeControllerRuntimeClient(t) + if tc.backupCM != nil { + require.NoError(t, client.Create(t.Context(), tc.backupCM)) + } + if tc.globalCM != nil { + require.NoError(t, client.Create(t.Context(), tc.globalCM)) + } + + b := backupWithPolicy(tc.backupRef) + + p, err := GetResourcePoliciesFromBackupWithGlobal(b, client, tc.globalCMName, "velero", logrus.New()) + if tc.expectErr { + require.Error(t, err) + return + } + require.NoError(t, err) + + if tc.expectedNumPolicies == 0 { + assert.Nil(t, p) + return + } + require.NotNil(t, p) + assert.Len(t, p.volumePolicies, tc.expectedNumPolicies) + assert.Equal(t, tc.expectedGp2Action, firstActionFor(p, "gp2")) + }) + } +} + +func TestGetGlobalResourcePoliciesIgnoresNonVolumePolicies(t *testing.T) { + data := `version: v1 +volumePolicies: + - conditions: + storageClass: + - gp2 + action: + type: skip +namespacedFilterPolicies: +- namespaces: ["frontend"] + resourceFilters: + - kinds: ["Pod"] +` + client := velerotest.NewFakeControllerRuntimeClient(t) + require.NoError(t, client.Create(t.Context(), globalPolicyConfigMap("global", data))) + + p, err := GetGlobalResourcePolicies(client, "velero", "global", logrus.New()) + require.NoError(t, err) + require.NotNil(t, p) + + // Only volumePolicies are kept; the namespaced filter policy is dropped. + assert.Len(t, p.volumePolicies, 1) + assert.Empty(t, p.GetNamespacedFilterPolicies()) + assert.Nil(t, p.GetIncludeExcludePolicy()) + assert.Nil(t, p.GetClusterScopedFilterPolicy()) +} diff --git a/internal/resourcepolicies/volume_resources.go b/internal/resourcepolicies/volume_resources.go index 4ad34f484a..29514b9ee7 100644 --- a/internal/resourcepolicies/volume_resources.go +++ b/internal/resourcepolicies/volume_resources.go @@ -18,12 +18,14 @@ package resourcepolicies import ( "bytes" "fmt" + "slices" "strings" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" - "github.com/pkg/errors" - "gopkg.in/yaml.v3" + "github.com/cockroachdb/errors" + "go.yaml.in/yaml/v3" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) @@ -45,13 +47,15 @@ type capacity struct { } type structuredVolume struct { - capacity resource.Quantity - storageClass string - nfs *nFSVolumeSource - csi *csiVolumeSource - volumeType SupportedVolume - pvcLabels map[string]string - pvcPhase string + capacity resource.Quantity + storageClass string + nfs *nFSVolumeSource + csi *csiVolumeSource + volumeType SupportedVolume + pvcLabels map[string]string + pvcPhase string + pvcVolumeMode string + pvcAccessModes []string } func (s *structuredVolume) parsePV(pv *corev1api.PersistentVolume) { @@ -76,6 +80,15 @@ func (s *structuredVolume) parsePVC(pvc *corev1api.PersistentVolumeClaim) { s.pvcLabels = pvc.Labels } s.pvcPhase = string(pvc.Status.Phase) + if pvc.Spec.VolumeMode != nil { + s.pvcVolumeMode = string(*pvc.Spec.VolumeMode) + } + if len(pvc.Spec.AccessModes) > 0 { + s.pvcAccessModes = make([]string, 0, len(pvc.Spec.AccessModes)) + for _, accessMode := range pvc.Spec.AccessModes { + s.pvcAccessModes = append(s.pvcAccessModes, string(accessMode)) + } + } } } @@ -127,18 +140,55 @@ func (c *pvcPhaseCondition) match(v *structuredVolume) bool { if v.pvcPhase == "" { return false } - for _, phase := range c.phases { - if v.pvcPhase == phase { - return true - } - } - return false + return slices.Contains(c.phases, v.pvcPhase) } func (c *pvcPhaseCondition) validate() error { return nil } +// pvcVolumeModeCondition defines a condition that matches if the PVC's volume mode matches the provided volume mode. +type pvcVolumeModeCondition struct { + volumeMode string +} + +func (c *pvcVolumeModeCondition) match(v *structuredVolume) bool { + // No volume mode specified: always match. + if c.volumeMode == "" { + return true + } + + // Here allows unknown strings for forward compatibility. If Kubernetes adds another volume mode later, + // Velero would not reject the policy just because the string is unfamiliar. + return v.pvcVolumeMode == c.volumeMode +} + +func (c *pvcVolumeModeCondition) validate() error { + return nil +} + +// pvcAccessModesCondition defines a condition that matches if the PVC has exactly the provided access modes. +type pvcAccessModesCondition struct { + accessModes []string +} + +func (c *pvcAccessModesCondition) match(v *structuredVolume) bool { + // No access modes specified: always match. + if len(c.accessModes) == 0 { + return true + } + + if len(v.pvcAccessModes) != len(c.accessModes) { + return false + } + + return sets.New(c.accessModes...).Equal(sets.New(v.pvcAccessModes...)) +} + +func (c *pvcAccessModesCondition) validate() error { + return nil +} + type capacityCondition struct { capacity capacity } diff --git a/internal/resourcepolicies/volume_resources_test.go b/internal/resourcepolicies/volume_resources_test.go index b556923264..02850bb7f1 100644 --- a/internal/resourcepolicies/volume_resources_test.go +++ b/internal/resourcepolicies/volume_resources_test.go @@ -430,6 +430,38 @@ func TestUnmarshalVolumeConditions(t *testing.T) { }, expectedError: "!!str `production` into map[string]string", }, + { + name: "Valid pvcVolumeMode input", + input: map[string]any{ + "capacity": "1Gi,10Gi", + "pvcVolumeMode": "Block", + }, + expectedError: "", + }, + { + name: "Invalid pvcVolumeMode input: not a string", + input: map[string]any{ + "capacity": "1Gi,10Gi", + "pvcVolumeMode": []string{"Filesystem", "Block"}, + }, + expectedError: "cannot unmarshal !!seq", + }, + { + name: "Valid pvcAccessModes input", + input: map[string]any{ + "capacity": "1Gi,10Gi", + "pvcAccessModes": []string{"ReadWriteOnce", "ReadWriteMany"}, + }, + expectedError: "", + }, + { + name: "Invalid pvcAccessModes input: not a list", + input: map[string]any{ + "capacity": "1Gi,10Gi", + "pvcAccessModes": "ReadWriteOnce", + }, + expectedError: "cannot unmarshal !!str", + }, } for _, tc := range testCases { diff --git a/internal/resourcepolicies/volume_resources_validator.go b/internal/resourcepolicies/volume_resources_validator.go index 652c41d306..928e17df61 100644 --- a/internal/resourcepolicies/volume_resources_validator.go +++ b/internal/resourcepolicies/volume_resources_validator.go @@ -19,8 +19,8 @@ import ( "fmt" "io" - "github.com/pkg/errors" - "gopkg.in/yaml.v3" + "github.com/cockroachdb/errors" + "go.yaml.in/yaml/v3" ) const currentSupportDataVersion = "v1" @@ -40,13 +40,15 @@ type nFSVolumeSource struct { // volumeConditions defined the current format of conditions we parsed type volumeConditions struct { - Capacity string `yaml:"capacity,omitempty"` - StorageClass []string `yaml:"storageClass,omitempty"` - NFS *nFSVolumeSource `yaml:"nfs,omitempty"` - CSI *csiVolumeSource `yaml:"csi,omitempty"` - VolumeTypes []SupportedVolume `yaml:"volumeTypes,omitempty"` - PVCLabels map[string]string `yaml:"pvcLabels,omitempty"` - PVCPhase []string `yaml:"pvcPhase,omitempty"` + Capacity string `yaml:"capacity,omitempty"` + StorageClass []string `yaml:"storageClass,omitempty"` + NFS *nFSVolumeSource `yaml:"nfs,omitempty"` + CSI *csiVolumeSource `yaml:"csi,omitempty"` + VolumeTypes []SupportedVolume `yaml:"volumeTypes,omitempty"` + PVCLabels map[string]string `yaml:"pvcLabels,omitempty"` + PVCPhase []string `yaml:"pvcPhase,omitempty"` + PVCVolumeMode string `yaml:"pvcVolumeMode,omitempty"` + PVCAccessModes []string `yaml:"pvcAccessModes,omitempty"` } func (c *capacityCondition) validate() error { diff --git a/internal/resourcepolicies/volume_resources_validator_test.go b/internal/resourcepolicies/volume_resources_validator_test.go index f2812a786d..f2e6bf0e0c 100644 --- a/internal/resourcepolicies/volume_resources_validator_test.go +++ b/internal/resourcepolicies/volume_resources_validator_test.go @@ -568,3 +568,85 @@ func TestValidate(t *testing.T) { }) } } + +func TestValidateForRestore(t *testing.T) { + testCases := []struct { + name string + res *ResourcePolicies + wantErr bool + }{ + { + name: "valid restore policies", + res: &ResourcePolicies{ + Version: "v1", + ClusterScopedFilterPolicy: &ClusterScopedFilterPolicy{ + ResourceFilters: []ResourceFilter{ + { + Kinds: []string{"ClusterRole"}, + }, + }, + }, + NamespacedFilterPolicies: []NamespacedFilterPolicy{ + { + Namespaces: []string{"default"}, + ResourceFilters: []ResourceFilter{ + { + Kinds: []string{"Pod"}, + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "unsupported volumePolicies for restore", + res: &ResourcePolicies{ + Version: "v1", + VolumePolicies: []VolumePolicy{ + { + Action: Action{Type: "skip"}, + Conditions: map[string]any{ + "capacity": "10Gi", + }, + }, + }, + }, + wantErr: true, + }, + { + name: "unsupported includeExcludePolicy for restore", + res: &ResourcePolicies{ + Version: "v1", + IncludeExcludePolicy: &IncludeExcludePolicy{ + IncludedClusterScopedResources: []string{"persistentvolumes"}, + }, + }, + wantErr: true, + }, + { + name: "wrong version", + res: &ResourcePolicies{ + Version: "v2", + }, + wantErr: true, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + policies := &Policies{} + err1 := policies.BuildPolicy(tc.res) + err2 := policies.ValidateForRestore() + + if tc.wantErr { + if err1 == nil && err2 == nil { + t.Fatalf("Expected error %v, but not get error", tc.wantErr) + } + } else { + if err1 != nil || err2 != nil { + t.Fatalf("Expected error %v, but got error %v %v", tc.wantErr, err1, err2) + } + } + }) + } +} diff --git a/internal/restartabletest/restartable_delegate.go b/internal/restartabletest/restartable_delegate.go index 41d56cf31a..cfc1668c72 100644 --- a/internal/restartabletest/restartable_delegate.go +++ b/internal/restartabletest/restartable_delegate.go @@ -19,7 +19,7 @@ import ( "reflect" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" diff --git a/internal/storage/storagelocation.go b/internal/storage/storagelocation.go index d5fe548c0a..59afe6b79b 100644 --- a/internal/storage/storagelocation.go +++ b/internal/storage/storagelocation.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/internal/volume/snapshotlocation.go b/internal/volume/snapshotlocation.go index ad23fa1f4c..594fbf3a5a 100644 --- a/internal/volume/snapshotlocation.go +++ b/internal/volume/snapshotlocation.go @@ -17,7 +17,7 @@ limitations under the License. package volume import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/internal/volume/volumes_information.go b/internal/volume/volumes_information.go index 4d5961bdb0..69214ef459 100644 --- a/internal/volume/volumes_information.go +++ b/internal/volume/volumes_information.go @@ -22,8 +22,8 @@ import ( "strings" "sync" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,6 +36,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/features" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/util/stringptr" ) type Method string @@ -494,7 +495,8 @@ func (v *BackupVolumesInformation) generateVolumeInfoForCSIVolumeSnapshot() { tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo) } else { - v.logger.Warnf("cannot find info for PVC %s/%s", volumeSnapshot.Namespace, volumeSnapshot.Spec.Source.PersistentVolumeClaimName) + v.logger.Warnf("cannot find info for PVC %s/%s", volumeSnapshot.Namespace, + stringptr.GetString(volumeSnapshot.Spec.Source.PersistentVolumeClaimName)) continue } } diff --git a/internal/volumehelper/volume_policy_helper.go b/internal/volumehelper/volume_policy_helper.go index 339b800119..6931697c9b 100644 --- a/internal/volumehelper/volume_policy_helper.go +++ b/internal/volumehelper/volume_policy_helper.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/apis/velero/v1/labels_annotations.go b/pkg/apis/velero/v1/labels_annotations.go index 921af498ea..b34f05ed9e 100644 --- a/pkg/apis/velero/v1/labels_annotations.go +++ b/pkg/apis/velero/v1/labels_annotations.go @@ -80,6 +80,11 @@ const ( // timeout value for backup to plugins. ResourceTimeoutAnnotation = "velero.io/resource-timeout" + // GlobalBackupVolumePolicyConfigMapAnnotation is the annotation key used to record the + // name of the cluster-wide global backup volume policies ConfigMap that contributed to a + // backup, so that `velero backup describe` can surface it. + GlobalBackupVolumePolicyConfigMapAnnotation = "velero.io/global-backup-volume-policy-configmap" + // AsyncOperationIDLabel is the label key used to identify the async operation ID AsyncOperationIDLabel = "velero.io/async-operation-id" @@ -161,6 +166,14 @@ const ( // Velero checks this annotation to determine whether to skip resource excluding check. MustIncludeAdditionalItemAnnotation = "backup.velero.io/must-include-additional-items" + // MustIncludeAdditionalItemRestoreAnnotation is set by RestoreItemActions on the UpdatedItem + // to tell Velero to bypass global resource/namespace exclusion checks (and IncludeClusterResources=false) + // for that action's AdditionalItems. Value must be "true" to enable the bypass. The annotation is + // always stripped before the item is applied to the cluster when present, including non-"true" values. + // + // Notice: SkipRestore on the Execute output takes precedence. If SkipRestore is true, the + // annotation is never inspected and AdditionalItems are not processed. + MustIncludeAdditionalItemRestoreAnnotation = "restore.velero.io/must-include-additional-items" // SkippedNoCSIPVAnnotation - Velero checks this annotation on processed PVC to // find out if the snapshot was skipped b/c the PV is not provisioned via CSI SkippedNoCSIPVAnnotation = "backup.velero.io/skipped-no-csi-pv" diff --git a/pkg/apis/velero/v1/restore_types.go b/pkg/apis/velero/v1/restore_types.go index 5dd99edb7a..f6e6bf9cfb 100644 --- a/pkg/apis/velero/v1/restore_types.go +++ b/pkg/apis/velero/v1/restore_types.go @@ -125,6 +125,16 @@ type RestoreSpec struct { // +nullable ResourceModifier *corev1api.TypedLocalObjectReference `json:"resourceModifier,omitempty"` + // ResourcePolicy specifies the reference to a ConfigMap containing resource + // filter policies for this restore. The ConfigMap can contain a + // namespacedFilterPolicies section that specifies per-namespace resource type + // filters, label selectors, and resource name patterns, and a + // clusterScopedFilterPolicy section for per-kind filtering of cluster-scoped + // resources. The ConfigMap format is the same as for BackupSpec.ResourcePolicy. + // +optional + // +nullable + ResourcePolicy *corev1api.TypedLocalObjectReference `json:"resourcePolicy,omitempty"` + // UploaderConfig specifies the configuration for the restore. // +optional // +nullable diff --git a/pkg/apis/velero/v1/zz_generated.deepcopy.go b/pkg/apis/velero/v1/zz_generated.deepcopy.go index 0702f8623e..c40fbb8065 100644 --- a/pkg/apis/velero/v1/zz_generated.deepcopy.go +++ b/pkg/apis/velero/v1/zz_generated.deepcopy.go @@ -1415,6 +1415,11 @@ func (in *RestoreSpec) DeepCopyInto(out *RestoreSpec) { *out = new(corev1.TypedLocalObjectReference) (*in).DeepCopyInto(*out) } + if in.ResourcePolicy != nil { + in, out := &in.ResourcePolicy, &out.ResourcePolicy + *out = new(corev1.TypedLocalObjectReference) + (*in).DeepCopyInto(*out) + } if in.UploaderConfig != nil { in, out := &in.UploaderConfig, &out.UploaderConfig *out = new(UploaderConfigForRestore) diff --git a/pkg/archive/parser.go b/pkg/archive/parser.go index 166e03114d..4268138016 100644 --- a/pkg/archive/parser.go +++ b/pkg/archive/parser.go @@ -21,7 +21,7 @@ import ( "path/filepath" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/backup/actions/backup_pv_action.go b/pkg/backup/actions/backup_pv_action.go index c3f378fac1..32275544e2 100644 --- a/pkg/backup/actions/backup_pv_action.go +++ b/pkg/backup/actions/backup_pv_action.go @@ -19,7 +19,7 @@ package actions import ( "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/backup/actions/csi/pvc_action.go b/pkg/backup/actions/csi/pvc_action.go index 7f5fd2afa9..b2c7439bb0 100644 --- a/pkg/backup/actions/csi/pvc_action.go +++ b/pkg/backup/actions/csi/pvc_action.go @@ -24,9 +24,9 @@ import ( "k8s.io/client-go/util/retry" + "github.com/cockroachdb/errors" volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" storagev1api "k8s.io/api/storage/v1" @@ -47,8 +47,10 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" veleroclient "github.com/vmware-tanzu/velero/pkg/client" + "github.com/vmware-tanzu/velero/pkg/datamover" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/label" + "github.com/vmware-tanzu/velero/pkg/nodeagent" plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/utils/volumehelper" "github.com/vmware-tanzu/velero/pkg/plugin/velero" @@ -335,6 +337,17 @@ func (p *pvcBackupItemAction) Execute( return nil, nil, "", nil, err } + // validate that the node-agent daemonset is ready when snapshot data movement with + // the built-in data mover is requested. Without this, the DataUpload CR will be + // created but never processed (the DataUpload controller runs inside node-agent), + // causing the backup to hang until itemOperationTimeout expires. + if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) && datamover.IsBuiltInUploader(backup.Spec.DataMover) { + if err := nodeagent.IsReady(context.TODO(), backup.Namespace, p.crClient); err != nil { + p.log.WithError(err).Error("cannot perform snapshot data movement without running node-agent pods") + return nil, nil, "", nil, errors.Wrap(err, "CSI PVC BIA cannot proceed: node-agent is not ready for snapshot data movement") + } + } + vs, err := p.getVolumeSnapshotReference(context.TODO(), pvc, backup) if err != nil { return nil, nil, "", nil, err @@ -1183,7 +1196,7 @@ func setPVCRequestSizeToVSRestoreSize( logger logrus.FieldLogger, ) { if vsc.Status.RestoreSize != nil { - logger.Debugf("Patching PVC request size to fit the volumesnapshot restore size %d", vsc.Status.RestoreSize) + logger.Debugf("Patching PVC request size to fit the volumesnapshot restore size %d", *vsc.Status.RestoreSize) restoreSize := *resource.NewQuantity(*vsc.Status.RestoreSize, resource.BinarySI) // It is possible that the volume provider allocated a larger diff --git a/pkg/backup/actions/csi/pvc_action_test.go b/pkg/backup/actions/csi/pvc_action_test.go index 4cada562f6..dabc136299 100644 --- a/pkg/backup/actions/csi/pvc_action_test.go +++ b/pkg/backup/actions/csi/pvc_action_test.go @@ -29,7 +29,7 @@ import ( "github.com/stretchr/testify/assert" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" "github.com/vmware-tanzu/velero/pkg/label" @@ -37,9 +37,10 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" + appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" storagev1api "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -94,6 +95,7 @@ func TestExecute(t *testing.T) { expectedDataUpload *velerov2alpha1.DataUpload expectedPVC *corev1api.PersistentVolumeClaim resourcePolicy *corev1api.ConfigMap + extraObjects []runtime.Object failVSCreate bool skipVSReadyUpdate bool // New flag to control VS readiness }{ @@ -122,12 +124,21 @@ func TestExecute(t *testing.T) { expectErr: true, // Expect an error, but the exact message can vary }, { - name: "Test SnapshotMoveData", - backup: builder.ForBackup("velero", "test").SnapshotMoveData(true).CSISnapshotTimeout(1 * time.Minute).Result(), - pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1api.ClaimBound).Result(), - pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), - sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), - vsClass: builder.ForVolumeSnapshotClass("testVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), + name: "Test SnapshotMoveData", + backup: builder.ForBackup("velero", "test").SnapshotMoveData(true).CSISnapshotTimeout(1 * time.Minute).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1api.ClaimBound).Result(), + pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), + sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), + vsClass: builder.ForVolumeSnapshotClass("testVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), + extraObjects: []runtime.Object{ + &corev1api.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "linux-node", Labels: map[string]string{"kubernetes.io/os": "linux"}}, + }, + &appsv1api.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Namespace: "velero", Name: "node-agent"}, + Status: appsv1api.DaemonSetStatus{NumberReady: 3}, + }, + }, operationID: ".", expectedDataUpload: &velerov2alpha1.DataUpload{ TypeMeta: metav1.TypeMeta{ @@ -167,18 +178,37 @@ func TestExecute(t *testing.T) { }, }, { - name: "Verify PVC is modified as expected", - backup: builder.ForBackup("velero", "test").SnapshotMoveData(true).CSISnapshotTimeout(1 * time.Minute).Result(), - pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1api.ClaimBound).Result(), - pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), - sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), - vsClass: builder.ForVolumeSnapshotClass("tescVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), + name: "Verify PVC is modified as expected", + backup: builder.ForBackup("velero", "test").SnapshotMoveData(true).CSISnapshotTimeout(1 * time.Minute).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1api.ClaimBound).Result(), + pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), + sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), + vsClass: builder.ForVolumeSnapshotClass("tescVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), + extraObjects: []runtime.Object{ + &corev1api.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "linux-node", Labels: map[string]string{"kubernetes.io/os": "linux"}}, + }, + &appsv1api.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Namespace: "velero", Name: "node-agent"}, + Status: appsv1api.DaemonSetStatus{NumberReady: 3}, + }, + }, operationID: ".", expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC"). ObjectMeta(builder.WithAnnotations(velerov1api.MustIncludeAdditionalItemAnnotation, "true", velerov1api.DataUploadNameAnnotation, "velero/"), builder.WithLabels(velerov1api.BackupNameLabel, "test")). VolumeName("testPV").StorageClass("testSC").Phase(corev1api.ClaimBound).Result(), }, + { + name: "Test SnapshotMoveData without node-agent", + backup: builder.ForBackup("velero", "test").SnapshotMoveData(true).CSISnapshotTimeout(1 * time.Minute).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1api.ClaimBound).Result(), + pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), + sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), + vsClass: builder.ForVolumeSnapshotClass("testVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), + expectErr: true, + skipVSReadyUpdate: true, + }, { name: "Test ResourcePolicy", backup: builder.ForBackup("velero", "test").ResourcePolicies("resourcePolicy").SnapshotVolumes(false).CSISnapshotTimeout(time.Duration(3600) * time.Second).Result(), @@ -210,6 +240,7 @@ func TestExecute(t *testing.T) { if tc.resourcePolicy != nil { objects = append(objects, tc.resourcePolicy) } + objects = append(objects, tc.extraObjects...) var crClient crclient.Client if tc.failVSCreate { @@ -668,7 +699,7 @@ func TestFilterPVCsByVolumePolicy(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "pvc-1", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-1", - StorageClassName: pointer.String("sc-1"), + StorageClassName: ptr.To("sc-1"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -676,7 +707,7 @@ func TestFilterPVCsByVolumePolicy(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "pvc-2", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-2", - StorageClassName: pointer.String("sc-1"), + StorageClassName: ptr.To("sc-1"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -708,7 +739,7 @@ func TestFilterPVCsByVolumePolicy(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "pvc-csi", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-csi", - StorageClassName: pointer.String("sc-1"), + StorageClassName: ptr.To("sc-1"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -716,7 +747,7 @@ func TestFilterPVCsByVolumePolicy(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "pvc-nfs", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-nfs", - StorageClassName: pointer.String("sc-nfs"), + StorageClassName: ptr.To("sc-nfs"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -759,7 +790,7 @@ volumePolicies: ObjectMeta: metav1.ObjectMeta{Name: "pvc-nfs-1", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-nfs-1", - StorageClassName: pointer.String("sc-nfs"), + StorageClassName: ptr.To("sc-nfs"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -767,7 +798,7 @@ volumePolicies: ObjectMeta: metav1.ObjectMeta{Name: "pvc-nfs-2", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-nfs-2", - StorageClassName: pointer.String("sc-nfs"), + StorageClassName: ptr.To("sc-nfs"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -817,7 +848,7 @@ volumePolicies: }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-linstor", - StorageClassName: pointer.String("sc-linstor"), + StorageClassName: ptr.To("sc-linstor"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -829,7 +860,7 @@ volumePolicies: }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-nfs", - StorageClassName: pointer.String("sc-nfs"), + StorageClassName: ptr.To("sc-nfs"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -942,7 +973,7 @@ func TestFilterPVCsByVolumePolicyWithVolumeHelper(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "pvc-csi", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-csi", - StorageClassName: pointer.String("sc-csi"), + StorageClassName: ptr.To("sc-csi"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -950,7 +981,7 @@ func TestFilterPVCsByVolumePolicyWithVolumeHelper(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "pvc-nfs", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-nfs", - StorageClassName: pointer.String("sc-nfs"), + StorageClassName: ptr.To("sc-nfs"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -1364,7 +1395,7 @@ func TestWaitForVGSAssociatedVS(t *testing.T) { }, Spec: snapshotv1api.VolumeSnapshotSpec{ Source: snapshotv1api.VolumeSnapshotSource{ - PersistentVolumeClaimName: pointer.String(pvcName), + PersistentVolumeClaimName: ptr.To(pvcName), }, }, } @@ -1372,7 +1403,7 @@ func TestWaitForVGSAssociatedVS(t *testing.T) { if hasStatus { vs.Status = &snapshotv1api.VolumeSnapshotStatus{} if hasVGSName { - vs.Status.VolumeGroupSnapshotName = pointer.String(vgs.Name) + vs.Status.VolumeGroupSnapshotName = ptr.To(vgs.Name) } } @@ -1526,12 +1557,12 @@ func TestUpdateVGSCreatedVS(t *testing.T) { }, }, Status: &snapshotv1api.VolumeSnapshotStatus{ - ReadyToUse: pointer.Bool(true), + ReadyToUse: ptr.To(true), VolumeGroupSnapshotName: vgsNamePtr, }, Spec: snapshotv1api.VolumeSnapshotSpec{ Source: snapshotv1api.VolumeSnapshotSource{ - PersistentVolumeClaimName: pointer.String(pvcName), + PersistentVolumeClaimName: ptr.To(pvcName), }, }, } @@ -1546,7 +1577,7 @@ func TestUpdateVGSCreatedVS(t *testing.T) { }{ { name: "should update owned VS", - vs: makeVS("vs-owned", true, pointer.String(vgs.Name), "pvc-1"), + vs: makeVS("vs-owned", true, ptr.To(vgs.Name), "pvc-1"), expectOwnerCleared: true, expectFinalizersCleared: true, expectLabelPatched: true, @@ -1639,7 +1670,7 @@ func TestPatchVGSCDeletionPolicy(t *testing.T) { Namespace: "ns", }, Status: &volumegroupsnapshotv1beta2.VolumeGroupSnapshotStatus{ - BoundVolumeGroupSnapshotContentName: pointer.String("test-vgsc"), + BoundVolumeGroupSnapshotContentName: ptr.To("test-vgsc"), }, } @@ -1694,14 +1725,14 @@ func TestDeleteVGSAndVGSC(t *testing.T) { }{ { name: "deletes both VGSC and VGS", - vgs: makeVGS("test-vgs", "ns", pointer.String("test-vgsc")), + vgs: makeVGS("test-vgs", "ns", ptr.To("test-vgsc")), existingVGSC: makeVGSC("test-vgsc"), expectVGSCDelete: true, expectVGSDelete: true, }, { name: "VGSC not found, still deletes VGS", - vgs: makeVGS("test-vgs", "ns", pointer.String("missing-vgsc")), + vgs: makeVGS("test-vgs", "ns", ptr.To("missing-vgsc")), existingVGSC: nil, expectVGSCDelete: false, expectVGSDelete: true, @@ -1767,7 +1798,7 @@ func TestFindExistingVSForBackup(t *testing.T) { }, Spec: snapshotv1api.VolumeSnapshotSpec{ Source: snapshotv1api.VolumeSnapshotSource{ - PersistentVolumeClaimName: pointer.String(pvc), + PersistentVolumeClaimName: ptr.To(pvc), }, }, } @@ -2120,7 +2151,7 @@ func TestPVCRequestSize(t *testing.T) { Name: "testVSC", }, Status: &snapshotv1api.VolumeSnapshotContentStatus{ - RestoreSize: pointer.Int64(rsQty.Value()), + RestoreSize: ptr.To(rsQty.Value()), }, } diff --git a/pkg/backup/actions/csi/volumesnapshot_action.go b/pkg/backup/actions/csi/volumesnapshot_action.go index 0e0e9a8404..16d5a16ffc 100644 --- a/pkg/backup/actions/csi/volumesnapshot_action.go +++ b/pkg/backup/actions/csi/volumesnapshot_action.go @@ -22,8 +22,8 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -269,8 +269,8 @@ func (p *volumeSnapshotBackupItemAction) Progress( } var err error if progress.Started, err = time.Parse(time.RFC3339, operationIDParts[2]); err != nil { - p.log.Errorf("error parsing operation ID's StartedTime", - "part into time %s: %s", operationID, err.Error()) + p.log.Errorf("error parsing operation ID's StartedTime part into time %s: %s", + operationID, err.Error()) return progress, errors.WithStack(err) } diff --git a/pkg/backup/actions/csi/volumesnapshotclass_action.go b/pkg/backup/actions/csi/volumesnapshotclass_action.go index 8200b465f1..8d70fae175 100644 --- a/pkg/backup/actions/csi/volumesnapshotclass_action.go +++ b/pkg/backup/actions/csi/volumesnapshotclass_action.go @@ -17,7 +17,7 @@ limitations under the License. package csi import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" diff --git a/pkg/backup/actions/csi/volumesnapshotcontent_action.go b/pkg/backup/actions/csi/volumesnapshotcontent_action.go index d4cd6d46ca..fc93cfb876 100644 --- a/pkg/backup/actions/csi/volumesnapshotcontent_action.go +++ b/pkg/backup/actions/csi/volumesnapshotcontent_action.go @@ -19,8 +19,8 @@ package csi import ( "fmt" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -107,8 +107,7 @@ func (p *volumeSnapshotContentBackupItemAction) Execute( } p.log.Infof( - "Returning from VolumeSnapshotContentBackupItemAction", - "with %d additionalItems to backup", + "Returning from VolumeSnapshotContentBackupItemAction with %d additionalItems to backup", len(additionalItems), ) return &unstructured.Unstructured{Object: snapContMap}, additionalItems, "", nil, nil diff --git a/pkg/backup/actions/pod_action.go b/pkg/backup/actions/pod_action.go index 8ed5e3b44e..f8693228f1 100644 --- a/pkg/backup/actions/pod_action.go +++ b/pkg/backup/actions/pod_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/backup/actions/remap_crd_version_action.go b/pkg/backup/actions/remap_crd_version_action.go index 3f8c2f79d4..59a84ee9e2 100644 --- a/pkg/backup/actions/remap_crd_version_action.go +++ b/pkg/backup/actions/remap_crd_version_action.go @@ -20,7 +20,7 @@ import ( "context" "encoding/json" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" diff --git a/pkg/backup/actions/service_account_action.go b/pkg/backup/actions/service_account_action.go index b563f7a036..544fbcb23c 100644 --- a/pkg/backup/actions/service_account_action.go +++ b/pkg/backup/actions/service_account_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/backup/backup.go b/pkg/backup/backup.go index 8c21bea776..77921a090e 100644 --- a/pkg/backup/backup.go +++ b/pkg/backup/backup.go @@ -26,16 +26,19 @@ import ( "io" "os" "path/filepath" + "strings" "sync" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" + "github.com/gobwas/glob" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" kubeerrs "k8s.io/apimachinery/pkg/util/errors" @@ -43,6 +46,7 @@ import ( kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/hook" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" "github.com/vmware-tanzu/velero/internal/volumehelper" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -333,6 +337,52 @@ func (kb *kubernetesBackupper) BackupWithResolvers( backupRequest.ResourceIncludesExcludes = srie } + if backupRequest.ResPolicies != nil { + clusterScopedFilterPolicy := backupRequest.ResPolicies.GetClusterScopedFilterPolicy() + if clusterScopedFilterPolicy != nil { + backupRequest.ClusterScopedFilterMap, err = resolveClusterScopedFilterPolicy( + clusterScopedFilterPolicy, + kb.discoveryHelper, + log, + ) + if err != nil { + return err + } + log.Infof("Resolved clusterScopedFilterPolicy: %d kind group(s) in cluster-scoped filter map", + len(backupRequest.ClusterScopedFilterMap)) + } + + nfPolicies := backupRequest.ResPolicies.GetNamespacedFilterPolicies() + if len(nfPolicies) > 0 { + backupRequest.NamespacedFilterMap, backupRequest.NamespacedFilterPatterns, err = resolveNamespacedFilterPolicies( + nfPolicies, + kb.discoveryHelper, + log, + ) + if err != nil { + return err + } + log.Infof("Resolved namespacedFilterPolicies: %d namespace pattern(s) registered", + len(backupRequest.NamespacedFilterPatterns)) + for _, p := range backupRequest.NamespacedFilterPatterns { + nsf := backupRequest.NamespacedFilterMap[p.Pattern] + log.WithFields(logrus.Fields{ + "namespacePattern": p.Pattern, + "kindCount": len(nsf.ResourceFilterMap), + "hasCatchAll": nsf.CatchAllFilter != nil, + }).Debug("namespacedFilterPolicies: namespace pattern registered") + for kind := range nsf.ResourceFilterMap { + if backupRequest.ResourceIncludesExcludes.ShouldExclude(kind) { + log.WithFields(logrus.Fields{ + "namespacePattern": p.Pattern, + "kind": kind, + }).Warn("namespacedFilterPolicies entry lists a kind that is globally excluded by includeExcludePolicy; the per-namespace filter entry has no effect") + } + } + } + } + } + log.Infof("Backing up all volumes using pod volume backup: %t", boolptr.IsSetToTrue(backupRequest.Backup.Spec.DefaultVolumesToFsBackup)) backupRequest.ResourceHooks, err = getResourceHooks(backupRequest.Spec.Hooks.Resources, kb.discoveryHelper) @@ -1194,21 +1244,12 @@ func buildFinalTarball(tr *tar.Reader, tw tarWriter, updateFiles map[string]File return errors.WithStack(err) } delete(updateFiles, header.Name) - // skip over file contents from old tarball - _, err := io.ReadAll(tr) - if err != nil { - return errors.WithStack(err) - } } else { // Add original content to new tarball, as item wasn't updated - oldContents, err := io.ReadAll(tr) - if err != nil { - return errors.WithStack(err) - } if err := tw.WriteHeader(header); err != nil { return errors.WithStack(err) } - if _, err := tw.Write(oldContents); err != nil { + if _, err := io.Copy(tw, tr); err != nil { return errors.WithStack(err) } } @@ -1322,3 +1363,138 @@ func putVolumeInfos( return backupStore.PutBackupVolumeInfos(backupName, backupVolumeInfoBuf) } + +func resolveClusterScopedFilterPolicy( + policy *resourcepolicies.ClusterScopedFilterPolicy, + helper discovery.Helper, + log logrus.FieldLogger, +) (map[string]*ResolvedResourceFilter, error) { + rfMap := make(map[string]*ResolvedResourceFilter) + + for _, rf := range policy.ResourceFilters { + resolved, err := resolveResourceFilter(rf) + if err != nil { + return nil, err + } + + for _, kind := range rf.Kinds { + gr, apiResource, err := helper.ResourceFor( + schema.GroupVersionResource{Resource: kind}, + ) + if err != nil { + log.WithField("kind", kind).Warnf( + "Cannot resolve kind via discovery, using as-is: %v", err) + rfMap[kind] = resolved + continue + } + if apiResource.Namespaced { + log.WithField("kind", kind).Warnf( + "kind %q in clusterScopedFilterPolicy is namespace-scoped; "+ + "it will never match in a cluster-scoped filter — did you mean namespacedFilterPolicies?", kind) + } + rfMap[gr.GroupResource().String()] = resolved + } + } + + return rfMap, nil +} + +func resolveResourceFilter(rf resourcepolicies.ResourceFilter) (*ResolvedResourceFilter, error) { + selector, err := resourcepolicies.SelectorFromPolicyLabelSelector(rf.LabelSelector) + if err != nil { + return nil, fmt.Errorf("invalid label selector in resource filter: %w", err) + } + + var orSelectors []labels.Selector + for _, ols := range rf.OrLabelSelectors { + s, err := resourcepolicies.SelectorFromPolicyLabelSelector(ols) + if err != nil { + return nil, fmt.Errorf("invalid OR label selector in resource filter: %w", err) + } + if s != nil { + orSelectors = append(orSelectors, s) + } + } + + var nameIE *collections.IncludesExcludes + if len(rf.Names) > 0 || len(rf.ExcludedNames) > 0 { + nameIE = collections.NewIncludesExcludes() + nameIE.Includes(rf.Names...) + nameIE.Excludes(rf.ExcludedNames...) + } + + return &ResolvedResourceFilter{ + LabelSelector: selector, + OrLabelSelectors: orSelectors, + NameIE: nameIE, + }, nil +} + +func resolveNamespacedFilterPolicies( + policies []resourcepolicies.NamespacedFilterPolicy, + helper discovery.Helper, + log logrus.FieldLogger, +) (map[string]*ResolvedNamespaceFilter, []NamespacedFilterPattern, error) { + result := make(map[string]*ResolvedNamespaceFilter) + var patternOrder []NamespacedFilterPattern + + for _, policy := range policies { + rfMap := make(map[string]*ResolvedResourceFilter) + var nsFilter *ResolvedNamespaceFilter + + for _, rf := range policy.ResourceFilters { + resolved, err := resolveResourceFilter(rf) + if err != nil { + return nil, nil, err + } + + if rf.IsCatchAll() { + if nsFilter == nil { + nsFilter = &ResolvedNamespaceFilter{ResourceFilterMap: rfMap} + } + nsFilter.CatchAllFilter = resolved + } else { + // Resolve each kind to a fully-qualified group-resource string with improved error handling + for _, kind := range rf.Kinds { + gr, apiResource, err := helper.ResourceFor( + schema.GroupVersionResource{Resource: kind}, + ) + if err != nil { + // Log warning but continue - allows for forward compatibility + log.WithField("kind", kind).Warnf( + "Cannot resolve kind via discovery, using as-is: %v", err) + rfMap[kind] = resolved + continue + } + if !apiResource.Namespaced { + log.WithField("kind", kind).Warnf( + "kind %q in namespacedFilterPolicies is cluster-scoped; "+ + "it will never match in a namespace-scoped filter — did you mean clusterScopedFilterPolicy?", kind) + } + rfMap[gr.GroupResource().String()] = resolved + } + } + } + + if nsFilter == nil { + nsFilter = &ResolvedNamespaceFilter{} + } + nsFilter.ResourceFilterMap = rfMap + for _, nsPattern := range policy.Namespaces { + result[nsPattern] = nsFilter + // Pre-compile glob patterns once here; exact names are matched via map + // and never reach the pattern loop, so only wildcard patterns need Compiled set. + entry := NamespacedFilterPattern{Pattern: nsPattern} + if strings.ContainsAny(nsPattern, "*?[") { + if compiled, cerr := glob.Compile(nsPattern); cerr == nil { + entry.Compiled = compiled + } else { + // Pattern already validated; this branch should not be reached + log.WithField("pattern", nsPattern).Warnf("Failed to pre-compile glob pattern: %v", cerr) + } + } + patternOrder = append(patternOrder, entry) + } + } + return result, patternOrder, nil +} diff --git a/pkg/backup/backup_test.go b/pkg/backup/backup_test.go index 5607db9dd3..10416f3812 100644 --- a/pkg/backup/backup_test.go +++ b/pkg/backup/backup_test.go @@ -30,7 +30,8 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" + "github.com/gobwas/glob" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -40,7 +41,9 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" @@ -5736,3 +5739,481 @@ func (f *fakeSingleObjectBackupStoreGetter) Get(*velerov1.BackupStorageLocation, func NewFakeSingleObjectBackupStoreGetter(store persistence.BackupStore) persistence.ObjectBackupStoreGetter { return &fakeSingleObjectBackupStoreGetter{store: store} } +func TestResolveResourceFilter(t *testing.T) { + tests := []struct { + name string + rf resourcepolicies.ResourceFilter + expectErr bool + checkResult func(*testing.T, *ResolvedResourceFilter) + }{ + { + name: "valid label selector", + rf: resourcepolicies.ResourceFilter{ + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"app": "foo"}}, + }, + expectErr: false, + checkResult: func(t *testing.T, r *ResolvedResourceFilter) { + t.Helper() + require.NotNil(t, r) + require.NotNil(t, r.LabelSelector) + assert.True(t, r.LabelSelector.Matches(labels.Set{"app": "foo"})) + }, + }, + { + name: "invalid label selector", + rf: resourcepolicies.ResourceFilter{ + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"invalid/label/key": "value"}}, + }, + expectErr: true, + }, + { + name: "valid or label selectors", + rf: resourcepolicies.ResourceFilter{ + OrLabelSelectors: []*resourcepolicies.PolicyLabelSelector{ + {MatchLabels: map[string]string{"app": "foo"}}, + {MatchLabels: map[string]string{"app": "bar"}}, + }, + }, + expectErr: false, + checkResult: func(t *testing.T, r *ResolvedResourceFilter) { + t.Helper() + require.NotNil(t, r) + require.Len(t, r.OrLabelSelectors, 2) + }, + }, + { + name: "invalid or label selectors", + rf: resourcepolicies.ResourceFilter{ + OrLabelSelectors: []*resourcepolicies.PolicyLabelSelector{ + {MatchLabels: map[string]string{"invalid/label/key": "value"}}, + }, + }, + expectErr: true, + }, + { + name: "names and excluded names", + rf: resourcepolicies.ResourceFilter{ + Names: []string{"inc1", "inc2"}, + ExcludedNames: []string{"exc1"}, + }, + expectErr: false, + checkResult: func(t *testing.T, r *ResolvedResourceFilter) { + t.Helper() + require.NotNil(t, r) + require.NotNil(t, r.NameIE) + assert.True(t, r.NameIE.ShouldInclude("inc1")) + assert.False(t, r.NameIE.ShouldInclude("exc1")) + }, + }, + { + name: "empty labelSelector is no filter", + rf: resourcepolicies.ResourceFilter{ + LabelSelector: &resourcepolicies.PolicyLabelSelector{}, + }, + expectErr: false, + checkResult: func(t *testing.T, r *ResolvedResourceFilter) { + t.Helper() + require.NotNil(t, r) + assert.Nil(t, r.LabelSelector) + }, + }, + { + name: "set-based In and DoesNotExist", + rf: resourcepolicies.ResourceFilter{ + LabelSelector: &resourcepolicies.PolicyLabelSelector{ + MatchExpressions: []resourcepolicies.PolicyLabelSelectorRequirement{ + {Key: "environment", Operator: "In", Values: []string{"prod", "staging"}}, + {Key: "do-not-backup", Operator: "DoesNotExist"}, + }, + }, + }, + expectErr: false, + checkResult: func(t *testing.T, r *ResolvedResourceFilter) { + t.Helper() + require.NotNil(t, r.LabelSelector) + assert.True(t, r.LabelSelector.Matches(labels.Set{"environment": "prod"})) + assert.True(t, r.LabelSelector.Matches(labels.Set{"environment": "staging"})) + assert.False(t, r.LabelSelector.Matches(labels.Set{"environment": "dev"})) + assert.False(t, r.LabelSelector.Matches(labels.Set{"environment": "prod", "do-not-backup": "true"})) + }, + }, + { + name: "set-based NotIn and Exists", + rf: resourcepolicies.ResourceFilter{ + LabelSelector: &resourcepolicies.PolicyLabelSelector{ + MatchExpressions: []resourcepolicies.PolicyLabelSelectorRequirement{ + {Key: "tier", Operator: "NotIn", Values: []string{"debug"}}, + {Key: "app", Operator: "Exists"}, + }, + }, + }, + expectErr: false, + checkResult: func(t *testing.T, r *ResolvedResourceFilter) { + t.Helper() + require.NotNil(t, r.LabelSelector) + assert.True(t, r.LabelSelector.Matches(labels.Set{"app": "web", "tier": "frontend"})) + assert.False(t, r.LabelSelector.Matches(labels.Set{"app": "web", "tier": "debug"})) + assert.False(t, r.LabelSelector.Matches(labels.Set{"tier": "frontend"})) + }, + }, + { + name: "invalid operator", + rf: resourcepolicies.ResourceFilter{ + LabelSelector: &resourcepolicies.PolicyLabelSelector{ + MatchExpressions: []resourcepolicies.PolicyLabelSelectorRequirement{ + {Key: "env", Operator: "Equals", Values: []string{"prod"}}, + }, + }, + }, + expectErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + res, err := resolveResourceFilter(tc.rf) + if tc.expectErr { + require.Error(t, err) + } else { + assert.NoError(t, err) + if tc.checkResult != nil { + tc.checkResult(t, res) + } + } + }) + } +} + +type mockDiscoveryHelper struct { + discovery.Helper + ResourceForFunc func(input schema.GroupVersionResource) (schema.GroupVersionResource, metav1.APIResource, error) +} + +func (m *mockDiscoveryHelper) ResourceFor(input schema.GroupVersionResource) (schema.GroupVersionResource, metav1.APIResource, error) { + if m.ResourceForFunc != nil { + return m.ResourceForFunc(input) + } + return m.Helper.ResourceFor(input) +} + +func TestResolveClusterScopedFilterPolicy(t *testing.T) { + helper := test.NewFakeDiscoveryHelper(true, nil) + log := test.NewLogger() + + policy := &resourcepolicies.ClusterScopedFilterPolicy{ + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"pods", "secrets"}, + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"app": "foo"}}, + }, + { + Kinds: []string{"invalid-kind"}, + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"invalid/label/key": "value"}}, + }, + }, + } + + // Test with invalid label selector to trigger error + _, err := resolveClusterScopedFilterPolicy(policy, helper, log) + require.Error(t, err) + + // Test valid policy + validPolicy := &resourcepolicies.ClusterScopedFilterPolicy{ + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"pods", "secrets"}, + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"app": "foo"}}, + }, + }, + } + res, err := resolveClusterScopedFilterPolicy(validPolicy, helper, log) + require.NoError(t, err) + require.Len(t, res, 2) + assert.Contains(t, res, "pods") + assert.Contains(t, res, "secrets") + assert.True(t, res["pods"].LabelSelector.Matches(labels.Set{"app": "foo"})) + + // Test warning branches + mockHelper := &mockDiscoveryHelper{ + Helper: helper, + ResourceForFunc: func(input schema.GroupVersionResource) (schema.GroupVersionResource, metav1.APIResource, error) { + if input.Resource == "invalid-resource" { + return schema.GroupVersionResource{}, metav1.APIResource{}, errors.New("cannot resolve") + } + if input.Resource == "namespaced-resource" { + return schema.GroupVersionResource{Resource: "namespaced-resource"}, metav1.APIResource{Namespaced: true, Name: "namespaced-resource"}, nil + } + return helper.ResourceFor(input) + }, + } + + policyWithWarns := &resourcepolicies.ClusterScopedFilterPolicy{ + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"invalid-resource", "namespaced-resource"}, + }, + }, + } + res2, err2 := resolveClusterScopedFilterPolicy(policyWithWarns, mockHelper, log) + require.NoError(t, err2) + assert.Contains(t, res2, "invalid-resource") + assert.Contains(t, res2, "namespaced-resource") +} + +func TestResolveNamespacedFilterPolicies(t *testing.T) { + helper := test.NewFakeDiscoveryHelper(true, nil) + log := test.NewLogger() + + policies := []resourcepolicies.NamespacedFilterPolicy{ + { + Namespaces: []string{"ns1", "ns-*"}, + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"pods"}, + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"app": "foo"}}, + }, + { + Kinds: []string{"*"}, + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"catch": "all"}}, + }, + }, + }, + } + + res, patterns, err := resolveNamespacedFilterPolicies(policies, helper, log) + require.NoError(t, err) + require.Len(t, res, 2) + require.Len(t, patterns, 2) + + assert.Contains(t, res, "ns1") + assert.Contains(t, res, "ns-*") + + ns1Filter := res["ns1"] + require.NotNil(t, ns1Filter) + require.NotNil(t, ns1Filter.CatchAllFilter) + assert.True(t, ns1Filter.CatchAllFilter.LabelSelector.Matches(labels.Set{"catch": "all"})) + require.Contains(t, ns1Filter.ResourceFilterMap, "pods") + assert.True(t, ns1Filter.ResourceFilterMap["pods"].LabelSelector.Matches(labels.Set{"app": "foo"})) + + // Test with invalid label selector + invalidPolicies := []resourcepolicies.NamespacedFilterPolicy{ + { + Namespaces: []string{"ns1"}, + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"pods"}, + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"invalid/label/key": "value"}}, + }, + }, + }, + } + _, _, err = resolveNamespacedFilterPolicies(invalidPolicies, helper, log) + require.Error(t, err) + + // Test warning branches + mockHelper := &mockDiscoveryHelper{ + Helper: helper, + ResourceForFunc: func(input schema.GroupVersionResource) (schema.GroupVersionResource, metav1.APIResource, error) { + if input.Resource == "invalid-resource" { + return schema.GroupVersionResource{}, metav1.APIResource{}, errors.New("cannot resolve") + } + if input.Resource == "cluster-scoped-resource" { + return schema.GroupVersionResource{Resource: "cluster-scoped-resource"}, metav1.APIResource{Namespaced: false, Name: "cluster-scoped-resource"}, nil + } + return schema.GroupVersionResource{Resource: input.Resource}, metav1.APIResource{Namespaced: true, Name: input.Resource}, nil + }, + } + + policyWithWarns := []resourcepolicies.NamespacedFilterPolicy{ + { + Namespaces: []string{"ns1"}, + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"invalid-resource", "cluster-scoped-resource"}, + }, + }, + }, + } + resWarns, _, errWarns := resolveNamespacedFilterPolicies(policyWithWarns, mockHelper, log) + require.NoError(t, errWarns) + require.Contains(t, resWarns["ns1"].ResourceFilterMap, "invalid-resource") + require.Contains(t, resWarns["ns1"].ResourceFilterMap, "cluster-scoped-resource") +} + +func TestBackupWithResPoliciesLogs(t *testing.T) { + itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) + defer itemBlockPool.Stop() + + h := newHarness(t, itemBlockPool) + + // Add some resources so discovery helper knows about them + h.addItems(t, test.Pods(builder.ForPod("ns1", "pod-1").Result())) + h.addItems(t, test.PVs(builder.ForPersistentVolume("pv-1").Result())) + + backupReq := &Request{ + Backup: defaultBackup().ExcludedNamespaceScopedResources("pods").Result(), + SkippedPVTracker: NewSkipPVTracker(), + BackedUpItems: NewBackedUpItemsMap(), + WorkerPool: itemBlockPool, + } + + p := new(resourcepolicies.Policies) + inputPolicy := &resourcepolicies.ResourcePolicies{ + Version: "v1", + ClusterScopedFilterPolicy: &resourcepolicies.ClusterScopedFilterPolicy{ + ResourceFilters: []resourcepolicies.ResourceFilter{ + {Kinds: []string{"pods", "invalid-cluster-kind"}}, + }, + }, + NamespacedFilterPolicies: []resourcepolicies.NamespacedFilterPolicy{ + { + Namespaces: []string{"ns1"}, + ResourceFilters: []resourcepolicies.ResourceFilter{ + {Kinds: []string{"persistentvolumes", "pods", "invalid-ns-kind"}}, + }, + }, + }, + } + require.NoError(t, p.BuildPolicy(inputPolicy)) + backupReq.ResPolicies = p + + backupFile := bytes.NewBuffer([]byte{}) + err := h.backupper.Backup(h.log, backupReq, backupFile, nil, nil, nil) + require.NoError(t, err) + + // Add test to cover error returns from resolve policies + badClusterPol := &resourcepolicies.ClusterScopedFilterPolicy{ + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"pods"}, + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"invalid/label/key": "value"}}, + }, + }, + } + pBadCluster := new(resourcepolicies.Policies) + require.NoError(t, pBadCluster.BuildPolicy(&resourcepolicies.ResourcePolicies{ + Version: "v1", + ClusterScopedFilterPolicy: badClusterPol, + })) + backupReq.ResPolicies = pBadCluster + err = h.backupper.Backup(h.log, backupReq, backupFile, nil, nil, nil) + require.Error(t, err) + + badNsPol := []resourcepolicies.NamespacedFilterPolicy{ + { + Namespaces: []string{"ns1"}, + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"pods"}, + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"invalid/label/key": "value"}}, + }, + }, + }, + } + pBadNs := new(resourcepolicies.Policies) + require.NoError(t, pBadNs.BuildPolicy(&resourcepolicies.ResourcePolicies{ + Version: "v1", + NamespacedFilterPolicies: badNsPol, + })) + backupReq.ResPolicies = pBadNs + err = h.backupper.Backup(h.log, backupReq, backupFile, nil, nil, nil) + require.Error(t, err) +} + +func TestGetNamespaceFilter(t *testing.T) { + // Pre-compile our globs to simulate what resolveNamespacedFilterPolicies does + teamFrontendGlob, err := glob.Compile("team-frontend-*") + require.NoError(t, err) + + teamGlob, err := glob.Compile("team-*") + require.NoError(t, err) + + // Define our filter map + filterMap := map[string]*ResolvedNamespaceFilter{ + "exact-match-ns": {CatchAllFilter: &ResolvedResourceFilter{}}, + "team-frontend-*": {CatchAllFilter: &ResolvedResourceFilter{}}, + "team-*": {CatchAllFilter: &ResolvedResourceFilter{}}, + } + + // Create request with patterns in a specific order (first-match semantics) + req := &Request{ + NamespacedFilterMap: filterMap, + NamespacedFilterPatterns: []NamespacedFilterPattern{ + {Pattern: "team-frontend-*", Compiled: teamFrontendGlob}, // Most specific first + {Pattern: "team-*", Compiled: teamGlob}, // Broader second + }, + } + + tests := []struct { + name string + namespace string + expectNil bool + expectMatched string // The pattern or exact string that should match + }{ + { + name: "exact string match bypasses glob matching", + namespace: "exact-match-ns", + expectNil: false, + expectMatched: "exact-match-ns", + }, + { + name: "reviewer requested: glob pattern matching", + namespace: "team-backend-prod", + expectNil: false, + expectMatched: "team-*", + }, + { + name: "reviewer requested: first-match ordering", + namespace: "team-frontend-prod", + expectNil: false, + expectMatched: "team-frontend-*", // Should match this because it's first in NamespacedFilterPatterns + }, + { + name: "no match returns nil", + namespace: "unrelated-ns", + expectNil: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // First call (populates cache) + result := req.GetNamespaceFilter(tt.namespace) + + if tt.expectNil { + assert.Nil(t, result) + + // Verify negative cache + val, ok := req.NamespaceFilterCache.Load(tt.namespace) + assert.True(t, ok) + assert.Nil(t, val) + } else { + assert.NotNil(t, result) + // Ensure the returned filter points to the correct reference in our map + assert.Same(t, filterMap[tt.expectMatched], result) + + // Verify positive cache + val, ok := req.NamespaceFilterCache.Load(tt.namespace) + assert.True(t, ok) + assert.Same(t, filterMap[tt.expectMatched], val) + } + + // Second call (hits cache) + result2 := req.GetNamespaceFilter(tt.namespace) + assert.Same(t, result, result2) + }) + } +} + +func TestGetNamespaceFilter_CacheBypass(t *testing.T) { + req := &Request{ + NamespacedFilterMap: make(map[string]*ResolvedNamespaceFilter), + } + + cachedFilter := &ResolvedNamespaceFilter{} + req.NamespaceFilterCache.Store("cached-ns", cachedFilter) + + // Since NamespacedFilterMap is empty, this would normally return nil, + // but the cache should return our cachedFilter. + assert.Same(t, cachedFilter, req.GetNamespaceFilter("cached-ns")) +} diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index 2ca266e91d..f438882520 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -24,7 +24,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -142,6 +142,42 @@ func (ib *itemBackupper) itemInclusionChecks(log logrus.FieldLogger, mustInclude log.Info("Excluding item because resource is excluded") return false } + + // Per-kind name filter from ResourcePolicy namespace filter. + if namespace != "" { + if nsFilter := ib.backupRequest.GetNamespaceFilter(namespace); nsFilter != nil { + rf := nsFilter.ResourceFilterMap[groupResource.String()] + if rf == nil { + rf = nsFilter.CatchAllFilter + } + // When rf is still nil the item's kind is not listed in the namespace filter and + // there is no catch-all entry. This is an intentional permissive passthrough: + // plugin-injected additional items (returned by BackupItemAction) must be able + // to reach the archive even when their kind was not explicitly listed in + // namespacedFilterPolicies, because excluding them at Stage 2 would break backup + // completeness. For example, a CSI plugin may inject a VolumeSnapshotContent + // as an additional item that is required for a correct restore. Kind-level + // exclusion for the primary collection pass is enforced earlier in + // item_collector.go (Stage 1). + if rf != nil && rf.NameIE != nil { + if !rf.NameIE.ShouldInclude(metadata.GetName()) { + log.Infof("Excluding item: name does not match resource filter for kind %s", + groupResource) + return false + } + } + } + } else { + // Cluster-scoped resource name filter + if ib.backupRequest.ClusterScopedFilterMap != nil { + if rf, ok := ib.backupRequest.ClusterScopedFilterMap[groupResource.String()]; ok && rf.NameIE != nil { + if !rf.NameIE.ShouldInclude(metadata.GetName()) { + log.Infof("Excluding item: name does not match clusterScopedFilterPolicy for kind %s", groupResource) + return false + } + } + } + } } if metadata.GetDeletionTimestamp() != nil { diff --git a/pkg/backup/item_backupper_test.go b/pkg/backup/item_backupper_test.go index be91b6d344..f3769a998d 100644 --- a/pkg/backup/item_backupper_test.go +++ b/pkg/backup/item_backupper_test.go @@ -21,20 +21,20 @@ import ( "testing" "github.com/sirupsen/logrus" - "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/runtime/schema" - ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake" - - "github.com/vmware-tanzu/velero/internal/resourcepolicies" - "github.com/vmware-tanzu/velero/pkg/kuberesource" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" + "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/util/collections" ) func Test_resourceKey(t *testing.T) { @@ -494,3 +494,284 @@ func TestUnTrackSkippedPV_PendingLostPVC(t *testing.T) { }) } } + +// includeAllIE is a minimal IncludesExcludesInterface that includes everything — +// used in tests where the global resource include/exclude logic is not under test. +type includeAllIE struct{} + +func (includeAllIE) ShouldInclude(string) bool { return true } +func (includeAllIE) ShouldExclude(string) bool { return false } + +// makeTestUnstructured creates an unstructured object with the given namespace, name, and labels. +func makeTestUnstructured(namespace, name string, labels map[string]string) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetNamespace(namespace) + obj.SetName(name) + if labels != nil { + obj.SetLabels(labels) + } + return obj +} + +// makeNameIE creates an IncludesExcludes that includes only the given glob patterns. +func makeNameIE(include ...string) *collections.IncludesExcludes { + ie := collections.NewIncludesExcludes() + ie.Includes(include...) + return ie +} + +// newTestItemBackupper builds a minimal itemBackupper suitable for itemInclusionChecks tests. +func newTestItemBackupper(req *Request) *itemBackupper { + return &itemBackupper{ + backupRequest: req, + } +} + +// baseRequest returns a Request with NamespaceIncludesExcludes and ResourceIncludesExcludes +// configured to include everything, so only the filter-map logic under test is exercised. +func baseRequest() *Request { + return &Request{ + Backup: builder.ForBackup("velero", "test-backup").Result(), + NamespaceIncludesExcludes: collections.NewNamespaceIncludesExcludes().Includes("*"), + ResourceIncludesExcludes: includeAllIE{}, + SkippedPVTracker: NewSkipPVTracker(), + } +} + +var configMapsGR = schema.GroupResource{Group: "", Resource: "configmaps"} +var clusterRolesGR = schema.GroupResource{Group: "rbac.authorization.k8s.io", Resource: "clusterroles"} + +// TestItemInclusionChecks_ExcludeLabel_OverridesNamespaceFilter verifies that +// velero.io/exclude-from-backup=true takes precedence over a namespacedFilterPolicies +// entry that would otherwise include the resource. +func TestItemInclusionChecks_ExcludeLabel_OverridesNamespaceFilter(t *testing.T) { + req := baseRequest() + req.NamespacedFilterMap = map[string]*ResolvedNamespaceFilter{ + "ns-a": { + ResourceFilterMap: map[string]*ResolvedResourceFilter{ + configMapsGR.String(): {}, // include all ConfigMaps in ns-a + }, + }, + } + req.NamespacedFilterPatterns = []NamespacedFilterPattern{} + + ib := newTestItemBackupper(req) + log := logrus.New() + + obj := makeTestUnstructured("ns-a", "my-config", map[string]string{ + velerov1api.ExcludeFromBackupLabel: "true", + }) + + result := ib.itemInclusionChecks(log, false, obj, obj, configMapsGR) + assert.False(t, result, "resource with exclude-from-backup=true must be excluded even when matched by namespacedFilterPolicies") +} + +// TestItemInclusionChecks_ExcludeLabel_OverridesCatchAll verifies that +// velero.io/exclude-from-backup=true takes precedence over the catch-all filter. +func TestItemInclusionChecks_ExcludeLabel_OverridesCatchAll(t *testing.T) { + catchAllFilter := &ResolvedResourceFilter{} // include everything via catch-all + req := baseRequest() + req.NamespacedFilterMap = map[string]*ResolvedNamespaceFilter{ + "ns-a": { + ResourceFilterMap: map[string]*ResolvedResourceFilter{}, + CatchAllFilter: catchAllFilter, + }, + } + req.NamespacedFilterPatterns = []NamespacedFilterPattern{} + + ib := newTestItemBackupper(req) + log := logrus.New() + + obj := makeTestUnstructured("ns-a", "my-config", map[string]string{ + velerov1api.ExcludeFromBackupLabel: "true", + }) + + result := ib.itemInclusionChecks(log, false, obj, obj, configMapsGR) + assert.False(t, result, "resource with exclude-from-backup=true must be excluded even when matched by catch-all filter") +} + +// TestItemInclusionChecks_ExcludeLabel_OverridesClusterScopedFilter verifies that +// velero.io/exclude-from-backup=true takes precedence over clusterScopedFilterPolicy. +func TestItemInclusionChecks_ExcludeLabel_OverridesClusterScopedFilter(t *testing.T) { + req := baseRequest() + req.ClusterScopedFilterMap = map[string]*ResolvedResourceFilter{ + clusterRolesGR.String(): {}, // include all ClusterRoles + } + + ib := newTestItemBackupper(req) + log := logrus.New() + + // Cluster-scoped object: no namespace + obj := makeTestUnstructured("", "my-role", map[string]string{ + velerov1api.ExcludeFromBackupLabel: "true", + }) + + result := ib.itemInclusionChecks(log, false, obj, obj, clusterRolesGR) + assert.False(t, result, "cluster-scoped resource with exclude-from-backup=true must be excluded even when in clusterScopedFilterPolicy") +} + +// TestItemInclusionChecks_ClusterScoped_NotInFilterMap_PassesThrough verifies that +// a dynamically injected cluster-scoped resource NOT listed in ClusterScopedFilterMap +// passes through itemInclusionChecks (permissive passthrough at Stage 2). +func TestItemInclusionChecks_ClusterScoped_NotInFilterMap_PassesThrough(t *testing.T) { + req := baseRequest() + req.ClusterScopedFilterMap = map[string]*ResolvedResourceFilter{ + clusterRolesGR.String(): {}, // only ClusterRoles are listed + } + + ib := newTestItemBackupper(req) + log := logrus.New() + + // VolumeSnapshotClass is NOT in the filter map + volumeSnapshotClassGR := schema.GroupResource{Group: "snapshot.storage.k8s.io", Resource: "volumesnapshotclasses"} + obj := makeTestUnstructured("", "standard", nil) + + result := ib.itemInclusionChecks(log, false, obj, obj, volumeSnapshotClassGR) + assert.True(t, result, "cluster-scoped resource not in ClusterScopedFilterMap must pass through (permissive Stage 2 for unlisted kinds)") +} + +// TestItemInclusionChecks_ClusterScoped_NameIE_Matching verifies that a cluster-scoped +// resource listed in ClusterScopedFilterMap with a NameIE filter is included/excluded +// based on its name. +func TestItemInclusionChecks_ClusterScoped_NameIE_Matching(t *testing.T) { + req := baseRequest() + req.ClusterScopedFilterMap = map[string]*ResolvedResourceFilter{ + clusterRolesGR.String(): { + NameIE: makeNameIE("my-app-*"), + }, + } + + ib := newTestItemBackupper(req) + log := logrus.New() + + // Matching name + matching := makeTestUnstructured("", "my-app-reader", nil) + assert.True(t, ib.itemInclusionChecks(log, false, matching, matching, clusterRolesGR), + "ClusterRole matching name pattern must be included") + + // Non-matching name + nonMatching := makeTestUnstructured("", "other-role", nil) + assert.False(t, ib.itemInclusionChecks(log, false, nonMatching, nonMatching, clusterRolesGR), + "ClusterRole not matching name pattern must be excluded") +} + +// TestItemInclusionChecks_GlobalExclusion_OverridesNamespaceFilter verifies that +// a resource kind globally excluded by includeExcludePolicy is rejected at Stage 2 +// even when a namespacedFilterPolicies entry lists that kind. The global +// ResourceIncludesExcludes.ShouldInclude check fires before the per-namespace filter. +func TestItemInclusionChecks_GlobalExclusion_OverridesNamespaceFilter(t *testing.T) { + // excludeSecretsIE excludes "secrets" globally, includes everything else. + excludeSecretsIE := &excludeResourceIE{excluded: "secrets"} + + req := &Request{ + Backup: builder.ForBackup("velero", "test-backup").Result(), + NamespaceIncludesExcludes: collections.NewNamespaceIncludesExcludes().Includes("*"), + ResourceIncludesExcludes: excludeSecretsIE, + SkippedPVTracker: NewSkipPVTracker(), + // namespacedFilterPolicies says to back up Secrets in ns-a + NamespacedFilterMap: map[string]*ResolvedNamespaceFilter{ + "ns-a": { + ResourceFilterMap: map[string]*ResolvedResourceFilter{ + "secrets.": {}, // Secret listed in per-namespace filter + }, + }, + }, + NamespacedFilterPatterns: []NamespacedFilterPattern{}, + } + + ib := newTestItemBackupper(req) + log := logrus.New() + + secretsGR := schema.GroupResource{Group: "", Resource: "secrets"} + obj := makeTestUnstructured("ns-a", "my-secret", nil) + + result := ib.itemInclusionChecks(log, false, obj, obj, secretsGR) + assert.False(t, result, + "Secret must be excluded because it is globally excluded by ResourceIncludesExcludes, "+ + "even though namespacedFilterPolicies lists it") +} + +// TestItemInclusionChecks_PluginItem_UnlistedKind_NoCatchAll_PassesThrough verifies the +// intentional permissive passthrough at Stage 2 for plugin-injected additional items. +// When a namespace has a namespacedFilterPolicies entry but the item's kind is not listed +// in that policy and there is no catch-all entry, itemInclusionChecks must still allow +// the item through. +// +// Rationale: plugin-injected additional items (returned by BackupItemAction) must be able +// to reach the archive even when their kind was not explicitly listed in the filter policy, +// because rejecting them here would break backup completeness. For example, a CSI plugin +// may inject a VolumeSnapshotContent that is required for a correct restore. +// Kind-level exclusion for the primary collection pass is enforced at Stage 1 in +// item_collector.go, not at Stage 2 here. +func TestItemInclusionChecks_PluginItem_UnlistedKind_NoCatchAll_PassesThrough(t *testing.T) { + req := baseRequest() + // Namespace filter only lists ConfigMaps; Secrets are not listed and there is no catch-all. + req.NamespacedFilterMap = map[string]*ResolvedNamespaceFilter{ + "ns-a": { + ResourceFilterMap: map[string]*ResolvedResourceFilter{ + configMapsGR.String(): {}, + }, + CatchAllFilter: nil, + }, + } + req.NamespacedFilterPatterns = []NamespacedFilterPattern{} + + ib := newTestItemBackupper(req) + log := logrus.New() + + secretsGR := schema.GroupResource{Group: "", Resource: "secrets"} + obj := makeTestUnstructured("ns-a", "plugin-injected-secret", nil) + + result := ib.itemInclusionChecks(log, false, obj, obj, secretsGR) + assert.True(t, result, + "plugin-injected additional item of an unlisted kind must pass through Stage 2 "+ + "even when its namespace has a namespacedFilterPolicies entry with no catch-all; "+ + "kind exclusion is enforced at Stage 1 (item_collector.go), not here") +} + +// TestItemInclusionChecks_PluginItem_UnlistedKind_WithCatchAll_PassesThrough verifies that +// a plugin-injected additional item of a kind not listed in the namespace filter also passes +// through Stage 2 when a catch-all entry is present. The catch-all is validated to never +// carry a NameIE (names/excludedNames are prohibited on catch-all entries), so the name +// check is always a no-op for catch-all-matched items and the item is included. +func TestItemInclusionChecks_PluginItem_UnlistedKind_WithCatchAll_PassesThrough(t *testing.T) { + req := baseRequest() + // Namespace filter lists ConfigMaps explicitly; a catch-all covers everything else. + // The catch-all has no NameIE — this is enforced by validation. + req.NamespacedFilterMap = map[string]*ResolvedNamespaceFilter{ + "ns-a": { + ResourceFilterMap: map[string]*ResolvedResourceFilter{ + configMapsGR.String(): {}, + }, + CatchAllFilter: &ResolvedResourceFilter{ + // NameIE intentionally nil: validation forbids names/excludedNames on catch-all + NameIE: nil, + }, + }, + } + req.NamespacedFilterPatterns = []NamespacedFilterPattern{} + + ib := newTestItemBackupper(req) + log := logrus.New() + + secretsGR := schema.GroupResource{Group: "", Resource: "secrets"} + obj := makeTestUnstructured("ns-a", "plugin-injected-secret", nil) + + result := ib.itemInclusionChecks(log, false, obj, obj, secretsGR) + assert.True(t, result, + "plugin-injected additional item matched by catch-all must pass through Stage 2; "+ + "the catch-all has no NameIE so the name check is a no-op") +} + +// excludeResourceIE is an IncludesExcludesInterface that excludes a single resource +// type and includes everything else. Used to simulate includeExcludePolicy global exclusions. +type excludeResourceIE struct { + excluded string +} + +func (e *excludeResourceIE) ShouldInclude(typeName string) bool { + return typeName != e.excluded +} +func (e *excludeResourceIE) ShouldExclude(typeName string) bool { + return typeName == e.excluded +} diff --git a/pkg/backup/item_collector.go b/pkg/backup/item_collector.go index 15efca2db6..4c3d6e275d 100644 --- a/pkg/backup/item_collector.go +++ b/pkg/backup/item_collector.go @@ -24,7 +24,7 @@ import ( "sort" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -462,6 +462,7 @@ func (r *itemCollector) getResourceItems( } clusterScoped := !resource.Namespaced + namespacesToList := getNamespacesToList(r.backupRequest.NamespaceIncludesExcludes) // If we get here, we're backing up something other than namespaces @@ -472,6 +473,16 @@ func (r *itemCollector) getResourceItems( var items []*kubernetesResource for _, namespace := range namespacesToList { + // Check per-namespace resource type filter from ResourcePolicy + if nsFilter := r.backupRequest.GetNamespaceFilter(namespace); nsFilter != nil { + _, hasSpecific := nsFilter.ResourceFilterMap[gr.String()] + if !hasSpecific && nsFilter.CatchAllFilter == nil { + log.Debugf("Skipping resource %s in namespace %s: not in resourceFilters", + gr, namespace) + continue + } + } + unstructuredItems, err := r.listResourceByLabelsPerNamespace( namespace, gr, gv, resource, log) if err != nil { @@ -528,13 +539,55 @@ func (r *itemCollector) listResourceByLabelsPerNamespace( return nil, err } + // 1. Start with global selectors (existing default behavior) var orLabelSelectors []string + var labelSelector string + if r.backupRequest.Spec.OrLabelSelectors != nil { for _, s := range r.backupRequest.Spec.OrLabelSelectors { orLabelSelectors = append(orLabelSelectors, metav1.FormatLabelSelector(s)) } - } else { - orLabelSelectors = []string{} + } + if selector := r.backupRequest.Spec.LabelSelector; selector != nil { + labelSelector = metav1.FormatLabelSelector(selector) + } + + // 2. Apply fine-grained filter overrides if applicable + if !resource.Namespaced && r.backupRequest.ClusterScopedFilterMap != nil { + if rf := r.backupRequest.ClusterScopedFilterMap[gr.String()]; rf != nil { + // Overwrite global selectors with specific filter + orLabelSelectors = nil + labelSelector = "" + if rf.LabelSelector != nil { + labelSelector = rf.LabelSelector.String() + } + for _, s := range rf.OrLabelSelectors { + orLabelSelectors = append(orLabelSelectors, s.String()) + } + } + // ClusterScopedFilterPolicy: If rf == nil, it intentionally falls back to the global selectors initialized above + } else if nsFilter := r.backupRequest.GetNamespaceFilter(namespace); nsFilter != nil { + rf := nsFilter.ResourceFilterMap[gr.String()] + if rf == nil { + rf = nsFilter.CatchAllFilter + } + + if rf != nil { + // Overwrite global selectors with specific filter + orLabelSelectors = nil + labelSelector = "" + if rf.LabelSelector != nil { + labelSelector = rf.LabelSelector.String() + } + for _, s := range rf.OrLabelSelectors { + orLabelSelectors = append(orLabelSelectors, s.String()) + } + } else { + // NamespacedFilterPolicies: namespacedFilterPolicies acts as an exclusive allowlist. + // If neither a kind-specific entry nor a catch-all entry exists, skip the kind. + logger.Debug("Skipping resource kind for namespace as it is not present in the namespace filter policy") + return nil, nil + } } logger.Info("Listing items") @@ -554,11 +607,6 @@ func (r *itemCollector) listResourceByLabelsPerNamespace( return nil, err } - var labelSelector string - if selector := r.backupRequest.Spec.LabelSelector; selector != nil { - labelSelector = metav1.FormatLabelSelector(selector) - } - // Listing items for labelSelector (singular) if len(orLabelSelectors) == 0 { unstructuredItems, err = r.listItemsForLabel( diff --git a/pkg/backup/item_collector_test.go b/pkg/backup/item_collector_test.go index 54e2ed4c3f..084d5b5ff1 100644 --- a/pkg/backup/item_collector_test.go +++ b/pkg/backup/item_collector_test.go @@ -26,7 +26,9 @@ import ( corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" @@ -279,8 +281,9 @@ func TestItemCollectorBackupNamespaces(t *testing.T) { Backup: tc.backup, NamespaceIncludesExcludes: tc.ie, }, - dynamicFactory: factory, - dir: tempDir, + dynamicFactory: factory, + discoveryHelper: test.NewFakeDiscoveryHelper(true, nil), + dir: tempDir, } if tc.converter == nil { @@ -305,3 +308,140 @@ func TestItemCollectorBackupNamespaces(t *testing.T) { }) } } + +// TestNamespacedFilterMap_GlobalExclusionPrecedence verifies the precedence rule: +// ResourceIncludesExcludes (set by includeExcludePolicy) is checked before the +// NamespacedFilterMap. This is enforced at both Stage 1 (item_collector.go line ~430) +// and Stage 2 (item_backupper.go itemInclusionChecks). The unit below confirms that +// GetNamespaceFilter still returns a filter for the namespace — it is the caller's +// responsibility to check ResourceIncludesExcludes first, which item_collector does. +// +// Full coverage of the Stage 2 enforcement is in item_backupper_test.go +// TestItemInclusionChecks_GlobalExclusion_OverridesNamespaceFilter. +func TestNamespacedFilterMap_GlobalExclusionPrecedence(t *testing.T) { + req := &Request{ + Backup: builder.ForBackup("velero", "test-backup").Result(), + NamespaceIncludesExcludes: collections.NewNamespaceIncludesExcludes().Includes("ns-a"), + NamespacedFilterMap: map[string]*ResolvedNamespaceFilter{ + "ns-a": { + ResourceFilterMap: map[string]*ResolvedResourceFilter{ + "secrets.": {}, + }, + }, + }, + NamespacedFilterPatterns: []NamespacedFilterPattern{}, + } + + // GetNamespaceFilter returns the filter regardless of global exclusions. + // The caller (item_collector) is responsible for checking ResourceIncludesExcludes first. + nsFilter := req.GetNamespaceFilter("ns-a") + require.NotNil(t, nsFilter, "GetNamespaceFilter should return a filter for ns-a") + _, hasSecrets := nsFilter.ResourceFilterMap["secrets."] + assert.True(t, hasSecrets, "ns-a filter should list secrets GR") + + // When a global excludeAllIE is set, item_collector would return nil before consulting the map. + // This is verified by the Stage 1 check: ShouldInclude("secrets.") == false → skip. + ie := &excludeAllIE{} + assert.False(t, ie.ShouldInclude("secrets."), + "global exclusion must reject secrets before the per-namespace filter is consulted") +} + +// excludeAllIE is an IncludesExcludesInterface that excludes every resource kind. +type excludeAllIE struct{} + +func (excludeAllIE) ShouldInclude(string) bool { return false } +func (excludeAllIE) ShouldExclude(string) bool { return true } + +func TestGetResourceItems(t *testing.T) { + tests := []struct { + name string + namespaces []string + clusterScopedFilterMap map[string]*ResolvedResourceFilter + namespacedFilterMap map[string]*ResolvedNamespaceFilter + resource metav1.APIResource + gr schema.GroupResource + }{ + { + name: "cluster scoped resource with filter", + namespaces: []string{""}, + resource: metav1.APIResource{ + Name: "persistentvolumes", + Namespaced: false, + }, + gr: schema.GroupResource{Resource: "persistentvolumes"}, + clusterScopedFilterMap: map[string]*ResolvedResourceFilter{ + "persistentvolumes": { + LabelSelector: labels.Set{"app": "foo"}.AsSelector(), + }, + }, + }, + { + name: "namespace scoped resource with filter", + namespaces: []string{"ns1"}, + resource: metav1.APIResource{ + Name: "pods", + Namespaced: true, + }, + gr: schema.GroupResource{Resource: "pods"}, + namespacedFilterMap: map[string]*ResolvedNamespaceFilter{ + "ns1": { + ResourceFilterMap: map[string]*ResolvedResourceFilter{ + "pods": { + LabelSelector: labels.Set{"app": "bar"}.AsSelector(), + }, + }, + }, + }, + }, + { + name: "namespace scoped resource skipped due to no filter match", + namespaces: []string{"ns1"}, + resource: metav1.APIResource{ + Name: "secrets", + Namespaced: true, + }, + gr: schema.GroupResource{Resource: "secrets"}, + namespacedFilterMap: map[string]*ResolvedNamespaceFilter{ + "ns1": { + ResourceFilterMap: map[string]*ResolvedResourceFilter{ + "pods": { + LabelSelector: labels.Set{"app": "bar"}.AsSelector(), + }, + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dc := &test.FakeDynamicClient{} + dc.On("List", mock.Anything).Return(&unstructured.UnstructuredList{}, nil) + + factory := &test.FakeDynamicFactory{} + factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil) + + req := &Request{ + Backup: builder.ForBackup("velero", "backup").Result(), + ClusterScopedFilterMap: tc.clusterScopedFilterMap, + NamespacedFilterMap: tc.namespacedFilterMap, + ResourceIncludesExcludes: includeAllIE{}, + } + if len(tc.namespaces) > 0 && tc.namespaces[0] != "" { + req.NamespaceIncludesExcludes = collections.NewNamespaceIncludesExcludes().Includes(tc.namespaces...) + } else { + req.NamespaceIncludesExcludes = collections.NewNamespaceIncludesExcludes().Includes("*") + } + + r := &itemCollector{ + backupRequest: req, + dynamicFactory: factory, + discoveryHelper: test.NewFakeDiscoveryHelper(true, nil), + log: test.NewLogger(), + } + + _, err := r.getResourceItems(test.NewLogger(), schema.GroupVersion{}, tc.resource, nil) + assert.NoError(t, err) + }) + } +} diff --git a/pkg/backup/itemblock.go b/pkg/backup/itemblock.go index dee553f721..4619e23aa8 100644 --- a/pkg/backup/itemblock.go +++ b/pkg/backup/itemblock.go @@ -20,7 +20,7 @@ import ( "encoding/json" "os" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/backup/request.go b/pkg/backup/request.go index eb9edcbe8b..7ace381255 100644 --- a/pkg/backup/request.go +++ b/pkg/backup/request.go @@ -19,6 +19,9 @@ package backup import ( "sync" + "github.com/gobwas/glob" + "k8s.io/apimachinery/pkg/labels" + "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" @@ -34,6 +37,21 @@ type itemKey struct { name string } +// ResolvedResourceFilter holds the materialized filter state for one kind-group +// within a namespace. +type ResolvedResourceFilter struct { + LabelSelector labels.Selector + OrLabelSelectors []labels.Selector + NameIE *collections.IncludesExcludes +} + +// ResolvedNamespaceFilter holds the materialized filter state for a namespace. +// ResourceFilterMap is keyed by the resolved group-resource string. +type ResolvedNamespaceFilter struct { + ResourceFilterMap map[string]*ResolvedResourceFilter + CatchAllFilter *ResolvedResourceFilter +} + type SynchronizedVSList struct { sync.Mutex VolumeSnapshotList []*volume.Snapshot @@ -70,6 +88,31 @@ type Request struct { SkippedPVTracker *skipPVTracker VolumesInformation volume.BackupVolumesInformation WorkerPool *ItemBlockWorkerPool + + // ClusterScopedFilterMap holds resolved global filters for cluster-scoped resources. + // Key is the resolved group-resource string. + ClusterScopedFilterMap map[string]*ResolvedResourceFilter + + // NamespacedFilterMap holds resolved per-namespace filters. + // Key is either an exact namespace name or a glob pattern. + NamespacedFilterMap map[string]*ResolvedNamespaceFilter + + // NamespacedFilterPatterns preserves the order of patterns for first-match semantics + // and caches pre-compiled globs to avoid repeated compilation in the hot path. + NamespacedFilterPatterns []NamespacedFilterPattern + + // NamespaceFilterCache memoizes the resolved filter for a given namespace. + // sync.Map is used because item backuppers access this concurrently. + NamespaceFilterCache sync.Map +} + +// NamespacedFilterPattern pairs a namespace pattern string with its pre-compiled +// glob so that GetNamespaceFilter does not recompile on every call. +// Compiled is nil for exact-match (non-glob) patterns, which are looked up +// directly in NamespacedFilterMap. +type NamespacedFilterPattern struct { + Pattern string + Compiled glob.Glob } // BackupVolumesInformation contains the information needs by generating @@ -107,3 +150,40 @@ func (r *Request) FillVolumesInformation() { func (r *Request) StopWorkerPool() { r.WorkerPool.Stop() } + +// GetNamespaceFilter returns the resolved filter for a namespace, or nil +// if the namespace should use global filters. Uses first-match semantics +// when multiple patterns could match the same namespace, but exact matches +// always take precedence over glob patterns regardless of definition order. +func (r *Request) GetNamespaceFilter(namespace string) *ResolvedNamespaceFilter { + if r.NamespacedFilterMap == nil { + return nil + } + + // 1. Check the concurrent cache first + if val, ok := r.NamespaceFilterCache.Load(namespace); ok { + if val == nil { + return nil + } + return val.(*ResolvedNamespaceFilter) + } + + // 2. Check for exact match first + if f, ok := r.NamespacedFilterMap[namespace]; ok { + r.NamespaceFilterCache.Store(namespace, f) + return f + } + + // 3. Walk patterns in definition order using pre-compiled globs + for _, p := range r.NamespacedFilterPatterns { + if p.Compiled != nil && p.Compiled.Match(namespace) { + filter := r.NamespacedFilterMap[p.Pattern] + r.NamespaceFilterCache.Store(namespace, filter) + return filter + } + } + + // 4. Cache the miss + r.NamespaceFilterCache.Store(namespace, nil) + return nil +} diff --git a/pkg/builder/container_builder.go b/pkg/builder/container_builder.go index 762462c864..e250026298 100644 --- a/pkg/builder/container_builder.go +++ b/pkg/builder/container_builder.go @@ -18,10 +18,12 @@ package builder import ( "encoding/json" + "fmt" "strings" corev1api "k8s.io/api/core/v1" apimachineryRuntime "k8s.io/apimachinery/pkg/runtime" + utilrand "k8s.io/apimachinery/pkg/util/rand" "github.com/vmware-tanzu/velero/pkg/label" ) @@ -42,15 +44,22 @@ func ForContainer(name, image string) *ContainerBuilder { } // ForPluginContainer is a helper builder specifically for plugin init containers -func ForPluginContainer(image string, pullPolicy corev1api.PullPolicy) *ContainerBuilder { +func ForPluginContainer(image string, pullPolicy corev1api.PullPolicy, existingContainers []corev1api.Container) *ContainerBuilder { volumeMount := ForVolumeMount("plugins", "/target").Result() - return ForContainer(getName(image), image).PullPolicy(pullPolicy).VolumeMounts(volumeMount) + return ForContainer(getName(image, existingContainers), image).PullPolicy(pullPolicy).VolumeMounts(volumeMount) } // getName returns the 'name' component of a docker image that includes the entire string // except the registry name, and transforms the combined string into a DNS-1123 compatible name // that fits within the 63-character limit for Kubernetes container names. -func getName(image string) string { +// It appends a random string if there is a collision with existing container names. +func getName(image string, existingContainers []corev1api.Container) string { + // Convert existingContainers to a map for O(1) collision lookups + existingNames := make(map[string]bool, len(existingContainers)) + for _, c := range existingContainers { + existingNames[c.Name] = true + } + slashIndex := strings.Index(image, "/") slashCount := 0 if slashIndex >= 0 { @@ -88,7 +97,20 @@ func getName(image string) string { name := re.Replace(image[start:end]) // Ensure the name doesn't exceed Kubernetes container name length limit - return label.GetValidName(name) + name = label.GetValidName(name) + + for existingNames[name] { + name = re.Replace(image[start:end]) + if len(name) > 57 { + // Leave 6 characters for "-xxxxx" random string + name = name[:57] + name = strings.TrimSuffix(name, "-") + } + name = fmt.Sprintf("%s-%s", name, utilrand.String(5)) + name = label.GetValidName(name) + } + + return name } // Result returns the built Container. diff --git a/pkg/builder/container_builder_test.go b/pkg/builder/container_builder_test.go index b23cbddfd5..e0af71f753 100644 --- a/pkg/builder/container_builder_test.go +++ b/pkg/builder/container_builder_test.go @@ -16,16 +16,19 @@ limitations under the License. package builder import ( + "strings" "testing" "github.com/stretchr/testify/assert" + corev1api "k8s.io/api/core/v1" ) func TestGetName(t *testing.T) { tests := []struct { - name string - image string - expected string + name string + image string + existingContainers []corev1api.Container + expected string }{ { name: "image name with registry hostname and tag", @@ -92,11 +95,25 @@ func TestGetName(t *testing.T) { image: "quay.io/vmware-tanzu/velero@sha256:a75f9e8c3ced3943515f249597be389f8233e1258d289b11184796edceaa7dab", expected: "vmware-tanzu-velero", }, + { + name: "duplicate plugin name", + image: "gcr.io/my-repo/my-image:latest", + existingContainers: []corev1api.Container{ + {Name: "my-repo-my-image"}, + }, + expected: "my-repo-my-image-", // we will check it has the prefix + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, getName(test.image)) + if test.name == "duplicate plugin name" { + result := getName(test.image, test.existingContainers) + assert.True(t, strings.HasPrefix(result, test.expected), "expected prefix %s in %s", test.expected, result) + assert.Len(t, result, len(test.expected)+5) + } else { + assert.Equal(t, test.expected, getName(test.image, test.existingContainers)) + } }) } } @@ -117,7 +134,7 @@ func TestGetNameWithLongPaths(t *testing.T) { // Should be exactly 63 characters (truncated with hash) assert.Len(t, result, 63) // Should be deterministic - result2 := getName("arohcpsvcdev.azurecr.io/redhat-user-workloads/ocp-art-tenant/oadp-hypershift-oadp-plugin-main@sha256:adb840bf3890b4904a8cdda1a74c82cf8d96c52eba9944ac10e795335d6fd450") + result2 := getName("arohcpsvcdev.azurecr.io/redhat-user-workloads/ocp-art-tenant/oadp-hypershift-oadp-plugin-main@sha256:adb840bf3890b4904a8cdda1a74c82cf8d96c52eba9944ac10e795335d6fd450", nil) assert.Equal(t, result, result2) }, }, @@ -142,7 +159,7 @@ func TestGetNameWithLongPaths(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - result := getName(test.image) + result := getName(test.image, nil) test.validate(t, result) }) } diff --git a/pkg/builder/restore_builder.go b/pkg/builder/restore_builder.go index bad4327e99..22e880a98d 100644 --- a/pkg/builder/restore_builder.go +++ b/pkg/builder/restore_builder.go @@ -19,6 +19,7 @@ package builder import ( "time" + corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -171,3 +172,12 @@ func (b *RestoreBuilder) ItemOperationTimeout(timeout time.Duration) *RestoreBui b.object.Spec.ItemOperationTimeout.Duration = timeout return b } + +// ResourcePoliciesConfigmap sets the Restore's resource policies configmap. +func (b *RestoreBuilder) ResourcePoliciesConfigmap(name string) *RestoreBuilder { + b.object.Spec.ResourcePolicy = &corev1api.TypedLocalObjectReference{ + Kind: "configmap", + Name: name, + } + return b +} diff --git a/pkg/builder/restore_builder_test.go b/pkg/builder/restore_builder_test.go new file mode 100644 index 0000000000..b45bd80c61 --- /dev/null +++ b/pkg/builder/restore_builder_test.go @@ -0,0 +1,36 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRestoreBuilder_ResourcePoliciesConfigmap(t *testing.T) { + restore := ForRestore("velero", "my-restore"). + ResourcePoliciesConfigmap("my-policy-cm"). + Result() + + assert.Equal(t, "velero", restore.Namespace) + assert.Equal(t, "my-restore", restore.Name) + assert.NotNil(t, restore.Spec.ResourcePolicy) + assert.Equal(t, "configmap", restore.Spec.ResourcePolicy.Kind) + assert.Equal(t, "my-policy-cm", restore.Spec.ResourcePolicy.Name) + assert.Equal(t, (*string)(nil), restore.Spec.ResourcePolicy.APIGroup) +} diff --git a/pkg/client/client.go b/pkg/client/client.go index dccdc05fcf..39cdc91416 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -20,7 +20,7 @@ import ( "fmt" "runtime" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" diff --git a/pkg/client/config.go b/pkg/client/config.go index 687c303e7b..2a96e3467b 100644 --- a/pkg/client/config.go +++ b/pkg/client/config.go @@ -23,7 +23,7 @@ import ( "strconv" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" ) const ( diff --git a/pkg/client/factory.go b/pkg/client/factory.go index 51dfb62c3c..17e2a243a2 100644 --- a/pkg/client/factory.go +++ b/pkg/client/factory.go @@ -27,8 +27,8 @@ import ( k8scheme "k8s.io/client-go/kubernetes/scheme" kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" diff --git a/pkg/cmd/cli/backup/delete.go b/pkg/cmd/cli/backup/delete.go index 692b82dbf0..f4eaf1b836 100644 --- a/pkg/cmd/cli/backup/delete.go +++ b/pkg/cmd/cli/backup/delete.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/cmd/cli/backup/download.go b/pkg/cmd/cli/backup/download.go index 8bb973ff00..e4afd216c0 100644 --- a/pkg/cmd/cli/backup/download.go +++ b/pkg/cmd/cli/backup/download.go @@ -23,7 +23,7 @@ import ( "path/filepath" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" controllerclient "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/cmd/cli/backuplocation/create.go b/pkg/cmd/cli/backuplocation/create.go index 343bc790af..391c7b3760 100644 --- a/pkg/cmd/cli/backuplocation/create.go +++ b/pkg/cmd/cli/backuplocation/create.go @@ -24,7 +24,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/cmd/cli/backuplocation/delete.go b/pkg/cmd/cli/backuplocation/delete.go index f2c3bcc3df..9c1e60507e 100644 --- a/pkg/cmd/cli/backuplocation/delete.go +++ b/pkg/cmd/cli/backuplocation/delete.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/cmd/cli/backuplocation/set.go b/pkg/cmd/cli/backuplocation/set.go index 8aa018e292..c1b52e536b 100644 --- a/pkg/cmd/cli/backuplocation/set.go +++ b/pkg/cmd/cli/backuplocation/set.go @@ -22,7 +22,7 @@ import ( "os" "path/filepath" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" diff --git a/pkg/cmd/cli/datamover/backup.go b/pkg/cmd/cli/datamover/backup.go index 2b647bb24e..b548d40385 100644 --- a/pkg/cmd/cli/datamover/backup.go +++ b/pkg/cmd/cli/datamover/backup.go @@ -21,7 +21,7 @@ import ( "time" "github.com/bombsimon/logrusr/v3" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" corev1api "k8s.io/api/core/v1" diff --git a/pkg/cmd/cli/datamover/restore.go b/pkg/cmd/cli/datamover/restore.go index b2efdbc340..1d3cf84f40 100644 --- a/pkg/cmd/cli/datamover/restore.go +++ b/pkg/cmd/cli/datamover/restore.go @@ -21,7 +21,7 @@ import ( "time" "github.com/bombsimon/logrusr/v3" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" corev1api "k8s.io/api/core/v1" diff --git a/pkg/cmd/cli/debug/debug.go b/pkg/cmd/cli/debug/debug.go index 1d511979de..fac49d6223 100644 --- a/pkg/cmd/cli/debug/debug.go +++ b/pkg/cmd/cli/debug/debug.go @@ -25,7 +25,7 @@ import ( "path/filepath" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/pflag" diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 81e2df126e..0df53eb32b 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -23,7 +23,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/cmd/cli/nodeagent/server.go b/pkg/cmd/cli/nodeagent/server.go index 7e7c86e6c0..f7bff536bc 100644 --- a/pkg/cmd/cli/nodeagent/server.go +++ b/pkg/cmd/cli/nodeagent/server.go @@ -26,8 +26,8 @@ import ( "time" "github.com/bombsimon/logrusr/v3" + "github.com/cockroachdb/errors" snapshotv1client "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" "github.com/spf13/cobra" diff --git a/pkg/cmd/cli/plugin/add.go b/pkg/cmd/cli/plugin/add.go index 9ea199fa9a..553a217dc9 100644 --- a/pkg/cmd/cli/plugin/add.go +++ b/pkg/cmd/cli/plugin/add.go @@ -22,8 +22,8 @@ import ( "fmt" "strings" + "github.com/cockroachdb/errors" jsonpatch "github.com/evanphx/json-patch/v5" - "github.com/pkg/errors" "github.com/spf13/cobra" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -111,7 +111,7 @@ func NewAddCommand(f client.Factory) *cobra.Command { } // add the plugin as an init container - plugin := *builder.ForPluginContainer(args[0], corev1api.PullPolicy(imagePullPolicyFlag.String())).Result() + plugin := *builder.ForPluginContainer(args[0], corev1api.PullPolicy(imagePullPolicyFlag.String()), veleroDeploy.Spec.Template.Spec.InitContainers).Result() veleroDeploy.Spec.Template.Spec.InitContainers = append(veleroDeploy.Spec.Template.Spec.InitContainers, plugin) diff --git a/pkg/cmd/cli/plugin/helpers.go b/pkg/cmd/cli/plugin/helpers.go index 28f681993e..16684a71c7 100644 --- a/pkg/cmd/cli/plugin/helpers.go +++ b/pkg/cmd/cli/plugin/helpers.go @@ -19,7 +19,7 @@ package plugin import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" appsv1api "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/cmd/cli/plugin/remove.go b/pkg/cmd/cli/plugin/remove.go index ed25dc680a..d9b95cb371 100644 --- a/pkg/cmd/cli/plugin/remove.go +++ b/pkg/cmd/cli/plugin/remove.go @@ -20,8 +20,8 @@ import ( "context" "encoding/json" + "github.com/cockroachdb/errors" jsonpatch "github.com/evanphx/json-patch/v5" - "github.com/pkg/errors" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/cmd/cli/podvolume/backup.go b/pkg/cmd/cli/podvolume/backup.go index ed2d6c09a4..8bef9c5741 100644 --- a/pkg/cmd/cli/podvolume/backup.go +++ b/pkg/cmd/cli/podvolume/backup.go @@ -21,7 +21,7 @@ import ( "time" "github.com/bombsimon/logrusr/v3" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" corev1api "k8s.io/api/core/v1" diff --git a/pkg/cmd/cli/podvolume/restore.go b/pkg/cmd/cli/podvolume/restore.go index 4c1596a045..ab65549990 100644 --- a/pkg/cmd/cli/podvolume/restore.go +++ b/pkg/cmd/cli/podvolume/restore.go @@ -21,7 +21,7 @@ import ( "time" "github.com/bombsimon/logrusr/v3" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" corev1api "k8s.io/api/core/v1" diff --git a/pkg/cmd/cli/repomantenance/maintenance.go b/pkg/cmd/cli/repomantenance/maintenance.go index 46c54f7d2a..f89aba2572 100644 --- a/pkg/cmd/cli/repomantenance/maintenance.go +++ b/pkg/cmd/cli/repomantenance/maintenance.go @@ -8,7 +8,7 @@ import ( "time" "github.com/bombsimon/logrusr/v3" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/pflag" diff --git a/pkg/cmd/cli/restore/create.go b/pkg/cmd/cli/restore/create.go index 2fb21433e9..3f59b6a6bb 100644 --- a/pkg/cmd/cli/restore/create.go +++ b/pkg/cmd/cli/restore/create.go @@ -22,7 +22,7 @@ import ( "sort" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" corev1api "k8s.io/api/core/v1" @@ -32,6 +32,7 @@ import ( kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/resourcemodifiers" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" @@ -61,7 +62,13 @@ func NewCreateCommand(f client.Factory, use string) *cobra.Command { velero restore create --from-schedule schedule-1 --allow-partially-failed # Create a restore for only persistentvolumeclaims and persistentvolumes within a backup. - velero restore create --from-backup backup-2 --include-resources persistentvolumeclaims,persistentvolumes`, + velero restore create --from-backup backup-2 --include-resources persistentvolumeclaims,persistentvolumes + +Notes: +- Global filters (--include-resources, --selector, etc.) apply to all included namespaces +- Namespace-scoped filters defined in --resource-policies-configmap refine global filters for matching namespaces (globally excluded kinds cannot be re-included) +- Fine-grained global filter policies defined in --resource-policies-configmap refine global filters for cluster-scoped resources +- Use 'velero restore describe' to view the referenced resource policies ConfigMap after restore creation`, Args: cobra.MaximumNArgs(1), Run: func(c *cobra.Command, args []string) { cmd.CheckError(o.Complete(args, f)) @@ -100,6 +107,7 @@ type CreateOptions struct { AllowPartiallyFailed flag.OptionalBool ItemOperationTimeout time.Duration ResourceModifierConfigMap string + ResourcePoliciesConfigMap string WriteSparseFiles flag.OptionalBool ParallelFilesDownload int client kbclient.WithWatch @@ -154,6 +162,8 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) { flags.StringVar(&o.ResourceModifierConfigMap, "resource-modifier-configmap", "", "Reference to the resource modifier configmap that restore will use") + flags.StringVar(&o.ResourcePoliciesConfigMap, "resource-policies-configmap", "", "Reference to the ConfigMap containing restore resource filter policies") + f = flags.VarPF(&o.WriteSparseFiles, "write-sparse-files", "", "Whether to write sparse files during restoring volumes") f.NoOptDefVal = cmd.TRUE @@ -310,6 +320,15 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { } } + var resPolicies *corev1api.TypedLocalObjectReference + + if o.ResourcePoliciesConfigMap != "" { + resPolicies = &corev1api.TypedLocalObjectReference{ + Kind: resourcepolicies.ConfigmapRefType, + Name: o.ResourcePoliciesConfigMap, + } + } + restore := &api.Restore{ ObjectMeta: metav1.ObjectMeta{ Namespace: f.Namespace(), @@ -332,6 +351,7 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { PreserveNodePorts: o.PreserveNodePorts.Value, IncludeClusterResources: o.IncludeClusterResources.Value, ResourceModifier: resModifiers, + ResourcePolicy: resPolicies, ItemOperationTimeout: metav1.Duration{ Duration: o.ItemOperationTimeout, }, diff --git a/pkg/cmd/cli/restore/create_test.go b/pkg/cmd/cli/restore/create_test.go index 8cc369deaa..9a6a926085 100644 --- a/pkg/cmd/cli/restore/create_test.go +++ b/pkg/cmd/cli/restore/create_test.go @@ -77,6 +77,8 @@ func TestCreateCommand(t *testing.T) { includeClusterResources := "true" allowPartiallyFailed := "true" itemOperationTimeout := "10m0s" + resourceModifierConfigMap := "modifier-cm" + ResourcePoliciesConfigMap := "policies-cm" writeSparseFiles := "true" parallel := 2 flags := new(pflag.FlagSet) @@ -101,6 +103,8 @@ func TestCreateCommand(t *testing.T) { flags.Parse([]string{"--include-cluster-resources", includeClusterResources}) flags.Parse([]string{"--allow-partially-failed", allowPartiallyFailed}) flags.Parse([]string{"--item-operation-timeout", itemOperationTimeout}) + flags.Parse([]string{"--resource-modifier-configmap", resourceModifierConfigMap}) + flags.Parse([]string{"--resource-policies-configmap", ResourcePoliciesConfigMap}) flags.Parse([]string{"--write-sparse-files", writeSparseFiles}) flags.Parse([]string{"--parallel-files-download", "2"}) client := velerotest.NewFakeControllerRuntimeClient(t).(kbclient.WithWatch) @@ -139,6 +143,8 @@ func TestCreateCommand(t *testing.T) { require.Equal(t, includeClusterResources, o.IncludeClusterResources.String()) require.Equal(t, allowPartiallyFailed, o.AllowPartiallyFailed.String()) require.Equal(t, itemOperationTimeout, o.ItemOperationTimeout.String()) + require.Equal(t, resourceModifierConfigMap, o.ResourceModifierConfigMap) + require.Equal(t, ResourcePoliciesConfigMap, o.ResourcePoliciesConfigMap) require.Equal(t, writeSparseFiles, o.WriteSparseFiles.String()) require.Equal(t, parallel, o.ParallelFilesDownload) }) @@ -189,4 +195,37 @@ func TestCreateCommand(t *testing.T) { err := o.Validate(c, []string{}, f) require.Equal(t, "backups.velero.io \"not-exist\" not found", err.Error()) }) + + t.Run("create a restore with resource policies configmap", func(t *testing.T) { + f := &factorymocks.Factory{} + c := NewCreateCommand(f, "") + require.Equal(t, "Create a restore", c.Short) + flags := new(pflag.FlagSet) + o := NewCreateOptions() + o.BindFlags(flags) + + backupName := "backup-with-policies" + ResourcePoliciesConfigMap := "test-policies-cm" + flags.Parse([]string{"--from-backup", backupName}) + flags.Parse([]string{"--resource-policies-configmap", ResourcePoliciesConfigMap}) + + kbclient := velerotest.NewFakeControllerRuntimeClient(t).(kbclient.WithWatch) + backup := builder.ForBackup(cmdtest.VeleroNameSpace, backupName).Phase(velerov1api.BackupPhaseCompleted).Result() + require.NoError(t, kbclient.Create(t.Context(), backup, &controllerclient.CreateOptions{})) + + f.On("Namespace").Return(cmdtest.VeleroNameSpace) + f.On("KubebuilderWatchClient").Return(kbclient, nil) + + require.NoError(t, o.Complete(args, f)) + require.NoError(t, o.Validate(c, []string{}, f)) + require.NoError(t, o.Run(c, f)) + + // Verify the created restore object + createdRestore := &velerov1api.Restore{} + err := kbclient.Get(t.Context(), controllerclient.ObjectKey{Namespace: cmdtest.VeleroNameSpace, Name: name}, createdRestore) + require.NoError(t, err) + require.NotNil(t, createdRestore.Spec.ResourcePolicy) + require.Equal(t, "configmap", createdRestore.Spec.ResourcePolicy.Kind) + require.Equal(t, ResourcePoliciesConfigMap, createdRestore.Spec.ResourcePolicy.Name) + }) } diff --git a/pkg/cmd/cli/restore/delete.go b/pkg/cmd/cli/restore/delete.go index a2e70953db..51c31e1da6 100644 --- a/pkg/cmd/cli/restore/delete.go +++ b/pkg/cmd/cli/restore/delete.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/cmd/cli/schedule/create.go b/pkg/cmd/cli/schedule/create.go index 47c19318f1..2e4a1e8e9c 100644 --- a/pkg/cmd/cli/schedule/create.go +++ b/pkg/cmd/cli/schedule/create.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" corev1api "k8s.io/api/core/v1" diff --git a/pkg/cmd/cli/schedule/delete.go b/pkg/cmd/cli/schedule/delete.go index 77b0bf883c..78e8c91040 100644 --- a/pkg/cmd/cli/schedule/delete.go +++ b/pkg/cmd/cli/schedule/delete.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/cmd/cli/schedule/pause.go b/pkg/cmd/cli/schedule/pause.go index 820e887a7d..41a17f3843 100644 --- a/pkg/cmd/cli/schedule/pause.go +++ b/pkg/cmd/cli/schedule/pause.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/cmd/cli/serverstatus/server_status.go b/pkg/cmd/cli/serverstatus/server_status.go index ab994e2d63..da052f5791 100644 --- a/pkg/cmd/cli/serverstatus/server_status.go +++ b/pkg/cmd/cli/serverstatus/server_status.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/util/wait" kbclient "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/cmd/cli/snapshotlocation/create.go b/pkg/cmd/cli/snapshotlocation/create.go index db55ad8349..d0f0203a8b 100644 --- a/pkg/cmd/cli/snapshotlocation/create.go +++ b/pkg/cmd/cli/snapshotlocation/create.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/cmd/cli/snapshotlocation/set.go b/pkg/cmd/cli/snapshotlocation/set.go index f6b8ac3688..0814bdfe71 100644 --- a/pkg/cmd/cli/snapshotlocation/set.go +++ b/pkg/cmd/cli/snapshotlocation/set.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" diff --git a/pkg/cmd/cli/uninstall/uninstall.go b/pkg/cmd/cli/uninstall/uninstall.go index 80a349c92f..93e0118c6e 100644 --- a/pkg/cmd/cli/uninstall/uninstall.go +++ b/pkg/cmd/cli/uninstall/uninstall.go @@ -23,7 +23,7 @@ import ( "sync" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" diff --git a/pkg/cmd/cli/version/version_test.go b/pkg/cmd/cli/version/version_test.go index 355626802f..71233431f9 100644 --- a/pkg/cmd/cli/version/version_test.go +++ b/pkg/cmd/cli/version/version_test.go @@ -21,7 +21,7 @@ import ( "fmt" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" kbclient "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/cmd/server/config/config.go b/pkg/cmd/server/config/config.go index e19086217b..c6be56260d 100644 --- a/pkg/cmd/server/config/config.go +++ b/pkg/cmd/server/config/config.go @@ -28,6 +28,11 @@ const ( defaultPodVolumeOperationTimeout = 240 * time.Minute defaultResourceTerminatingTimeout = 10 * time.Minute + // DefaultResourceTimeout is the default for --resource-timeout. It matches + // defaultResourceTerminatingTimeout so controller fallbacks stay aligned with + // server defaults (see pkg/cmd/server/config/config.go). + DefaultResourceTimeout = defaultResourceTerminatingTimeout + // server's client default qps and burst defaultClientQPS float32 = 100.0 defaultClientBurst int = 100 @@ -41,7 +46,7 @@ const ( defaultCSISnapshotTimeout = 10 * time.Minute defaultItemOperationTimeout = 4 * time.Hour - resourceTimeout = 10 * time.Minute + resourceTimeout = defaultResourceTerminatingTimeout defaultMaxConcurrentK8SConnections = 30 defaultDisableInformerCache = false @@ -145,42 +150,43 @@ var ( ) type Config struct { - PluginDir string - MetricsAddress string - DefaultBackupLocation string // TODO(2.0) Deprecate defaultBackupLocation - BackupSyncPeriod time.Duration - PodVolumeOperationTimeout time.Duration - ResourceTerminatingTimeout time.Duration - DefaultBackupTTL time.Duration - DefaultVGSLabelKey string - StoreValidationFrequency time.Duration - DefaultCSISnapshotTimeout time.Duration - DefaultItemOperationTimeout time.Duration - ResourceTimeout time.Duration - RestoreResourcePriorities types.Priorities - DefaultVolumeSnapshotLocations flag.Map - RestoreOnly bool - DisabledControllers []string - ClientQPS float32 - ClientBurst int - ClientPageSize int - ProfilerAddress string - LogLevel *logging.LevelFlag - LogFormat *logging.FormatFlag - RepoMaintenanceFrequency time.Duration - GarbageCollectionFrequency time.Duration - ItemOperationSyncFrequency time.Duration - DefaultVolumesToFsBackup bool - UploaderType string - MaxConcurrentK8SConnections int - DefaultSnapshotMoveData bool - DisableInformerCache bool - ScheduleSkipImmediately bool - CredentialsDirectory string - BackupRepoConfig string - RepoMaintenanceJobConfig string - ItemBlockWorkerCount int - ConcurrentBackups int + PluginDir string + MetricsAddress string + DefaultBackupLocation string // TODO(2.0) Deprecate defaultBackupLocation + BackupSyncPeriod time.Duration + PodVolumeOperationTimeout time.Duration + ResourceTerminatingTimeout time.Duration + DefaultBackupTTL time.Duration + DefaultVGSLabelKey string + StoreValidationFrequency time.Duration + DefaultCSISnapshotTimeout time.Duration + DefaultItemOperationTimeout time.Duration + ResourceTimeout time.Duration + RestoreResourcePriorities types.Priorities + DefaultVolumeSnapshotLocations flag.Map + RestoreOnly bool + DisabledControllers []string + ClientQPS float32 + ClientBurst int + ClientPageSize int + ProfilerAddress string + LogLevel *logging.LevelFlag + LogFormat *logging.FormatFlag + RepoMaintenanceFrequency time.Duration + GarbageCollectionFrequency time.Duration + ItemOperationSyncFrequency time.Duration + DefaultVolumesToFsBackup bool + UploaderType string + MaxConcurrentK8SConnections int + DefaultSnapshotMoveData bool + DisableInformerCache bool + ScheduleSkipImmediately bool + CredentialsDirectory string + BackupRepoConfig string + RepoMaintenanceJobConfig string + ItemBlockWorkerCount int + ConcurrentBackups int + GlobalBackupVolumePoliciesConfigMap string } func GetDefaultConfig() *Config { @@ -275,4 +281,10 @@ func (c *Config) BindFlags(flags *pflag.FlagSet) { c.ConcurrentBackups, "Number of backups to process concurrently. Default is one. Optional.", ) + flags.StringVar( + &c.GlobalBackupVolumePoliciesConfigMap, + "global-backup-volume-policies-configmap", + c.GlobalBackupVolumePoliciesConfigMap, + "The name of a ConfigMap in the Velero install namespace holding global backup volume policies that are merged into every backup. Optional.", + ) } diff --git a/pkg/cmd/server/config/config_test.go b/pkg/cmd/server/config/config_test.go index ba17437f11..a0e33c4138 100644 --- a/pkg/cmd/server/config/config_test.go +++ b/pkg/cmd/server/config/config_test.go @@ -5,6 +5,7 @@ import ( "github.com/spf13/pflag" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestGetDefaultConfig(t *testing.T) { @@ -17,3 +18,14 @@ func TestBindFlags(t *testing.T) { config.BindFlags(pflag.CommandLine) assert.Equal(t, 1, config.ItemBlockWorkerCount) } + +func TestGlobalBackupVolumePoliciesConfigMapFlag(t *testing.T) { + config := GetDefaultConfig() + // Opt-in: defaults to empty. + assert.Empty(t, config.GlobalBackupVolumePoliciesConfigMap) + + flags := pflag.NewFlagSet("test", pflag.ContinueOnError) + config.BindFlags(flags) + require.NoError(t, flags.Parse([]string{"--global-backup-volume-policies-configmap", "global-volume-policy"})) + assert.Equal(t, "global-volume-policy", config.GlobalBackupVolumePoliciesConfigMap) +} diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index cb38a40d0f..5cceceafe7 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -27,9 +27,9 @@ import ( "time" logrusr "github.com/bombsimon/logrusr/v3" + "github.com/cockroachdb/errors" volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -57,6 +57,7 @@ import ( "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/internal/hook" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/storage" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" @@ -390,6 +391,14 @@ func (s *server) setupBeforeControllerRun() error { if err := setDefaultBackupLocation(s.ctx, client, s.namespace, s.config.DefaultBackupLocation, s.logger); err != nil { return err } + + // Validate the global backup volume policies ConfigMap early, so misconfigurations fail fast. + if s.config.GlobalBackupVolumePoliciesConfigMap != "" { + if _, err := resourcepolicies.GetGlobalResourcePolicies(client, s.namespace, s.config.GlobalBackupVolumePoliciesConfigMap, s.logger); err != nil { + return err + } + s.logger.WithField("configmap", s.config.GlobalBackupVolumePoliciesConfigMap).Info("Loaded global backup volume policies") + } return nil } @@ -671,6 +680,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.config.ItemBlockWorkerCount, s.config.ConcurrentBackups, s.crClient, + s.config.GlobalBackupVolumePoliciesConfigMap, ).SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerBackup) } diff --git a/pkg/cmd/util/cacert/bsl_cacert.go b/pkg/cmd/util/cacert/bsl_cacert.go index d117299452..9d69d6c03d 100644 --- a/pkg/cmd/util/cacert/bsl_cacert.go +++ b/pkg/cmd/util/cacert/bsl_cacert.go @@ -19,7 +19,7 @@ package cacert import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/cmd/util/downloadrequest/downloadrequest.go b/pkg/cmd/util/downloadrequest/downloadrequest.go index 6e1d30c379..f0956b1cb8 100644 --- a/pkg/cmd/util/downloadrequest/downloadrequest.go +++ b/pkg/cmd/util/downloadrequest/downloadrequest.go @@ -28,8 +28,8 @@ import ( "os" "time" + "github.com/cockroachdb/errors" "github.com/google/uuid" - "github.com/pkg/errors" kbclient "sigs.k8s.io/controller-runtime/pkg/client" veleroV1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/pkg/cmd/util/flag/enum.go b/pkg/cmd/util/flag/enum.go index bc36aef68f..f2334a612f 100644 --- a/pkg/cmd/util/flag/enum.go +++ b/pkg/cmd/util/flag/enum.go @@ -17,7 +17,7 @@ limitations under the License. package flag import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" ) // Enum is a Cobra-compatible wrapper for defining diff --git a/pkg/cmd/util/flag/map.go b/pkg/cmd/util/flag/map.go index 1b4a6e21c8..fae6329e1d 100644 --- a/pkg/cmd/util/flag/map.go +++ b/pkg/cmd/util/flag/map.go @@ -21,7 +21,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" ) // Map is a Cobra-compatible wrapper for defining a flag containing diff --git a/pkg/cmd/util/output/backup_describer.go b/pkg/cmd/util/output/backup_describer.go index e0637a4bde..445ce3df56 100644 --- a/pkg/cmd/util/output/backup_describer.go +++ b/pkg/cmd/util/output/backup_describer.go @@ -28,8 +28,8 @@ import ( corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/fatih/color" kbclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -40,6 +40,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" "github.com/vmware-tanzu/velero/pkg/itemoperation" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" "github.com/vmware-tanzu/velero/pkg/util/collections" "github.com/vmware-tanzu/velero/pkg/util/results" @@ -93,6 +94,8 @@ func DescribeBackup( DescribeResourcePolicies(d, backup.Spec.ResourcePolicy) } + DescribeGlobalVolumePolicy(d, backup) + if backup.Spec.UploaderConfig != nil && backup.Spec.UploaderConfig.ParallelFilesUpload > 0 { d.Println() DescribeUploaderConfigForBackup(d, backup.Spec) @@ -130,6 +133,19 @@ func DescribeResourcePolicies(d *Describer, resPolicies *corev1api.TypedLocalObj d.Printf("\tName:\t%s\n", resPolicies.Name) } +// DescribeGlobalVolumePolicy describes the cluster-wide global backup volume policies +// ConfigMap that contributed to the backup, if any. +func DescribeGlobalVolumePolicy(d *Describer, backup *velerov1api.Backup) { + name := backup.Annotations[velerov1api.GlobalBackupVolumePolicyConfigMapAnnotation] + if name == "" { + return + } + d.Println() + d.Printf("Global volume policies:\n") + d.Printf("\tType:\t%s\n", resourcepolicies.ConfigmapRefType) + d.Printf("\tName:\t%s\n", name) +} + // DescribeUploaderConfigForBackup describes uploader config in human-readable format func DescribeUploaderConfigForBackup(d *Describer, spec velerov1api.BackupSpec) { d.Printf("Uploader config:\n") diff --git a/pkg/cmd/util/output/backup_describer_test.go b/pkg/cmd/util/output/backup_describer_test.go index 0de03bdaa4..da28f6c878 100644 --- a/pkg/cmd/util/output/backup_describer_test.go +++ b/pkg/cmd/util/output/backup_describer_test.go @@ -69,6 +69,34 @@ func TestDescribeResourcePolicies(t *testing.T) { assert.Equal(t, expect, d.buf.String()) } +func TestDescribeGlobalVolumePolicy(t *testing.T) { + newDescriber := func() *Describer { + d := &Describer{out: &tabwriter.Writer{}, buf: &bytes.Buffer{}} + d.out.Init(d.buf, 0, 8, 2, ' ', 0) + return d + } + + // No annotation: nothing is printed. + d := newDescriber() + DescribeGlobalVolumePolicy(d, builder.ForBackup("velero", "b").Result()) + d.out.Flush() + assert.Empty(t, d.buf.String()) + + // Annotation present: ConfigMap name is surfaced. + d = newDescriber() + backup := builder.ForBackup("velero", "b"). + ObjectMeta(builder.WithAnnotations(velerov1api.GlobalBackupVolumePolicyConfigMapAnnotation, "global-volume-policy")). + Result() + DescribeGlobalVolumePolicy(d, backup) + d.out.Flush() + expect := ` +Global volume policies: + Type: configmap + Name: global-volume-policy +` + assert.Equal(t, expect, d.buf.String()) +} + func TestDescribeBackupSpec(t *testing.T) { input1 := builder.ForBackup("test-ns", "test-backup-1"). IncludedNamespaces("inc-ns-1", "inc-ns-2"). diff --git a/pkg/cmd/util/output/backup_structured_describer.go b/pkg/cmd/util/output/backup_structured_describer.go index 904afa34e4..b2541df4b9 100644 --- a/pkg/cmd/util/output/backup_structured_describer.go +++ b/pkg/cmd/util/output/backup_structured_describer.go @@ -28,6 +28,7 @@ import ( kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" @@ -56,6 +57,8 @@ func DescribeBackupInSF( DescribeResourcePoliciesInSF(d, backup.Spec.ResourcePolicy) } + DescribeGlobalVolumePolicyInSF(d, backup) + status := backup.Status if len(status.ValidationErrors) > 0 { d.Describe("validationErrors", status.ValidationErrors) @@ -613,6 +616,19 @@ func DescribeResourcePoliciesInSF(d *StructuredDescriber, resPolicies *corev1api d.Describe("resourcePolicies", policiesInfo) } +// DescribeGlobalVolumePolicyInSF describes the global backup volume policies ConfigMap that +// contributed to the backup, if any, in structured format. +func DescribeGlobalVolumePolicyInSF(d *StructuredDescriber, backup *velerov1api.Backup) { + name := backup.Annotations[velerov1api.GlobalBackupVolumePolicyConfigMapAnnotation] + if name == "" { + return + } + d.Describe("globalVolumePolicies", map[string]any{ + "type": resourcepolicies.ConfigmapRefType, + "name": name, + }) +} + func describeResultInSF(m map[string]any, result results.Result) { m["velero"], m["cluster"], m["namespace"] = []string{}, []string{}, []string{} diff --git a/pkg/cmd/util/output/backup_structured_describer_test.go b/pkg/cmd/util/output/backup_structured_describer_test.go index c5ede1b36c..88af0f95fa 100644 --- a/pkg/cmd/util/output/backup_structured_describer_test.go +++ b/pkg/cmd/util/output/backup_structured_describer_test.go @@ -624,6 +624,27 @@ func TestDescribeResourcePoliciesInSF(t *testing.T) { assert.True(t, reflect.DeepEqual(sd.output, expect)) } +func TestDescribeGlobalVolumePolicyInSF(t *testing.T) { + // No annotation: nothing is added to the output. + sd := &StructuredDescriber{output: make(map[string]any), format: ""} + DescribeGlobalVolumePolicyInSF(sd, builder.ForBackup("velero", "b").Result()) + assert.Empty(t, sd.output) + + // Annotation present: the ConfigMap name is surfaced. + sd = &StructuredDescriber{output: make(map[string]any), format: ""} + backup := builder.ForBackup("velero", "b"). + ObjectMeta(builder.WithAnnotations(velerov1api.GlobalBackupVolumePolicyConfigMapAnnotation, "global-volume-policy")). + Result() + DescribeGlobalVolumePolicyInSF(sd, backup) + expectGlobal := map[string]any{ + "globalVolumePolicies": map[string]any{ + "type": "configmap", + "name": "global-volume-policy", + }, + } + assert.True(t, reflect.DeepEqual(sd.output, expectGlobal)) +} + func TestDescribeBackupResultInSF(t *testing.T) { input := results.Result{ Velero: []string{"msg-1", "msg-2"}, diff --git a/pkg/cmd/util/output/output.go b/pkg/cmd/util/output/output.go index a0f8ce704a..9dfca040b0 100644 --- a/pkg/cmd/util/output/output.go +++ b/pkg/cmd/util/output/output.go @@ -21,7 +21,7 @@ import ( "os" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/api/meta" diff --git a/pkg/cmd/util/output/restore_describer.go b/pkg/cmd/util/output/restore_describer.go index a89943e746..c33da9f692 100644 --- a/pkg/cmd/util/output/restore_describer.go +++ b/pkg/cmd/util/output/restore_describer.go @@ -219,6 +219,11 @@ func DescribeRestore( DescribeResourceModifier(d, restore.Spec.ResourceModifier) } + if restore.Spec.ResourcePolicy != nil { + d.Println() + DescribeResourcePolicies(d, restore.Spec.ResourcePolicy) + } + describeUploaderConfigForRestore(d, restore.Spec) d.Println() diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 490c79aed0..161119d0c8 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -24,8 +24,8 @@ import ( "slices" "time" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -84,32 +84,33 @@ var autoExcludeClusterScopedResources = []string{ } type backupReconciler struct { - ctx context.Context - logger logrus.FieldLogger - discoveryHelper discovery.Helper - backupper pkgbackup.Backupper - kbClient kbclient.Client - clock clock.WithTickerAndDelayedExecution - backupLogLevel logrus.Level - newPluginManager func(logrus.FieldLogger) clientmgmt.Manager - backupTracker BackupTracker - defaultBackupLocation string - defaultVolumesToFsBackup bool - defaultBackupTTL time.Duration - defaultVGSLabelKey string - defaultCSISnapshotTimeout time.Duration - resourceTimeout time.Duration - defaultItemOperationTimeout time.Duration - defaultSnapshotLocations map[string]string - metrics *metrics.ServerMetrics - backupStoreGetter persistence.ObjectBackupStoreGetter - formatFlag logging.Format - credentialFileStore credentials.FileStore - maxConcurrentK8SConnections int - defaultSnapshotMoveData bool - globalCRClient kbclient.Client - itemBlockWorkerCount int - concurrentBackups int + ctx context.Context + logger logrus.FieldLogger + discoveryHelper discovery.Helper + backupper pkgbackup.Backupper + kbClient kbclient.Client + clock clock.WithTickerAndDelayedExecution + backupLogLevel logrus.Level + newPluginManager func(logrus.FieldLogger) clientmgmt.Manager + backupTracker BackupTracker + defaultBackupLocation string + defaultVolumesToFsBackup bool + defaultBackupTTL time.Duration + defaultVGSLabelKey string + defaultCSISnapshotTimeout time.Duration + resourceTimeout time.Duration + defaultItemOperationTimeout time.Duration + defaultSnapshotLocations map[string]string + metrics *metrics.ServerMetrics + backupStoreGetter persistence.ObjectBackupStoreGetter + formatFlag logging.Format + credentialFileStore credentials.FileStore + maxConcurrentK8SConnections int + defaultSnapshotMoveData bool + globalCRClient kbclient.Client + itemBlockWorkerCount int + concurrentBackups int + globalVolumePoliciesConfigMap string } func NewBackupReconciler( @@ -138,34 +139,36 @@ func NewBackupReconciler( itemBlockWorkerCount int, concurrentBackups int, globalCRClient kbclient.Client, + globalVolumePoliciesConfigMap string, ) *backupReconciler { b := &backupReconciler{ - ctx: ctx, - discoveryHelper: discoveryHelper, - backupper: backupper, - clock: &clock.RealClock{}, - logger: logger, - backupLogLevel: backupLogLevel, - newPluginManager: newPluginManager, - backupTracker: backupTracker, - kbClient: kbClient, - defaultBackupLocation: defaultBackupLocation, - defaultVolumesToFsBackup: defaultVolumesToFsBackup, - defaultBackupTTL: defaultBackupTTL, - defaultVGSLabelKey: defaultVGSLabelKey, - defaultCSISnapshotTimeout: defaultCSISnapshotTimeout, - resourceTimeout: resourceTimeout, - defaultItemOperationTimeout: defaultItemOperationTimeout, - defaultSnapshotLocations: defaultSnapshotLocations, - metrics: metrics, - backupStoreGetter: backupStoreGetter, - formatFlag: formatFlag, - credentialFileStore: credentialStore, - maxConcurrentK8SConnections: maxConcurrentK8SConnections, - defaultSnapshotMoveData: defaultSnapshotMoveData, - itemBlockWorkerCount: itemBlockWorkerCount, - concurrentBackups: max(concurrentBackups, 1), - globalCRClient: globalCRClient, + ctx: ctx, + discoveryHelper: discoveryHelper, + backupper: backupper, + clock: &clock.RealClock{}, + logger: logger, + backupLogLevel: backupLogLevel, + newPluginManager: newPluginManager, + backupTracker: backupTracker, + kbClient: kbClient, + defaultBackupLocation: defaultBackupLocation, + defaultVolumesToFsBackup: defaultVolumesToFsBackup, + defaultBackupTTL: defaultBackupTTL, + defaultVGSLabelKey: defaultVGSLabelKey, + defaultCSISnapshotTimeout: defaultCSISnapshotTimeout, + resourceTimeout: resourceTimeout, + defaultItemOperationTimeout: defaultItemOperationTimeout, + defaultSnapshotLocations: defaultSnapshotLocations, + metrics: metrics, + backupStoreGetter: backupStoreGetter, + formatFlag: formatFlag, + credentialFileStore: credentialStore, + maxConcurrentK8SConnections: maxConcurrentK8SConnections, + defaultSnapshotMoveData: defaultSnapshotMoveData, + itemBlockWorkerCount: itemBlockWorkerCount, + concurrentBackups: max(concurrentBackups, 1), + globalCRClient: globalCRClient, + globalVolumePoliciesConfigMap: globalVolumePoliciesConfigMap, } b.updateTotalBackupMetric() return b @@ -595,14 +598,25 @@ func (b *backupReconciler) prepareBackupRequest(ctx context.Context, backup *vel request.Status.ValidationErrors = append(request.Status.ValidationErrors, "encountered labelSelector as well as orLabelSelectors in backup spec, only one can be specified") } - resourcePolicies, err := resourcepolicies.GetResourcePoliciesFromBackup(*request.Backup, b.kbClient, logger) + resourcePolicies, err := resourcepolicies.GetResourcePoliciesFromBackupWithGlobal( + *request.Backup, b.kbClient, b.globalVolumePoliciesConfigMap, request.Namespace, logger) if err != nil { request.Status.ValidationErrors = append(request.Status.ValidationErrors, err.Error()) + } else if b.globalVolumePoliciesConfigMap != "" { + // Record the contributing global volume policies ConfigMap so `velero backup describe` can surface it. + request.Annotations[velerov1api.GlobalBackupVolumePolicyConfigMapAnnotation] = b.globalVolumePoliciesConfigMap } if resourcePolicies != nil && resourcePolicies.GetIncludeExcludePolicy() != nil && collections.UseOldResourceFilters(request.Spec) { request.Status.ValidationErrors = append(request.Status.ValidationErrors, "include-resources, exclude-resources and include-cluster-resources are old filter parameters.\n"+ "They cannot be used with include-exclude policies.") } + // namespacedFilterPolicies and clusterScopedFilterPolicy incompatible with old-style filters + if resourcePolicies != nil && + (len(resourcePolicies.GetNamespacedFilterPolicies()) > 0 || resourcePolicies.GetClusterScopedFilterPolicy() != nil) && + collections.UseOldResourceFilters(request.Spec) { + request.Status.ValidationErrors = append(request.Status.ValidationErrors, "include-resources, exclude-resources and include-cluster-resources are old filter parameters.\n"+ + "They cannot be used with namespace-scoped or fine-grained global filter policies.") + } request.ResPolicies = resourcePolicies return request } diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index c65f1d15d2..6a7e681147 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -21,18 +21,20 @@ import ( "fmt" "io" "reflect" + "slices" "sort" "strings" "testing" "time" + "github.com/cockroachdb/errors" "github.com/google/go-cmp/cmp" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -43,6 +45,7 @@ import ( kbclient "sigs.k8s.io/controller-runtime/pkg/client" fakeClient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/builder" @@ -2019,3 +2022,331 @@ func TestPatchResourceWorksWithStatus(t *testing.T) { }) } } + +// TestPrepareBackupRequest_NamespacedFilterPoliciesIncompatibleWithOldFilters verifies +// that a backup referencing a ResourcePolicy ConfigMap with namespacedFilterPolicies +// produces a validation error when old-style resource filters are also set on the spec. +func TestPrepareBackupRequest_NamespacedFilterPoliciesIncompatibleWithOldFilters(t *testing.T) { + formatFlag := logging.FormatText + logger := logging.DefaultLogger(logrus.DebugLevel, formatFlag) + + policyYAML := `version: v1 +namespacedFilterPolicies: +- namespaces: ["production"] + resourceFilters: + - kinds: ["Deployment"] + names: ["api-server"] +` + policyConfigMap := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-filter-policy", + Namespace: velerov1api.DefaultNamespace, + }, + Data: map[string]string{"policy": policyYAML}, + } + + backup := defaultBackup().IncludedResources("deployments").Result() + backup.Spec.ResourcePolicy = &corev1api.TypedLocalObjectReference{ + Kind: "configmap", + Name: "my-filter-policy", + } + + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, policyConfigMap) + + apiServer := velerotest.NewAPIServer(t) + discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger) + require.NoError(t, err) + + c := &backupReconciler{ + logger: logger, + discoveryHelper: discoveryHelper, + kbClient: fakeClient, + clock: &clock.RealClock{}, + formatFlag: formatFlag, + } + + res := c.prepareBackupRequest(ctx, backup, logger) + + require.NotEmpty(t, res.Status.ValidationErrors) + + hasTargetError := slices.ContainsFunc(res.Status.ValidationErrors, func(e string) bool { + return strings.Contains(e, "namespace-scoped or fine-grained global filter policies") + }) + + assert.True(t, hasTargetError, "expected validation error about namespacedFilterPolicies incompatibility with old-style filters, got: %v", res.Status.ValidationErrors) +} + +// TestPrepareBackupRequest_GlobalVolumePolicies verifies that the cluster-wide global backup +// volume policies are merged into the request and that the contributing ConfigMap is recorded +// on the backup so `velero backup describe` can surface it. +func TestPrepareBackupRequest_GlobalVolumePolicies(t *testing.T) { + formatFlag := logging.FormatText + logger := logging.DefaultLogger(logrus.DebugLevel, formatFlag) + + globalCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "global-volume-policy", Namespace: velerov1api.DefaultNamespace}, + Data: map[string]string{"policies.yaml": `version: v1 +volumePolicies: + - conditions: + storageClass: + - gp2 + action: + type: skip +`}, + } + + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, globalCM, + builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "loc-1").Result()) + apiServer := velerotest.NewAPIServer(t) + discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger) + require.NoError(t, err) + + c := &backupReconciler{ + logger: logger, + discoveryHelper: discoveryHelper, + kbClient: fakeClient, + clock: &clock.RealClock{}, + formatFlag: formatFlag, + defaultBackupLocation: "loc-1", + globalVolumePoliciesConfigMap: "global-volume-policy", + } + + backup := defaultBackup().StorageLocation("loc-1").Result() + res := c.prepareBackupRequest(ctx, backup, logger) + defer res.WorkerPool.Stop() + + // The global volume policies must load cleanly (no policy-related validation error). + for _, e := range res.Status.ValidationErrors { + assert.NotContains(t, e, "global backup volume policies") + } + require.NotNil(t, res.ResPolicies) + assert.Equal(t, "global-volume-policy", res.Annotations[velerov1api.GlobalBackupVolumePolicyConfigMapAnnotation]) + + action, err := res.ResPolicies.GetMatchAction(resourcepolicies.VolumeFilterData{ + PersistentVolume: &corev1api.PersistentVolume{Spec: corev1api.PersistentVolumeSpec{StorageClassName: "gp2"}}, + }) + require.NoError(t, err) + require.NotNil(t, action) + assert.Equal(t, resourcepolicies.Skip, action.Type) +} + +// TestPrepareBackupRequest_GlobalVolumePolicies_LoadError verifies that when the configured +// global backup volume policies ConfigMap cannot be loaded, a validation error is recorded and +// the contributing-ConfigMap annotation is not set on the backup. +func TestPrepareBackupRequest_GlobalVolumePolicies_LoadError(t *testing.T) { + formatFlag := logging.FormatText + logger := logging.DefaultLogger(logrus.DebugLevel, formatFlag) + + // No ConfigMap with this name exists, so loading the global policies fails. + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, + builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "loc-1").Result()) + apiServer := velerotest.NewAPIServer(t) + discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger) + require.NoError(t, err) + + c := &backupReconciler{ + logger: logger, + discoveryHelper: discoveryHelper, + kbClient: fakeClient, + clock: &clock.RealClock{}, + formatFlag: formatFlag, + defaultBackupLocation: "loc-1", + globalVolumePoliciesConfigMap: "missing-global-volume-policy", + } + + backup := defaultBackup().StorageLocation("loc-1").Result() + res := c.prepareBackupRequest(ctx, backup, logger) + defer res.WorkerPool.Stop() + + // The failure to load the global policies must surface as a validation error. + var hasGlobalPolicyError bool + for _, e := range res.Status.ValidationErrors { + if strings.Contains(e, "global backup volume policies") { + hasGlobalPolicyError = true + } + } + assert.True(t, hasGlobalPolicyError, "expected a validation error about global backup volume policies, got: %v", res.Status.ValidationErrors) + // The annotation is only set when the policies load successfully. + assert.Empty(t, res.Annotations[velerov1api.GlobalBackupVolumePolicyConfigMapAnnotation]) +} + +// TestPrepareBackupRequest_ClusterScopedFilterPolicyIncompatibleWithOldFilters verifies +// that a backup referencing a ResourcePolicy ConfigMap with clusterScopedFilterPolicy +// produces a validation error when old-style resource filters are also set on the spec. +func TestPrepareBackupRequest_ClusterScopedFilterPolicyIncompatibleWithOldFilters(t *testing.T) { + formatFlag := logging.FormatText + logger := logging.DefaultLogger(logrus.DebugLevel, formatFlag) + + policyYAML := `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole"] + names: ["my-app-*"] +` + policyConfigMap := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster-filter-policy", + Namespace: velerov1api.DefaultNamespace, + }, + Data: map[string]string{"policy": policyYAML}, + } + + backup := defaultBackup().IncludedResources("clusterroles").Result() + backup.Spec.ResourcePolicy = &corev1api.TypedLocalObjectReference{ + Kind: "configmap", + Name: "my-cluster-filter-policy", + } + + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, policyConfigMap) + + apiServer := velerotest.NewAPIServer(t) + discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger) + require.NoError(t, err) + + c := &backupReconciler{ + logger: logger, + discoveryHelper: discoveryHelper, + kbClient: fakeClient, + clock: &clock.RealClock{}, + formatFlag: formatFlag, + } + + res := c.prepareBackupRequest(ctx, backup, logger) + + require.NotEmpty(t, res.Status.ValidationErrors) + + hasClusterError := slices.ContainsFunc(res.Status.ValidationErrors, func(e string) bool { + return strings.Contains(e, "namespace-scoped or fine-grained global filter policies") + }) + + assert.True(t, hasClusterError, "expected validation error about clusterScopedFilterPolicy incompatibility with old-style filters, got: %v", res.Status.ValidationErrors) +} + +const ( + namespacedFilterPolicyYAML = `version: v1 +namespacedFilterPolicies: +- namespaces: ["production"] + resourceFilters: + - kinds: ["Deployment"] + names: ["api-server"] +` + clusterScopedFilterPolicyYAML = `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole"] + names: ["my-app-*"] +` + bothFilterPoliciesYAML = `version: v1 +namespacedFilterPolicies: +- namespaces: ["production"] + resourceFilters: + - kinds: ["Deployment"] + names: ["api-server"] +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole"] + names: ["my-app-*"] +` +) + +// TestPrepareBackupRequest_FilterPoliciesWithNewFilters verifies that backups referencing +// a ResourcePolicy ConfigMap with namespacedFilterPolicies and/or clusterScopedFilterPolicy +// succeed when old-style resource filters are not set on the spec. +func TestPrepareBackupRequest_FilterPoliciesWithNewFilters(t *testing.T) { + tests := []struct { + name string + policyYAML string + policyConfigMapName string + backup *velerov1api.Backup + expectNamespacedPolicies int + expectClusterScopedPolicy bool + }{ + { + name: "namespacedFilterPolicies only", + policyYAML: namespacedFilterPolicyYAML, + policyConfigMapName: "my-filter-policy", + backup: defaultBackup().StorageLocation("loc-1").Result(), + expectNamespacedPolicies: 1, + }, + { + name: "clusterScopedFilterPolicy only", + policyYAML: clusterScopedFilterPolicyYAML, + policyConfigMapName: "my-cluster-filter-policy", + backup: defaultBackup().StorageLocation("loc-1").Result(), + expectClusterScopedPolicy: true, + }, + { + name: "both filter policies", + policyYAML: bothFilterPoliciesYAML, + policyConfigMapName: "my-combined-filter-policy", + backup: defaultBackup().StorageLocation("loc-1").Result(), + expectNamespacedPolicies: 1, + expectClusterScopedPolicy: true, + }, + { + name: "with new-style spec filters", + policyYAML: bothFilterPoliciesYAML, + policyConfigMapName: "my-combined-filter-policy", + backup: defaultBackup(). + StorageLocation("loc-1"). + IncludedNamespaceScopedResources("deployments"). + IncludedClusterScopedResources("clusterroles"). + Result(), + expectNamespacedPolicies: 1, + expectClusterScopedPolicy: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + formatFlag := logging.FormatText + logger := logging.DefaultLogger(logrus.DebugLevel, formatFlag) + + policyConfigMap := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: test.policyConfigMapName, + Namespace: velerov1api.DefaultNamespace, + }, + Data: map[string]string{"policy": test.policyYAML}, + } + + test.backup.Spec.ResourcePolicy = &corev1api.TypedLocalObjectReference{ + Kind: "configmap", + Name: test.policyConfigMapName, + } + + backupLocation := builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "loc-1"). + Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, backupLocation, policyConfigMap) + + apiServer := velerotest.NewAPIServer(t) + discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger) + require.NoError(t, err) + + c := &backupReconciler{ + logger: logger, + discoveryHelper: discoveryHelper, + kbClient: fakeClient, + clock: &clock.RealClock{}, + formatFlag: formatFlag, + } + + res := c.prepareBackupRequest(ctx, test.backup, logger) + defer res.WorkerPool.Stop() + + assert.Empty(t, res.Status.ValidationErrors) + hasIncompatibilityError := slices.ContainsFunc(res.Status.ValidationErrors, func(e string) bool { + return strings.Contains(e, "namespace-scoped or fine-grained global filter policies") + }) + assert.False(t, hasIncompatibilityError) + + require.NotNil(t, res.ResPolicies) + assert.Len(t, res.ResPolicies.GetNamespacedFilterPolicies(), test.expectNamespacedPolicies) + if test.expectClusterScopedPolicy { + assert.NotNil(t, res.ResPolicies.GetClusterScopedFilterPolicy()) + } else { + assert.Nil(t, res.ResPolicies.GetClusterScopedFilterPolicy()) + } + }) + } +} diff --git a/pkg/controller/backup_deletion_controller.go b/pkg/controller/backup_deletion_controller.go index 5fe29c5f11..6f067e6d1b 100644 --- a/pkg/controller/backup_deletion_controller.go +++ b/pkg/controller/backup_deletion_controller.go @@ -23,9 +23,9 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" jsonpatch "github.com/evanphx/json-patch/v5" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -321,6 +321,10 @@ func (r *backupDeletionReconciler) Reconcile(ctx context.Context, req ctrl.Reque volumeSnapshotters[snapshot.Spec.Location] = volumeSnapshotter } + if snapshot.Status.ProviderSnapshotID == "" { + log.WithField("volumeSnapshot", snapshot.Spec.PersistentVolumeName).Warn("Skipping snapshot deletion: empty ProviderSnapshotID") + continue + } if err := volumeSnapshotter.DeleteSnapshot(snapshot.Status.ProviderSnapshotID); err != nil { errs = append(errs, errors.Wrapf(err, "error deleting snapshot %s", snapshot.Status.ProviderSnapshotID).Error()) } diff --git a/pkg/controller/backup_deletion_controller_test.go b/pkg/controller/backup_deletion_controller_test.go index dfca4f137e..3b8307a444 100644 --- a/pkg/controller/backup_deletion_controller_test.go +++ b/pkg/controller/backup_deletion_controller_test.go @@ -397,6 +397,74 @@ func TestBackupDeletionControllerReconcile(t *testing.T) { // Make sure snapshot was deleted assert.Equal(t, 0, td.volumeSnapshotter.SnapshotsTaken.Len()) }) + t.Run("empty ProviderSnapshotID skips DeleteSnapshot call", func(t *testing.T) { + input := defaultTestDbr() + + backup := builder.ForBackup(velerov1api.DefaultNamespace, input.Spec.BackupName).Result() + backup.UID = "uid" + backup.Spec.StorageLocation = "primary" + + restore1 := builder.ForRestore(backup.Namespace, "restore-1"). + Phase(velerov1api.RestorePhaseCompleted). + Backup(backup.Name). + Result() + + location := &velerov1api.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: backup.Namespace, + Name: "primary", + }, + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "objStoreProvider", + StorageType: velerov1api.StorageType{ + ObjectStorage: &velerov1api.ObjectStorageLocation{ + Bucket: "bucket", + }, + }, + }, + Status: velerov1api.BackupStorageLocationStatus{ + Phase: velerov1api.BackupStorageLocationPhaseAvailable, + }, + } + + snapshotLocation := &velerov1api.VolumeSnapshotLocation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: backup.Namespace, + Name: "vsl-1", + }, + Spec: velerov1api.VolumeSnapshotLocationSpec{ + Provider: "provider-1", + }, + } + td := setupBackupDeletionControllerTest(t, input, backup, restore1, location, snapshotLocation) + + snapshots := []*volume.Snapshot{ + { + Spec: volume.SnapshotSpec{ + Location: "vsl-1", + PersistentVolumeName: "pv-1", + }, + Status: volume.SnapshotStatus{ + ProviderSnapshotID: "", + }, + }, + } + + pluginManager := &pluginmocks.Manager{} + pluginManager.On("GetVolumeSnapshotter", "provider-1").Return(td.volumeSnapshotter, nil) + pluginManager.On("GetDeleteItemActions").Return(nil, nil) + pluginManager.On("CleanupClients") + td.controller.newPluginManager = func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager } + + td.backupStore.On("GetBackupVolumeSnapshots", input.Spec.BackupName).Return(snapshots, nil) + td.backupStore.On("GetBackupContents", input.Spec.BackupName).Return(io.NopCloser(bytes.NewReader([]byte("hello world"))), nil) + td.backupStore.On("DeleteBackup", input.Spec.BackupName).Return(nil) + + _, err := td.controller.Reconcile(t.Context(), td.req) + require.NoError(t, err) + + td.backupStore.AssertCalled(t, "DeleteBackup", input.Spec.BackupName) + }) t.Run("full delete, no errors, with backup name greater than 63 chars", func(t *testing.T) { backup := defaultBackup(). ObjectMeta( diff --git a/pkg/controller/backup_finalizer_controller.go b/pkg/controller/backup_finalizer_controller.go index b24c132fa9..2d722ed510 100644 --- a/pkg/controller/backup_finalizer_controller.go +++ b/pkg/controller/backup_finalizer_controller.go @@ -22,7 +22,7 @@ import ( "os" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/controller/backup_operations_controller.go b/pkg/controller/backup_operations_controller.go index 1a5b49c0b2..eda913d3f9 100644 --- a/pkg/controller/backup_operations_controller.go +++ b/pkg/controller/backup_operations_controller.go @@ -24,7 +24,7 @@ import ( v2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go index ab53973db6..b95188186c 100644 --- a/pkg/controller/backup_queue_controller.go +++ b/pkg/controller/backup_queue_controller.go @@ -21,7 +21,7 @@ import ( "slices" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/controller/backup_repository_controller.go b/pkg/controller/backup_repository_controller.go index 16e0b740a4..4da55031d0 100644 --- a/pkg/controller/backup_repository_controller.go +++ b/pkg/controller/backup_repository_controller.go @@ -25,8 +25,8 @@ import ( "slices" "time" + "github.com/cockroachdb/errors" "github.com/petar/GoLLRB/llrb" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/controller/backup_storage_location_controller.go b/pkg/controller/backup_storage_location_controller.go index abcd1e59eb..32c7c69a33 100644 --- a/pkg/controller/backup_storage_location_controller.go +++ b/pkg/controller/backup_storage_location_controller.go @@ -24,7 +24,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/metrics" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" diff --git a/pkg/controller/backup_storage_location_controller_test.go b/pkg/controller/backup_storage_location_controller_test.go index 8a5dc4a4a1..7a99decafd 100644 --- a/pkg/controller/backup_storage_location_controller_test.go +++ b/pkg/controller/backup_storage_location_controller_test.go @@ -22,9 +22,9 @@ import ( "github.com/vmware-tanzu/velero/pkg/metrics" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/pkg/controller/backup_sync_controller.go b/pkg/controller/backup_sync_controller.go index b84ae6f0b4..865d9403ae 100644 --- a/pkg/controller/backup_sync_controller.go +++ b/pkg/controller/backup_sync_controller.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -164,17 +164,37 @@ func (b *backupSyncReconciler) Reconcile(ctx context.Context, req ctrl.Request) continue } - if backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperations || - backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed || - backup.Status.Phase == velerov1api.BackupPhaseFinalizing || - backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { + // Only sync backup metadata that has reached a phase Velero itself writes to + // object storage. Anything else (including an empty or New phase) would be + // created in the cluster as a backup that still looks pending, which the backup + // queue controller would then pick up and run as if it were a newly requested + // backup. + switch backup.Status.Phase { + case velerov1api.BackupPhaseCompleted, + velerov1api.BackupPhasePartiallyFailed, + velerov1api.BackupPhaseFailed: + // finished backups are synced as-is + case velerov1api.BackupPhaseWaitingForPluginOperations, + velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed, + velerov1api.BackupPhaseFinalizing, + velerov1api.BackupPhaseFinalizingPartiallyFailed: if backup.Status.Expiration == nil || backup.Status.Expiration.After(time.Now()) { log.Debugf("Skipping non-expired incomplete backup %v", backup.Name) continue } log.Debugf("%v Backup is past expiration, syncing for garbage collection", backup.Status.Phase) backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed + default: + log.Infof("Skipping backup %v, phase %q in the backup store is not a phase that can be synced", backup.Name, backup.Status.Phase) + continue } + + // A synced backup is a record of a backup that already ran somewhere else, not + // a backup to run here. Hooks are only read while a backup is being executed, + // so they have no consumer for a synced backup and are dropped rather than + // stored as an executable payload. + backup.Spec.Hooks = velerov1api.BackupHooks{} + backup.Namespace = b.namespace backup.ResourceVersion = "" diff --git a/pkg/controller/backup_sync_controller_test.go b/pkg/controller/backup_sync_controller_test.go index fe440ff093..e4d0b138ca 100644 --- a/pkg/controller/backup_sync_controller_test.go +++ b/pkg/controller/backup_sync_controller_test.go @@ -203,10 +203,10 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").Result(), + backup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), }, }, }, @@ -308,10 +308,10 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("velero"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").Result(), + backup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), }, }, }, @@ -321,10 +321,10 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").Result(), + backup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), }, }, existingBackups: []*velerov1api.Backup{ @@ -340,7 +340,7 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").Result(), + backup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), }, }, existingBackups: []*velerov1api.Backup{ @@ -355,10 +355,10 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").StorageLocation("foo").ObjectMeta(builder.WithLabels(velerov1api.StorageLocationLabel, "foo")).Result(), + backup: builder.ForBackup("ns-1", "backup-1").StorageLocation("foo").ObjectMeta(builder.WithLabels(velerov1api.StorageLocationLabel, "foo")).Phase(velerov1api.BackupPhaseCompleted).Result(), }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), }, }, }, @@ -369,10 +369,10 @@ var _ = Describe("Backup Sync Reconciler", func() { longLocationNameEnabled: true, cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").StorageLocation("foo").ObjectMeta(builder.WithLabels(velerov1api.StorageLocationLabel, "foo")).Result(), + backup: builder.ForBackup("ns-1", "backup-1").StorageLocation("foo").ObjectMeta(builder.WithLabels(velerov1api.StorageLocationLabel, "foo")).Phase(velerov1api.BackupPhaseCompleted).Result(), }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), }, }, }, @@ -382,13 +382,13 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").Result(), + backup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-1").Result(), }, }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-2").Result(), }, @@ -401,13 +401,13 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").Result(), + backup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-1").Result(), }, }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-3").Result(), }, @@ -556,6 +556,184 @@ var _ = Describe("Backup Sync Reconciler", func() { } }) + It("Test synced backups are never picked up by the backup queue controller", func() { + fakeClock := testclocks.NewFakeClock(time.Now()) + hooks := velerov1api.BackupHooks{ + Resources: []velerov1api.BackupResourceHookSpec{ + { + Name: "hook-1", + PreHooks: []velerov1api.BackupResourceHook{ + { + Exec: &velerov1api.ExecHook{ + Container: "container-1", + Command: []string{"/bin/sh", "-c", "echo hello"}, + }, + }, + }, + }, + }, + } + + tests := []struct { + name string + cloudBackup *velerov1api.Backup + expectSynced bool + // phase expected in the cluster after the sync and queue reconciles have run. + // only checked when expectSynced is true. + expectPhase velerov1api.BackupPhase + }{ + { + name: "backup metadata with an empty phase is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Hooks(hooks).Result(), + expectSynced: false, + }, + { + name: "backup metadata in phase New is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseNew).Hooks(hooks).Result(), + expectSynced: false, + }, + { + name: "backup metadata in phase Queued is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseQueued).Hooks(hooks).Result(), + expectSynced: false, + }, + { + name: "backup metadata in phase ReadyToStart is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseReadyToStart).Hooks(hooks).Result(), + expectSynced: false, + }, + { + name: "backup metadata in phase InProgress is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseInProgress).Hooks(hooks).Result(), + expectSynced: false, + }, + { + name: "backup metadata in phase Deleting is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseDeleting).Hooks(hooks).Result(), + expectSynced: false, + }, + { + name: "backup metadata in phase Completed is synced and stays Completed", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Hooks(hooks).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhaseCompleted, + }, + { + name: "backup metadata in phase PartiallyFailed is synced and stays PartiallyFailed", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhasePartiallyFailed).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhasePartiallyFailed, + }, + { + name: "backup metadata in phase Failed is synced and stays Failed", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseFailed).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhaseFailed, + }, + { + name: "non-expired backup waiting for plugin operations is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations). + Expiration(fakeClock.Now().Add(time.Hour)).Result(), + expectSynced: false, + }, + { + name: "expired backup waiting for plugin operations is synced as PartiallyFailed", + cloudBackup: builder.ForBackup("ns-1", "backup-1"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhasePartiallyFailed, + }, + { + name: "expired backup waiting for plugin operations partially failed is synced as PartiallyFailed", + cloudBackup: builder.ForBackup("ns-1", "backup-1"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhasePartiallyFailed, + }, + { + name: "expired finalizing backup is synced as PartiallyFailed", + cloudBackup: builder.ForBackup("ns-1", "backup-1"). + Phase(velerov1api.BackupPhaseFinalizing). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhasePartiallyFailed, + }, + { + name: "expired finalizing partially failed backup is synced as PartiallyFailed", + cloudBackup: builder.ForBackup("ns-1", "backup-1"). + Phase(velerov1api.BackupPhaseFinalizingPartiallyFailed). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhasePartiallyFailed, + }, + } + + queueScheme := runtime.NewScheme() + Expect(velerov1api.AddToScheme(queueScheme)).ShouldNot(HaveOccurred()) + + for _, test := range tests { + var ( + client = ctrlfake.NewClientBuilder().Build() + pluginManager = &pluginmocks.Manager{} + backupStores = make(map[string]*persistencemocks.BackupStore) + location = defaultLocation("ns-1") + ) + + pluginManager.On("CleanupClients").Return(nil) + syncReconciler := backupSyncReconciler{ + client: client, + namespace: "ns-1", + defaultBackupSyncPeriod: time.Second * 10, + newPluginManager: func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, + backupStoreGetter: NewFakeObjectBackupStoreGetter(backupStores), + logger: velerotest.NewLogger(), + } + + Expect(client.Create(ctx, location)).ShouldNot(HaveOccurred(), test.name) + backupStore := &persistencemocks.BackupStore{} + backupStores[location.Name] = backupStore + backupStore.On("ListBackups").Return([]string{test.cloudBackup.Name}, nil) + backupStore.On("BackupExists", "bucket-1", test.cloudBackup.Name).Return(true, nil) + backupStore.On("GetBackupMetadata", test.cloudBackup.Name).Return(test.cloudBackup, nil) + backupStore.On("GetPodVolumeBackups", test.cloudBackup.Name).Return(nil, nil) + + _, err := syncReconciler.Reconcile(ctx, ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: location.Namespace, Name: location.Name}, + }) + Expect(err).ShouldNot(HaveOccurred(), test.name) + + backupKey := types.NamespacedName{Namespace: "ns-1", Name: test.cloudBackup.Name} + synced := &velerov1api.Backup{} + err = client.Get(ctx, backupKey, synced) + + if !test.expectSynced { + Expect(apierrors.IsNotFound(err)).To(BeTrue(), test.name) + continue + } + Expect(err).ShouldNot(HaveOccurred(), test.name) + + // Reconcile the synced backup with the queue controller twice: the first + // reconcile would move a New/empty-phase backup to Queued, the second one + // would move it on to ReadyToStart, which is what hands it to the backup + // controller for execution. + queueReconciler := NewBackupQueueReconciler(client, queueScheme, velerotest.NewLogger(), 1, NewBackupTracker()) + for range 2 { + _, err = queueReconciler.Reconcile(ctx, ctrl.Request{NamespacedName: backupKey}) + Expect(err).ShouldNot(HaveOccurred(), test.name) + } + + after := &velerov1api.Backup{} + Expect(client.Get(ctx, backupKey, after)).ShouldNot(HaveOccurred(), test.name) + Expect(after.Status.Phase).To(BeEquivalentTo(test.expectPhase), test.name) + // Hooks are dropped on sync, so the stored metadata cannot carry a payload + // that a later code path could execute. + Expect(after.Spec.Hooks.Resources).To(BeEmpty(), test.name) + } + }) + It("Test deleting orphaned backups.", func() { longLabelName := "the-really-long-location-name-that-is-much-more-than-63-characters" diff --git a/pkg/controller/data_download_controller.go b/pkg/controller/data_download_controller.go index 36fec450be..adcb81a640 100644 --- a/pkg/controller/data_download_controller.go +++ b/pkg/controller/data_download_controller.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -131,6 +131,7 @@ func NewDataDownloadReconciler( // +kubebuilder:rbac:groups="",resources=pods,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumerclaims,verbs=get +// +kubebuilder:rbac:groups="",resources=secrets;configmaps,verbs=get;list;create;delete func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.logger.WithFields(logrus.Fields{ @@ -253,12 +254,12 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request if r.vgdpCounter != nil && r.vgdpCounter.IsConstrained(ctx, r.logger) { log.Debug("Data path initiation is constrained, requeue later") - return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil + return ctrl.Result{RequeueAfter: time.Second * 5}, nil } if _, err := r.getTargetPVC(ctx, dd); err != nil { log.WithField("error", err).Debugf("Cannot find target PVC for DataDownload yet. Retry later.") - return ctrl.Result{Requeue: true}, nil + return ctrl.Result{RequeueAfter: time.Second * 5}, nil } log.Info("Data download starting") @@ -349,7 +350,7 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request if err != nil { if err == datapath.ConcurrentLimitExceed { log.Debug("Data path instance is concurrent limited requeue later") - return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil + return ctrl.Result{RequeueAfter: time.Second * 5}, nil } else { return r.errorOut(ctx, dd, err, "error to create data path", log) } @@ -380,7 +381,7 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request log.WithError(err).Warnf("Failed to update datadownload %s to InProgress, will data path close and retry", dd.Name) r.closeDataPath(ctx, dd.Name) - return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil + return ctrl.Result{RequeueAfter: time.Second * 5}, nil } if terminated { @@ -688,11 +689,11 @@ func (r *DataDownloadReconciler) findSnapshotRestoreForPod(ctx context.Context, r.prepareDataDownload(dd) return true }); err != nil { - log.WithError(err).Warn("failed to update dataudownload, prepare will halt for this dataudownload") + log.WithError(err).Warn("failed to update datadownload, prepare will halt for this datadownload") return []reconcile.Request{} } } else if unrecoverable, reason := kube.IsPodUnrecoverable(pod, log); unrecoverable { - err := UpdateDataDownloadWithRetry(context.Background(), r.client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, r.logger.WithField("datadownlad", dd.Name), + err := UpdateDataDownloadWithRetry(context.Background(), r.client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, r.logger.WithField("datadownload", dd.Name), func(dataDownload *velerov2alpha1api.DataDownload) bool { if dataDownload.Spec.Cancel { return false diff --git a/pkg/controller/data_download_controller_test.go b/pkg/controller/data_download_controller_test.go index 397f931c08..a9ce47e28a 100644 --- a/pkg/controller/data_download_controller_test.go +++ b/pkg/controller/data_download_controller_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -315,12 +315,12 @@ func TestDataDownloadReconcile(t *testing.T) { dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), constrained: true, expected: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), - expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, + expectedResult: &ctrl.Result{RequeueAfter: time.Second * 5}, }, { name: "new dd but no target PVC", dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), - expectedResult: &ctrl.Result{Requeue: true}, + expectedResult: &ctrl.Result{RequeueAfter: time.Second * 5}, }, { name: "new dd but accept failed", @@ -390,7 +390,7 @@ func TestDataDownloadReconcile(t *testing.T) { dataMgr: datapath.NewManager(0), notNilExpose: true, notMockCleanUp: true, - expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, + expectedResult: &ctrl.Result{RequeueAfter: time.Second * 5}, }, { name: "data path init error", @@ -594,7 +594,6 @@ func TestDataDownloadReconcile(t *testing.T) { } if test.expectedResult != nil { - assert.Equal(t, test.expectedResult.Requeue, actualResult.Requeue) assert.Equal(t, test.expectedResult.RequeueAfter, actualResult.RequeueAfter) } diff --git a/pkg/controller/data_upload_controller.go b/pkg/controller/data_upload_controller.go index 13be039947..ba6cf997d6 100644 --- a/pkg/controller/data_upload_controller.go +++ b/pkg/controller/data_upload_controller.go @@ -22,8 +22,8 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -140,6 +140,7 @@ func NewDataUploadReconciler( // +kubebuilder:rbac:groups="",resources=pods,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumerclaims,verbs=get +// +kubebuilder:rbac:groups="",resources=secrets;configmaps,verbs=get;list;create;delete func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.logger.WithFields(logrus.Fields{ @@ -264,7 +265,7 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) if r.vgdpCounter != nil && r.vgdpCounter.IsConstrained(ctx, r.logger) { log.Debug("Data path initiation is constrained, requeue later") - return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil + return ctrl.Result{RequeueAfter: time.Second * 5}, nil } log.Info("Data upload starting") @@ -358,7 +359,7 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) if err != nil { if err == datapath.ConcurrentLimitExceed { log.Debug("Data path instance is concurrent limited requeue later") - return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil + return ctrl.Result{RequeueAfter: time.Second * 5}, nil } else { return r.errorOut(ctx, du, err, "error to create data path", log) } @@ -390,7 +391,7 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) log.WithError(err).Warnf("Failed to update dataupload %s to InProgress, will data path close and retry", du.Name) r.closeDataPath(ctx, du.Name) - return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil + return ctrl.Result{RequeueAfter: time.Second * 5}, nil } if terminated { diff --git a/pkg/controller/data_upload_controller_test.go b/pkg/controller/data_upload_controller_test.go index a73e3de943..7c8b4c4431 100644 --- a/pkg/controller/data_upload_controller_test.go +++ b/pkg/controller/data_upload_controller_test.go @@ -22,9 +22,9 @@ import ( "testing" "time" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" snapshotFake "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/fake" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -478,7 +478,7 @@ func TestReconcile(t *testing.T) { du: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), constrained: true, expected: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Result(), - expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, + expectedResult: &ctrl.Result{RequeueAfter: time.Second * 5}, }, { name: "new du but accept failed", @@ -548,7 +548,7 @@ func TestReconcile(t *testing.T) { name: "Error in data path is concurrent limited", du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).SnapshotType(fakeSnapshotType).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), dataMgr: datapath.NewManager(0), - expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, + expectedResult: &ctrl.Result{RequeueAfter: time.Second * 5}, }, { name: "data path init error", @@ -562,7 +562,7 @@ func TestReconcile(t *testing.T) { du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).SnapshotType(fakeSnapshotType).Finalizers([]string{DataUploadDownloadFinalizer}).Node("test-node").Result(), needErrs: []bool{false, false, true, false}, expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).Finalizers([]string{DataUploadDownloadFinalizer}).Result(), - expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, + expectedResult: &ctrl.Result{RequeueAfter: time.Second * 5}, }, { name: "data path start error", @@ -694,7 +694,6 @@ func TestReconcile(t *testing.T) { } if test.expectedResult != nil { - assert.Equal(t, test.expectedResult.Requeue, actualResult.Requeue) assert.Equal(t, test.expectedResult.RequeueAfter, actualResult.RequeueAfter) } diff --git a/pkg/controller/download_request_controller.go b/pkg/controller/download_request_controller.go index bd95658952..02d385bec5 100644 --- a/pkg/controller/download_request_controller.go +++ b/pkg/controller/download_request_controller.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/controller/gc_controller.go b/pkg/controller/gc_controller.go index bb3c60ae55..6b3ade484f 100644 --- a/pkg/controller/gc_controller.go +++ b/pkg/controller/gc_controller.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" clocks "k8s.io/utils/clock" diff --git a/pkg/controller/pod_volume_backup_controller.go b/pkg/controller/pod_volume_backup_controller.go index 0bcbfa6d27..b16e914dcf 100644 --- a/pkg/controller/pod_volume_backup_controller.go +++ b/pkg/controller/pod_volume_backup_controller.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -238,7 +238,7 @@ func (r *PodVolumeBackupReconciler) Reconcile(ctx context.Context, req ctrl.Requ if r.vgdpCounter != nil && r.vgdpCounter.IsConstrained(ctx, r.logger) { log.Debug("Data path initiation is constrained, requeue later") - return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil + return ctrl.Result{RequeueAfter: time.Second * 5}, nil } log.Info("Accepting PVB") @@ -314,7 +314,7 @@ func (r *PodVolumeBackupReconciler) Reconcile(ctx context.Context, req ctrl.Requ if err != nil { if err == datapath.ConcurrentLimitExceed { log.Debug("Data path instance is concurrent limited requeue later") - return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil + return ctrl.Result{RequeueAfter: time.Second * 5}, nil } else { return r.errorOut(ctx, pvb, err, "error to create data path", log) } @@ -346,7 +346,7 @@ func (r *PodVolumeBackupReconciler) Reconcile(ctx context.Context, req ctrl.Requ log.WithError(err).Warnf("Failed to update PVB %s to InProgress, will data path close and retry", pvb.Name) r.closeDataPath(ctx, pvb.Name) - return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil + return ctrl.Result{RequeueAfter: time.Second * 5}, nil } if terminated { diff --git a/pkg/controller/pod_volume_backup_controller_test.go b/pkg/controller/pod_volume_backup_controller_test.go index b49d7eb5b4..9161fd4bbe 100644 --- a/pkg/controller/pod_volume_backup_controller_test.go +++ b/pkg/controller/pod_volume_backup_controller_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -327,7 +327,7 @@ func TestPVBReconcile(t *testing.T) { pvb: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), constrained: true, expected: pvbBuilder().Finalizers([]string{PodVolumeFinalizer}).Result(), - expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, + expectedResult: &ctrl.Result{RequeueAfter: time.Second * 5}, }, { name: "new pvb but accept failed", @@ -394,7 +394,7 @@ func TestPVBReconcile(t *testing.T) { pvb: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), needMockExposer: true, dataMgr: datapath.NewManager(0), - expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, + expectedResult: &ctrl.Result{RequeueAfter: time.Second * 5}, }, { name: "data path init error", @@ -410,7 +410,7 @@ func TestPVBReconcile(t *testing.T) { needMockExposer: true, needErrs: []bool{false, false, true, false}, expected: pvbBuilder().Phase(velerov1api.PodVolumeBackupPhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Result(), - expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, + expectedResult: &ctrl.Result{RequeueAfter: time.Second * 5}, }, { name: "data path start error", @@ -537,7 +537,6 @@ func TestPVBReconcile(t *testing.T) { } if test.expectedResult != nil { - assert.Equal(t, test.expectedResult.Requeue, actualResult.Requeue) assert.Equal(t, test.expectedResult.RequeueAfter, actualResult.RequeueAfter) } diff --git a/pkg/controller/pod_volume_restore_controller.go b/pkg/controller/pod_volume_restore_controller.go index 0af0d8c868..a68c939286 100644 --- a/pkg/controller/pod_volume_restore_controller.go +++ b/pkg/controller/pod_volume_restore_controller.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -246,7 +246,7 @@ func (r *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Req if r.vgdpCounter != nil && r.vgdpCounter.IsConstrained(ctx, r.logger) { log.Debug("Data path initiation is constrained, requeue later") - return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil + return ctrl.Result{RequeueAfter: time.Second * 5}, nil } log.Info("Accepting PVR") @@ -328,7 +328,7 @@ func (r *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Req if err != nil { if err == datapath.ConcurrentLimitExceed { log.Debug("Data path instance is concurrent limited requeue later") - return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil + return ctrl.Result{RequeueAfter: time.Second * 5}, nil } else { return r.errorOut(ctx, pvr, err, "error to create data path", log) } @@ -358,7 +358,7 @@ func (r *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Req log.WithError(err).Warnf("Failed to update PVR %s to InProgress, will data path close and retry", pvr.Name) r.closeDataPath(ctx, pvr.Name) - return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil + return ctrl.Result{RequeueAfter: time.Second * 5}, nil } if terminated { diff --git a/pkg/controller/pod_volume_restore_controller_legacy.go b/pkg/controller/pod_volume_restore_controller_legacy.go index 731b70db90..9c952e6521 100644 --- a/pkg/controller/pod_volume_restore_controller_legacy.go +++ b/pkg/controller/pod_volume_restore_controller_legacy.go @@ -23,7 +23,7 @@ import ( "path/filepath" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -150,7 +150,7 @@ func (c *PodVolumeRestoreReconcilerLegacy) Reconcile(ctx context.Context, req ct fsRestore, err := c.dataPathMgr.CreateFileSystemBR(pvr.Name, pVBRRequestor, ctx, c.Client, pvr.Namespace, callbacks, log) if err != nil { if err == datapath.ConcurrentLimitExceed { - return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil + return ctrl.Result{RequeueAfter: time.Second * 5}, nil } else { return c.errorOut(ctx, pvr, err, "error to create data path", log) } diff --git a/pkg/controller/pod_volume_restore_controller_test.go b/pkg/controller/pod_volume_restore_controller_test.go index 9f2fe7a7f7..e2b41e0f4b 100644 --- a/pkg/controller/pod_volume_restore_controller_test.go +++ b/pkg/controller/pod_volume_restore_controller_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -790,7 +790,7 @@ func TestPodVolumeRestoreReconcile(t *testing.T) { targetPod: builder.ForPod("test-ns", "test-pod").InitContainers(&corev1api.Container{Name: restorehelper.WaitInitContainer}).InitContainerState(corev1api.ContainerState{Running: &corev1api.ContainerStateRunning{}}).Result(), constrained: true, expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Result(), - expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, + expectedResult: &ctrl.Result{RequeueAfter: time.Second * 5}, }, { name: "new pvr but accept failed", @@ -858,7 +858,7 @@ func TestPodVolumeRestoreReconcile(t *testing.T) { dataMgr: datapath.NewManager(0), notNilExpose: true, notMockCleanUp: true, - expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, + expectedResult: &ctrl.Result{RequeueAfter: time.Second * 5}, }, { name: "data path init error", @@ -1057,7 +1057,6 @@ func TestPodVolumeRestoreReconcile(t *testing.T) { } if test.expectedResult != nil { - assert.Equal(t, test.expectedResult.Requeue, actualResult.Requeue) assert.Equal(t, test.expectedResult.RequeueAfter, actualResult.RequeueAfter) } diff --git a/pkg/controller/restore_controller.go b/pkg/controller/restore_controller.go index 469951f273..0717d1d827 100644 --- a/pkg/controller/restore_controller.go +++ b/pkg/controller/restore_controller.go @@ -28,7 +28,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -44,6 +44,7 @@ import ( "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/internal/resourcemodifiers" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/constant" @@ -232,7 +233,7 @@ func (r *restoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct original := restore.DeepCopy() // Validate the restore and fetch the backup - info, resourceModifiers := r.validateAndComplete(restore) + info, resourceModifiers, restoreResPolicies := r.validateAndComplete(ctx, restore) // Register attempts after validation so we don't have to fetch the backup multiple times backupScheduleName := restore.Spec.ScheduleName @@ -267,7 +268,7 @@ func (r *restoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, nil } - if err := r.runValidatedRestore(restore, info, resourceModifiers); err != nil { + if err := r.runValidatedRestore(restore, info, resourceModifiers, restoreResPolicies); err != nil { log.WithError(err).Debug("Restore failed") restore.Status.Phase = api.RestorePhaseFailed restore.Status.FailureReason = err.Error() @@ -303,7 +304,7 @@ func (r *restoreReconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } -func (r *restoreReconciler) validateAndComplete(restore *api.Restore) (backupInfo, *resourcemodifiers.ResourceModifiers) { +func (r *restoreReconciler) validateAndComplete(ctx context.Context, restore *api.Restore) (backupInfo, *resourcemodifiers.ResourceModifiers, *resourcepolicies.Policies) { // add non-restorable resources to restore's excluded resources excludedResources := sets.NewString(restore.Spec.ExcludedResources...) for _, nonrestorable := range nonRestorableResources { @@ -338,7 +339,7 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) (backupInf // validate that exactly one of BackupName and ScheduleName have been specified if !backupXorScheduleProvided(restore) { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "Either a backup or schedule must be specified as a source for the restore, but not both") - return backupInfo{}, nil + return backupInfo{}, nil, nil } // validate Restore Init Hook's InitContainers @@ -372,9 +373,9 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) (backupInf })) backupList := &api.BackupList{} - if err := r.kbClient.List(context.Background(), backupList, &client.ListOptions{LabelSelector: selector}); err != nil { + if err := r.kbClient.List(ctx, backupList, &client.ListOptions{LabelSelector: selector}); err != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "Unable to list backups for schedule") - return backupInfo{}, nil + return backupInfo{}, nil, nil } if len(backupList.Items) == 0 { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "No backups found for schedule") @@ -384,19 +385,19 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) (backupInf restore.Spec.BackupName = backup.Name } else { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "No completed backups found for schedule") - return backupInfo{}, nil + return backupInfo{}, nil, nil } } info, err := r.fetchBackupInfo(restore.Spec.BackupName) if err != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Error retrieving backup: %v", err)) - return backupInfo{}, nil + return backupInfo{}, nil, nil } if !veleroutil.BSLIsAvailable(*info.location) { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("The BSL %s is unavailable, cannot retrieve the backup", info.location.Name)) - return backupInfo{}, nil + return backupInfo{}, nil, nil } // Fill in the ScheduleName so it's easier to consume for metrics. @@ -404,26 +405,40 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) (backupInf restore.Spec.ScheduleName = info.backup.GetLabels()[api.ScheduleNameLabel] } + var restoreResPolicies *resourcepolicies.Policies + if restore.Spec.ResourcePolicy != nil { + var err error + restoreResPolicies, err = resourcepolicies.GetResourcePoliciesFromRestore( + ctx, restore, r.kbClient, r.logger, + ) + if err != nil { + restore.Status.ValidationErrors = append( + restore.Status.ValidationErrors, err.Error(), + ) + return backupInfo{}, nil, nil + } + } + var resourceModifiers *resourcemodifiers.ResourceModifiers if restore.Spec.ResourceModifier != nil && strings.EqualFold(restore.Spec.ResourceModifier.Kind, resourcemodifiers.ConfigmapRefType) { ResourceModifierConfigMap := &corev1api.ConfigMap{} - err := r.kbClient.Get(context.Background(), client.ObjectKey{Namespace: restore.Namespace, Name: restore.Spec.ResourceModifier.Name}, ResourceModifierConfigMap) + err := r.kbClient.Get(ctx, client.ObjectKey{Namespace: restore.Namespace, Name: restore.Spec.ResourceModifier.Name}, ResourceModifierConfigMap) if err != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("failed to get resource modifiers configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name)) - return backupInfo{}, nil + return backupInfo{}, nil, nil } resourceModifiers, err = resourcemodifiers.GetResourceModifiersFromConfig(ResourceModifierConfigMap) if err != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, errors.Wrapf(err, "Error in parsing resource modifiers provided in configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name).Error()) - return backupInfo{}, nil + return backupInfo{}, nil, nil } else if err = resourceModifiers.Validate(); err != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, errors.Wrapf(err, "Validation error in resource modifiers provided in configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name).Error()) - return backupInfo{}, nil + return backupInfo{}, nil, nil } r.logger.Infof("Retrieved Resource modifiers provided in configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name) } - return info, resourceModifiers + return info, resourceModifiers, restoreResPolicies } // backupXorScheduleProvided returns true if exactly one of BackupName and @@ -496,7 +511,7 @@ func fetchBackupInfoInternal(kbClient client.Client, namespace, backupName strin // The log and results files are uploaded to backup storage. Any error returned from this function // means that the restore failed. This function updates the restore API object with warning and error // counts, but *does not* update its phase or patch it via the API. -func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backupInfo, resourceModifiers *resourcemodifiers.ResourceModifiers) error { +func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backupInfo, resourceModifiers *resourcemodifiers.ResourceModifiers, restoreResPolicies *resourcepolicies.Policies) error { // instantiate the per-restore logger that will output both to a temp file // (for upload to object storage) and to stdout. restoreLog, err := logging.NewTempFileLogger(r.restoreLogLevel, r.logFormat, nil, logrus.Fields{"restore": kubeutil.NamespaceAndName(restore)}) @@ -575,6 +590,7 @@ func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backu VolumeSnapshots: volumeSnapshots, BackupReader: backupFile, ResourceModifiers: resourceModifiers, + ResPolicies: restoreResPolicies, DisableInformerCache: r.disableInformerCache, CSIVolumeSnapshots: csiVolumeSnapshots, BackupVolumeInfoMap: backupVolumeInfoMap, diff --git a/pkg/controller/restore_controller_test.go b/pkg/controller/restore_controller_test.go index b013ee64db..c591a29c62 100644 --- a/pkg/controller/restore_controller_test.go +++ b/pkg/controller/restore_controller_test.go @@ -22,8 +22,8 @@ import ( "testing" "time" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -747,7 +747,7 @@ func TestValidateAndCompleteWhenScheduleNameSpecified(t *testing.T) { Phase(velerov1api.BackupPhaseCompleted). Result())) - r.validateAndComplete(restore) + r.validateAndComplete(t.Context(), restore) assert.Contains(t, restore.Status.ValidationErrors, "No backups found for schedule") assert.Empty(t, restore.Spec.BackupName) @@ -763,7 +763,7 @@ func TestValidateAndCompleteWhenScheduleNameSpecified(t *testing.T) { Result(), )) - r.validateAndComplete(restore) + r.validateAndComplete(t.Context(), restore) assert.Contains(t, restore.Status.ValidationErrors, "No completed backups found for schedule") assert.Empty(t, restore.Spec.BackupName) @@ -794,11 +794,140 @@ func TestValidateAndCompleteWhenScheduleNameSpecified(t *testing.T) { ScheduleName: "schedule-1", }, } - r.validateAndComplete(restore) + r.validateAndComplete(t.Context(), restore) assert.Nil(t, restore.Status.ValidationErrors) assert.Equal(t, "foo", restore.Spec.BackupName) } +func TestValidateAndCompleteWithResourcePolicySpecified(t *testing.T) { + formatFlag := logging.FormatText + + var ( + logger = velerotest.NewLogger() + pluginManager = &pluginmocks.Manager{} + fakeClient = velerotest.NewFakeControllerRuntimeClient(t) + fakeGlobalClient = velerotest.NewFakeControllerRuntimeClient(t) + backupStore = &persistencemocks.BackupStore{} + ) + + r := NewRestoreReconciler( + t.Context(), + velerov1api.DefaultNamespace, + nil, + fakeClient, + logger, + logrus.DebugLevel, + func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, + NewFakeSingleObjectBackupStoreGetter(backupStore), + metrics.NewServerMetrics(), + formatFlag, + 60*time.Minute, + false, + fakeGlobalClient, + 10*time.Minute, + ) + + restore := &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1api.DefaultNamespace, + Name: "restore-1", + }, + Spec: velerov1api.RestoreSpec{ + BackupName: "backup-1", + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: "configmap", + Name: "test-configmap", + }, + }, + } + + location := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() + require.NoError(t, r.kbClient.Create(t.Context(), location)) + + require.NoError(t, r.kbClient.Create( + t.Context(), + defaultBackup(). + ObjectMeta( + builder.WithName("backup-1"), + ).StorageLocation("default"). + Phase(velerov1api.BackupPhaseCompleted). + Result(), + )) + + r.validateAndComplete(t.Context(), restore) + assert.Contains(t, restore.Status.ValidationErrors[0], "fail to get ResourcePolicies velero/test-configmap ConfigMap") + + restore1 := &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1api.DefaultNamespace, + Name: "restore-1", + }, + Spec: velerov1api.RestoreSpec{ + BackupName: "backup-1", + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: "configmap", + Name: "test-configmap", + }, + }, + } + + cm1 := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: velerov1api.DefaultNamespace, + }, + Data: map[string]string{ + "policy.yaml": `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: + - pods +`, + }, + } + require.NoError(t, r.kbClient.Create(t.Context(), cm1)) + + r.validateAndComplete(t.Context(), restore1) + assert.Nil(t, restore1.Status.ValidationErrors) + + restore2 := &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1api.DefaultNamespace, + Name: "restore-1", + }, + Spec: velerov1api.RestoreSpec{ + BackupName: "backup-1", + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + // intentional to ensure case insensitivity works as expected + Kind: "confIGMaP", + Name: "test-configmap-invalid", + }, + }, + } + + cm2 := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap-invalid", + Namespace: velerov1api.DefaultNamespace, + }, + Data: map[string]string{ + "policy.yaml": `version: v1 +volumePolicies: + - conditions: + capacity: '0,10Gi' + csi: + driver: disks.csi.driver + action: + type: invalid_action +`, + }, + } + require.NoError(t, r.kbClient.Create(t.Context(), cm2)) + + r.validateAndComplete(t.Context(), restore2) + assert.Contains(t, restore2.Status.ValidationErrors[0], "fail to validate ResourcePolicies in ConfigMap velero/test-configmap-invalid") +} + func TestValidateAndCompleteWithResourceModifierSpecified(t *testing.T) { formatFlag := logging.FormatText @@ -854,7 +983,7 @@ func TestValidateAndCompleteWithResourceModifierSpecified(t *testing.T) { Result(), )) - r.validateAndComplete(restore) + r.validateAndComplete(t.Context(), restore) assert.Contains(t, restore.Status.ValidationErrors[0], "failed to get resource modifiers configmap") restore1 := &velerov1api.Restore{ @@ -882,7 +1011,7 @@ func TestValidateAndCompleteWithResourceModifierSpecified(t *testing.T) { } require.NoError(t, r.kbClient.Create(t.Context(), cm1)) - r.validateAndComplete(restore1) + r.validateAndComplete(t.Context(), restore1) assert.Nil(t, restore1.Status.ValidationErrors) restore2 := &velerov1api.Restore{ @@ -911,7 +1040,7 @@ func TestValidateAndCompleteWithResourceModifierSpecified(t *testing.T) { } require.NoError(t, r.kbClient.Create(t.Context(), invalidVersionCm)) - r.validateAndComplete(restore2) + r.validateAndComplete(t.Context(), restore2) assert.Contains(t, restore2.Status.ValidationErrors[0], "Error in parsing resource modifiers provided in configmap") restore3 := &velerov1api.Restore{ @@ -939,7 +1068,7 @@ func TestValidateAndCompleteWithResourceModifierSpecified(t *testing.T) { } require.NoError(t, r.kbClient.Create(t.Context(), invalidOperatorCm)) - r.validateAndComplete(restore3) + r.validateAndComplete(t.Context(), restore3) assert.Contains(t, restore3.Status.ValidationErrors[0], "Validation error in resource modifiers provided in configmap") } diff --git a/pkg/controller/restore_finalizer_controller.go b/pkg/controller/restore_finalizer_controller.go index 93652c0c28..3ff4337bfa 100644 --- a/pkg/controller/restore_finalizer_controller.go +++ b/pkg/controller/restore_finalizer_controller.go @@ -22,9 +22,9 @@ import ( "sync" "time" + "github.com/cockroachdb/errors" volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" storagev1api "k8s.io/api/storage/v1" @@ -39,6 +39,7 @@ import ( "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + serverconfig "github.com/vmware-tanzu/velero/pkg/cmd/server/config" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/metrics" @@ -570,8 +571,18 @@ func (ctx *finalizerContext) WaitRestoreExecHook() (errs results.Result) { log := ctx.logger.WithField("restore", ctx.restore.Name) log.Info("Waiting for restore exec hooks starts") - // wait for restore exec hooks to finish - err := wait.PollUntilContextCancel(context.Background(), 1*time.Second, true, func(context.Context) (bool, error) { + // Bound the wait by resourceTimeout (the same budget Velero already + // applies to other finalizer phases). Previously this poll had no + // deadline, so a hook that was registered via Add() but never + // recorded as executed left the restore stuck in Finalizing forever + // and blocked every other restore on the cluster. + timeout := ctx.resourceTimeout + if timeout <= 0 { + timeout = serverconfig.DefaultResourceTimeout + } + pollCtx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + err := wait.PollUntilContextCancel(pollCtx, 1*time.Second, true, func(context.Context) (bool, error) { log.Debug("Checking the progress of hooks execution") if ctx.multiHookTracker.IsComplete(ctx.restore.Name) { return true, nil diff --git a/pkg/controller/restore_finalizer_controller_test.go b/pkg/controller/restore_finalizer_controller_test.go index f07d2576cf..5e72955500 100644 --- a/pkg/controller/restore_finalizer_controller_test.go +++ b/pkg/controller/restore_finalizer_controller_test.go @@ -482,6 +482,10 @@ func TestWaitRestoreExecHook(t *testing.T) { hookFailed, hookErr := true, fmt.Errorf("hook failed") hookTracker3.Add(restoreName3, podNs, podName, container, source, hookName, hook.PhasePre, 0) + hookTracker4 := hook.NewMultiHookTracker() + restoreName4 := "restore4" + hookTracker4.Add(restoreName4, "ns", "pod", "con1", "s1", "h1", hook.PhasePre, 0) + tests := []struct { name string hookTracker *hook.MultiHookTracker @@ -497,6 +501,8 @@ func TestWaitRestoreExecHook(t *testing.T) { hookName string hookFailed bool hookErr error + resourceTimeout time.Duration + expectTimeoutErr bool }{ { name: "no restore exec hooks", @@ -530,6 +536,16 @@ func TestWaitRestoreExecHook(t *testing.T) { hookFailed: hookFailed, hookErr: hookErr, }, + { + name: "hook never recorded should timeout instead of hanging", + hookTracker: hookTracker4, + restore: builder.ForRestore(velerov1api.DefaultNamespace, restoreName4).Result(), + expectedHooksAttempted: 0, + expectedHooksFailed: 0, + expectedHookErrs: 1, + resourceTimeout: 3 * time.Second, + expectTimeoutErr: true, + }, } for _, tc := range tests { @@ -542,6 +558,7 @@ func TestWaitRestoreExecHook(t *testing.T) { crClient: fakeClient, restore: tc.restore, multiHookTracker: tc.hookTracker, + resourceTimeout: tc.resourceTimeout, } require.NoError(t, ctx.crClient.Create(t.Context(), tc.restore)) @@ -553,6 +570,10 @@ func TestWaitRestoreExecHook(t *testing.T) { } errs := ctx.WaitRestoreExecHook() + if tc.expectTimeoutErr { + assert.NotEmpty(t, errs.Namespaces, "expected timeout error but got none") + continue + } assert.Len(t, errs.Namespaces, tc.expectedHookErrs) updated := &velerov1api.Restore{} diff --git a/pkg/controller/restore_operations_controller.go b/pkg/controller/restore_operations_controller.go index 0539e21a40..301e5f438d 100644 --- a/pkg/controller/restore_operations_controller.go +++ b/pkg/controller/restore_operations_controller.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/controller/schedule_controller.go b/pkg/controller/schedule_controller.go index ec8894571c..c299008887 100644 --- a/pkg/controller/schedule_controller.go +++ b/pkg/controller/schedule_controller.go @@ -21,8 +21,8 @@ import ( "fmt" "time" - "github.com/pkg/errors" - cron "github.com/robfig/cron/v3" + "github.com/cockroachdb/errors" + cron "github.com/netresearch/go-cron" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/controller/schedule_controller_test.go b/pkg/controller/schedule_controller_test.go index f4585763ca..85b87474aa 100644 --- a/pkg/controller/schedule_controller_test.go +++ b/pkg/controller/schedule_controller_test.go @@ -20,14 +20,14 @@ import ( "testing" "time" - cron "github.com/robfig/cron/v3" + cron "github.com/netresearch/go-cron" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" testclocks "k8s.io/utils/clock/testing" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -94,7 +94,7 @@ func TestReconcileOfSchedule(t *testing.T) { }, { name: "schedule with phase New and SkipImmediately gets validated and does not trigger a backup", - schedule: newScheduleBuilder(velerov1.SchedulePhaseNew).CronSchedule("@every 5m").SkipImmediately(pointer.Bool(true)).Result(), + schedule: newScheduleBuilder(velerov1.SchedulePhaseNew).CronSchedule("@every 5m").SkipImmediately(ptr.To(true)).Result(), fakeClockTime: "2017-01-01 12:00:00", expectedPhase: string(velerov1.SchedulePhaseEnabled), expectedLastSkipped: "2017-01-01 12:00:00", @@ -123,7 +123,7 @@ func TestReconcileOfSchedule(t *testing.T) { }, { name: "schedule that's already run but has SkippedImmediately=false gets LastBackup updated", - schedule: newScheduleBuilder(velerov1.SchedulePhaseEnabled).CronSchedule("@every 5m").LastBackupTime("2000-01-01 00:00:00").SkipImmediately(pointer.Bool(false)).Result(), + schedule: newScheduleBuilder(velerov1.SchedulePhaseEnabled).CronSchedule("@every 5m").LastBackupTime("2000-01-01 00:00:00").SkipImmediately(ptr.To(false)).Result(), fakeClockTime: "2017-01-01 12:00:00", expectedBackupCreate: builder.ForBackup("ns", "name-20170101120000").ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "name")).Result(), expectedLastBackup: "2017-01-01 12:00:00", @@ -138,7 +138,7 @@ func TestReconcileOfSchedule(t *testing.T) { }, { name: "schedule that's already run but has SkippedImmediately=true do not get LastBackup updated", - schedule: newScheduleBuilder(velerov1.SchedulePhaseEnabled).CronSchedule("@every 5m").LastBackupTime("2000-01-01 00:00:00").SkipImmediately(pointer.Bool(true)).Result(), + schedule: newScheduleBuilder(velerov1.SchedulePhaseEnabled).CronSchedule("@every 5m").LastBackupTime("2000-01-01 00:00:00").SkipImmediately(ptr.To(true)).Result(), fakeClockTime: "2017-01-01 12:00:00", expectedLastBackup: "2000-01-01 00:00:00", expectedLastSkipped: "2017-01-01 12:00:00", @@ -216,7 +216,7 @@ func TestReconcileOfSchedule(t *testing.T) { // we expect reconcile to flip SkipImmediately to false if it's true or the server is configured to skip immediately and the schedule doesn't have it set if scheduleb4reconcile.Spec.SkipImmediately != nil && *scheduleb4reconcile.Spec.SkipImmediately || test.reconcilerSkipImmediately && scheduleb4reconcile.Spec.SkipImmediately == nil { - assert.Equal(t, schedule.Spec.SkipImmediately, pointer.Bool(false)) + assert.Equal(t, schedule.Spec.SkipImmediately, ptr.To(false)) } backups := &velerov1.BackupList{} diff --git a/pkg/controller/server_status_request_controller.go b/pkg/controller/server_status_request_controller.go index 3fb1af80bd..c779f6f1b6 100644 --- a/pkg/controller/server_status_request_controller.go +++ b/pkg/controller/server_status_request_controller.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/controller/server_status_request_controller_test.go b/pkg/controller/server_status_request_controller_test.go index eb95c7e87e..b41939a80a 100644 --- a/pkg/controller/server_status_request_controller_test.go +++ b/pkg/controller/server_status_request_controller_test.go @@ -125,7 +125,7 @@ var _ = Describe("Server Status Request Reconciler", func() { }, }). Result(), - expectedRequeue: ctrl.Result{Requeue: false, RequeueAfter: statusRequestResyncPeriod}, + expectedRequeue: ctrl.Result{RequeueAfter: statusRequestResyncPeriod}, }), Entry("with phase=new will be processed and phased successfully patched", request{ req: statusRequestBuilder("1"). @@ -158,7 +158,7 @@ var _ = Describe("Server Status Request Reconciler", func() { }, }). Result(), - expectedRequeue: ctrl.Result{Requeue: false, RequeueAfter: statusRequestResyncPeriod}, + expectedRequeue: ctrl.Result{RequeueAfter: statusRequestResyncPeriod}, }), Entry("with phase=Processed does not get deleted if not expired", request{ req: statusRequestBuilder("1"). @@ -191,7 +191,7 @@ var _ = Describe("Server Status Request Reconciler", func() { }, }). Result(), - expectedRequeue: ctrl.Result{Requeue: false, RequeueAfter: statusRequestResyncPeriod}, + expectedRequeue: ctrl.Result{RequeueAfter: statusRequestResyncPeriod}, }), Entry("with phase=Processed gets deleted if expired", request{ req: statusRequestBuilder("1"). @@ -214,7 +214,7 @@ var _ = Describe("Server Status Request Reconciler", func() { }, }, expected: nil, - expectedRequeue: ctrl.Result{Requeue: false, RequeueAfter: statusRequestResyncPeriod}, + expectedRequeue: ctrl.Result{RequeueAfter: statusRequestResyncPeriod}, }), Entry("with invalid phase returns an error and does not requeue", request{ req: statusRequestBuilder("1"). @@ -237,7 +237,7 @@ var _ = Describe("Server Status Request Reconciler", func() { }, }, expectedErrMsg: "unexpected ServerStatusRequest phase", - expectedRequeue: ctrl.Result{Requeue: false, RequeueAfter: 0}, + expectedRequeue: ctrl.Result{RequeueAfter: 0}, }), ) }) diff --git a/pkg/datamover/backup_micro_service.go b/pkg/datamover/backup_micro_service.go index 61c308f15d..dcc1addc58 100644 --- a/pkg/datamover/backup_micro_service.go +++ b/pkg/datamover/backup_micro_service.go @@ -21,7 +21,7 @@ import ( "encoding/json" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" diff --git a/pkg/datamover/backup_micro_service_test.go b/pkg/datamover/backup_micro_service_test.go index c5a9a44871..e39edcfb4a 100644 --- a/pkg/datamover/backup_micro_service_test.go +++ b/pkg/datamover/backup_micro_service_test.go @@ -23,7 +23,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/datamover/dataupload_delete_action.go b/pkg/datamover/dataupload_delete_action.go index 6b36e10680..d1fa5132b2 100644 --- a/pkg/datamover/dataupload_delete_action.go +++ b/pkg/datamover/dataupload_delete_action.go @@ -5,7 +5,7 @@ import ( "encoding/json" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/datamover/restore_micro_service.go b/pkg/datamover/restore_micro_service.go index 0cb5f18ecc..c67da0c13b 100644 --- a/pkg/datamover/restore_micro_service.go +++ b/pkg/datamover/restore_micro_service.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/datamover/restore_micro_service_test.go b/pkg/datamover/restore_micro_service_test.go index a53e4d6d52..a1d7b01326 100644 --- a/pkg/datamover/restore_micro_service_test.go +++ b/pkg/datamover/restore_micro_service_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/datapath/file_system.go b/pkg/datapath/file_system.go index f0f84acdb0..1816018c31 100644 --- a/pkg/datapath/file_system.go +++ b/pkg/datapath/file_system.go @@ -20,7 +20,7 @@ import ( "context" "sync" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/datapath/file_system_test.go b/pkg/datapath/file_system_test.go index 3887a82e31..28fe91b1d1 100644 --- a/pkg/datapath/file_system_test.go +++ b/pkg/datapath/file_system_test.go @@ -20,7 +20,7 @@ import ( "context" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" diff --git a/pkg/datapath/manager.go b/pkg/datapath/manager.go index 0b790a5cc9..959d75b887 100644 --- a/pkg/datapath/manager.go +++ b/pkg/datapath/manager.go @@ -20,7 +20,7 @@ import ( "context" "sync" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/datapath/micro_service_watcher.go b/pkg/datapath/micro_service_watcher.go index 665b84b813..3e8ace6514 100644 --- a/pkg/datapath/micro_service_watcher.go +++ b/pkg/datapath/micro_service_watcher.go @@ -24,7 +24,7 @@ import ( "sync" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/discovery/helper.go b/pkg/discovery/helper.go index 11c2d623be..884455dc53 100644 --- a/pkg/discovery/helper.go +++ b/pkg/discovery/helper.go @@ -21,7 +21,7 @@ import ( "strings" "sync" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/exposer/csi_snapshot.go b/pkg/exposer/csi_snapshot.go index 079a4b527f..fa888984ab 100644 --- a/pkg/exposer/csi_snapshot.go +++ b/pkg/exposer/csi_snapshot.go @@ -21,9 +21,9 @@ import ( "fmt" "time" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -41,6 +41,11 @@ import ( "github.com/vmware-tanzu/velero/pkg/util/kube" ) +// BackupPVCSecretLabel is the label applied to secrets and configmaps copied to the +// Velero namespace for backup PVC provisioning. The value is the owning DataUpload/DataDownload +// UID, which is a stable, valid label value (the owner name may exceed the label-value limit). +const BackupPVCSecretLabel = "velero.io/backup-pvc-secret" //nolint:gosec // not a credential + // CSISnapshotExposeParam define the input param for Expose of CSI snapshots type CSISnapshotExposeParam struct { // SnapshotName is the original volume snapshot name @@ -142,6 +147,28 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O curLog.Info("Volumesnapshot is ready") + // Copy secrets and configmaps from source namespace to Velero namespace if configured. + // Done before creating any intermediate objects so failure doesn't require cleanup. + // These are needed by CSI drivers that require namespace-scoped resources for volume + // provisioning (e.g., encrypted volumes with KMS tokens and tenant Vault configs). + if value, exists := csiExposeParam.BackupPVCConfig[csiExposeParam.StorageClass]; exists { + copyLabels := map[string]string{BackupPVCSecretLabel: string(ownerObject.UID)} + for _, secretName := range value.SecretNames { + if copyErr := kube.CopySecret(ctx, e.kubeClient.CoreV1(), secretName, + csiExposeParam.SourceNamespace, ownerObject.Namespace, copyLabels, curLog); copyErr != nil { + return errors.Wrapf(copyErr, "error copying secret %s from %s to %s", + secretName, csiExposeParam.SourceNamespace, ownerObject.Namespace) + } + } + for _, cmName := range value.ConfigMapNames { + if copyErr := kube.CopyConfigMap(ctx, e.kubeClient.CoreV1(), cmName, + csiExposeParam.SourceNamespace, ownerObject.Namespace, copyLabels, curLog); copyErr != nil { + return errors.Wrapf(copyErr, "error copying configmap %s from %s to %s", + cmName, csiExposeParam.SourceNamespace, ownerObject.Namespace) + } + } + } + vsc, err := csi.GetVolumeSnapshotContentForVolumeSnapshot(volumeSnapshot, e.csiSnapshotClient) if err != nil { return errors.Wrap(err, "error to get volume snapshot content") @@ -446,6 +473,11 @@ func (e *csiSnapshotExposer) CleanUp(ctx context.Context, ownerObject corev1api. kube.DeletePodIfAny(ctx, e.kubeClient.CoreV1(), backupPodName, ownerObject.Namespace, e.log) kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), backupPVCName, ownerObject.Namespace, cleanUpTimeout, e.log) + kube.DeleteSecretsWithLabel(ctx, e.kubeClient.CoreV1(), ownerObject.Namespace, + BackupPVCSecretLabel, string(ownerObject.UID), e.log) + kube.DeleteConfigMapsWithLabel(ctx, e.kubeClient.CoreV1(), ownerObject.Namespace, + BackupPVCSecretLabel, string(ownerObject.UID), e.log) + csi.DeleteVolumeSnapshotIfAny(ctx, e.csiSnapshotClient, backupVSName, ownerObject.Namespace, e.log) csi.DeleteVolumeSnapshotIfAny(ctx, e.csiSnapshotClient, vsName, sourceNamespace, e.log) } diff --git a/pkg/exposer/csi_snapshot_test.go b/pkg/exposer/csi_snapshot_test.go index e1a9860eb1..54c793f418 100644 --- a/pkg/exposer/csi_snapshot_test.go +++ b/pkg/exposer/csi_snapshot_test.go @@ -21,9 +21,9 @@ import ( "testing" "time" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" snapshotFake "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/fake" - "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" @@ -33,7 +33,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" clientTesting "k8s.io/client-go/testing" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -1408,7 +1408,7 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { Kind: backup.Kind, Name: backup.Name, UID: backup.UID, - Controller: pointer.BoolPtr(true), + Controller: ptr.To(true), }, }, }, @@ -1419,7 +1419,7 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { VolumeMode: &volumeMode, DataSource: dataSource, DataSourceRef: nil, - StorageClassName: pointer.String("fake-storage-class"), + StorageClassName: ptr.To("fake-storage-class"), Resources: corev1api.VolumeResourceRequirements{ Requests: corev1api.ResourceList{ corev1api.ResourceStorage: resource.MustParse("1Gi"), @@ -1439,7 +1439,7 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { Kind: backup.Kind, Name: backup.Name, UID: backup.UID, - Controller: pointer.BoolPtr(true), + Controller: ptr.To(true), }, }, }, @@ -1450,7 +1450,7 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { VolumeMode: &volumeMode, DataSource: dataSource, DataSourceRef: nil, - StorageClassName: pointer.String("fake-storage-class"), + StorageClassName: ptr.To("fake-storage-class"), Resources: corev1api.VolumeResourceRequirements{ Requests: corev1api.ResourceList{ corev1api.ResourceStorage: resource.MustParse("1Gi"), @@ -1989,3 +1989,186 @@ end diagnose CSI exposer`, }) } } + +func TestExpose_SecretCopy(t *testing.T) { + backup := &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: velerov1.SchemeGroupVersion.String(), + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "fake-backup", + UID: "fake-uid", + }, + } + + ownerObject := corev1api.ObjectReference{ + Kind: backup.Kind, + Namespace: backup.Namespace, + Name: backup.Name, + UID: backup.UID, + APIVersion: backup.APIVersion, + } + + // The secret/configmap copy runs after GetVolumeTopology and WaitVolumeSnapshotReady, + // so a StorageClass and a ready VolumeSnapshot are needed to reach the copy block. + scObj := &storagev1api.StorageClass{ + ObjectMeta: metav1.ObjectMeta{Name: "encrypted-sc"}, + } + readyVS := func() *snapshotv1api.VolumeSnapshot { + vscName := "fake-vsc" + return &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{Name: "fake-vs", Namespace: "app-ns"}, + Spec: snapshotv1api.VolumeSnapshotSpec{ + Source: snapshotv1api.VolumeSnapshotSource{VolumeSnapshotContentName: &vscName}, + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &vscName, + ReadyToUse: boolptr.True(), + RestoreSize: resource.NewQuantity(1234, ""), + }, + } + } + + param := func() *CSISnapshotExposeParam { + return &CSISnapshotExposeParam{ + SourceNamespace: "app-ns", + SourcePVName: "fake-pv", + SnapshotName: "fake-vs", + StorageClass: "encrypted-sc", + OperationTimeout: time.Millisecond, + ExposeTimeout: time.Second, + } + } + + t.Run("copies secret from source namespace", func(t *testing.T) { + srcSecret := &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "kms-token", Namespace: "app-ns"}, + Data: map[string][]byte{"token": []byte("vault-token")}, + Type: corev1api.SecretTypeOpaque, + } + fakeKubeClient := fake.NewSimpleClientset(srcSecret, scObj) + fakeSnapshotClient := snapshotFake.NewSimpleClientset(readyVS()) + + exposer := csiSnapshotExposer{ + kubeClient: fakeKubeClient, + csiSnapshotClient: fakeSnapshotClient.SnapshotV1(), + log: velerotest.NewLogger(), + } + + p := param() + p.BackupPVCConfig = map[string]velerotypes.BackupPVC{ + "encrypted-sc": {SecretNames: []string{"kms-token"}}, + } + + // Expose will fail later (no VSC exists), but the secret copy should succeed + _ = exposer.Expose(t.Context(), ownerObject, p) + + copied, err := fakeKubeClient.CoreV1().Secrets(ownerObject.Namespace).Get( + t.Context(), "kms-token", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, []byte("vault-token"), copied.Data["token"]) + assert.Equal(t, string(ownerObject.UID), copied.Labels[BackupPVCSecretLabel]) + }) + + t.Run("copies configmap from source namespace", func(t *testing.T) { + srcCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "kms-config", Namespace: "app-ns"}, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + } + fakeKubeClient := fake.NewSimpleClientset(srcCM, scObj) + fakeSnapshotClient := snapshotFake.NewSimpleClientset(readyVS()) + + exposer := csiSnapshotExposer{ + kubeClient: fakeKubeClient, + csiSnapshotClient: fakeSnapshotClient.SnapshotV1(), + log: velerotest.NewLogger(), + } + + p := param() + p.BackupPVCConfig = map[string]velerotypes.BackupPVC{ + "encrypted-sc": {ConfigMapNames: []string{"kms-config"}}, + } + + _ = exposer.Expose(t.Context(), ownerObject, p) + + copied, err := fakeKubeClient.CoreV1().ConfigMaps(ownerObject.Namespace).Get( + t.Context(), "kms-config", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, "https://vault.example.com", copied.Data["vaultAddress"]) + assert.Equal(t, string(ownerObject.UID), copied.Labels[BackupPVCSecretLabel]) + }) + + t.Run("returns error when source secret missing", func(t *testing.T) { + fakeKubeClient := fake.NewSimpleClientset(scObj) + fakeSnapshotClient := snapshotFake.NewSimpleClientset(readyVS()) + + exposer := csiSnapshotExposer{ + kubeClient: fakeKubeClient, + csiSnapshotClient: fakeSnapshotClient.SnapshotV1(), + log: velerotest.NewLogger(), + } + + p := param() + p.BackupPVCConfig = map[string]velerotypes.BackupPVC{ + "encrypted-sc": {SecretNames: []string{"missing-secret"}}, + } + + err := exposer.Expose(t.Context(), ownerObject, p) + require.Error(t, err) + assert.Contains(t, err.Error(), "error copying secret") + }) +} + +func TestCleanUp_SecretsAndConfigMaps(t *testing.T) { + ownerObject := corev1api.ObjectReference{ + Kind: "Backup", + Namespace: "velero", + Name: "du-123", + UID: "fake-uid", + APIVersion: "v1", + } + + secret := &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kms-token", Namespace: "velero", + Labels: map[string]string{BackupPVCSecretLabel: string(ownerObject.UID)}, + UID: "secret-uid", + }, + } + cm := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kms-config", Namespace: "velero", + Labels: map[string]string{BackupPVCSecretLabel: string(ownerObject.UID)}, + UID: "cm-uid", + }, + } + unrelatedSecret := &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "other-secret", Namespace: "velero", + Labels: map[string]string{BackupPVCSecretLabel: "other-owner-uid"}, + UID: "other-uid", + }, + } + + fakeKubeClient := fake.NewSimpleClientset(secret, cm, unrelatedSecret) + fakeSnapshotClient := snapshotFake.NewSimpleClientset() + + exposer := csiSnapshotExposer{ + kubeClient: fakeKubeClient, + csiSnapshotClient: fakeSnapshotClient.SnapshotV1(), + log: velerotest.NewLogger(), + } + + exposer.CleanUp(t.Context(), ownerObject, "", "app-ns") + + _, err := fakeKubeClient.CoreV1().Secrets("velero").Get(t.Context(), "kms-token", metav1.GetOptions{}) + require.Error(t, err, "owned secret should be deleted") + + _, err = fakeKubeClient.CoreV1().ConfigMaps("velero").Get(t.Context(), "kms-config", metav1.GetOptions{}) + require.Error(t, err, "owned configmap should be deleted") + + _, err = fakeKubeClient.CoreV1().Secrets("velero").Get(t.Context(), "other-secret", metav1.GetOptions{}) + assert.NoError(t, err, "unrelated secret should not be deleted") +} diff --git a/pkg/exposer/generic_restore.go b/pkg/exposer/generic_restore.go index b711e7364f..05308baa5a 100644 --- a/pkg/exposer/generic_restore.go +++ b/pkg/exposer/generic_restore.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -176,6 +176,36 @@ func (e *genericRestoreExposer) Expose(ctx context.Context, ownerObject corev1ap } } + // Copy secrets and configmaps from the target namespace to the Velero namespace if configured. + // These are needed by CSI drivers that require namespace-scoped resources for volume + // provisioning of the restorePVC (e.g., encrypted volumes with KMS tokens and tenant Vault configs). + copyLabels := map[string]string{BackupPVCSecretLabel: string(ownerObject.UID)} + for _, secretName := range param.RestorePVCConfig.SecretNames { + if copyErr := kube.CopySecret(ctx, e.kubeClient.CoreV1(), secretName, + param.TargetNamespace, ownerObject.Namespace, copyLabels, curLog); copyErr != nil { + err = errors.Wrapf(copyErr, "error copying secret %s from %s to %s", + secretName, param.TargetNamespace, ownerObject.Namespace) + return err + } + } + for _, cmName := range param.RestorePVCConfig.ConfigMapNames { + if copyErr := kube.CopyConfigMap(ctx, e.kubeClient.CoreV1(), cmName, + param.TargetNamespace, ownerObject.Namespace, copyLabels, curLog); copyErr != nil { + err = errors.Wrapf(copyErr, "error copying configmap %s from %s to %s", + cmName, param.TargetNamespace, ownerObject.Namespace) + return err + } + } + + defer func() { + if err != nil { + kube.DeleteSecretsWithLabel(ctx, e.kubeClient.CoreV1(), ownerObject.Namespace, + BackupPVCSecretLabel, string(ownerObject.UID), curLog) + kube.DeleteConfigMapsWithLabel(ctx, e.kubeClient.CoreV1(), ownerObject.Namespace, + BackupPVCSecretLabel, string(ownerObject.UID), curLog) + } + }() + restorePod, err := e.createRestorePod( ctx, ownerObject, @@ -377,6 +407,11 @@ func (e *genericRestoreExposer) CleanUp(ctx context.Context, ownerObject corev1a kube.DeletePodIfAny(ctx, e.kubeClient.CoreV1(), restorePodName, ownerObject.Namespace, e.log) kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), restorePVCName, ownerObject.Namespace, 0, e.log) kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), cachePVCName, ownerObject.Namespace, 0, e.log) + + kube.DeleteSecretsWithLabel(ctx, e.kubeClient.CoreV1(), ownerObject.Namespace, + BackupPVCSecretLabel, string(ownerObject.UID), e.log) + kube.DeleteConfigMapsWithLabel(ctx, e.kubeClient.CoreV1(), ownerObject.Namespace, + BackupPVCSecretLabel, string(ownerObject.UID), e.log) } func (e *genericRestoreExposer) RebindVolume(ctx context.Context, ownerObject corev1api.ObjectReference, targetPVCName string, targetNamespace string, timeout time.Duration) error { diff --git a/pkg/exposer/generic_restore_test.go b/pkg/exposer/generic_restore_test.go index 799719a50b..d033de5605 100644 --- a/pkg/exposer/generic_restore_test.go +++ b/pkg/exposer/generic_restore_test.go @@ -20,7 +20,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" @@ -34,6 +34,7 @@ import ( velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" + velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util/kube" ) @@ -306,6 +307,88 @@ func TestRestoreExpose(t *testing.T) { } } +func TestRestoreExpose_SecretCopy(t *testing.T) { + scName := "fake-sc" + restore := &velerov1.Restore{ + TypeMeta: metav1.TypeMeta{APIVersion: velerov1.SchemeGroupVersion.String(), Kind: "Restore"}, + ObjectMeta: metav1.ObjectMeta{Namespace: velerov1.DefaultNamespace, Name: "fake-restore", UID: "fake-uid"}, + } + ownerObject := corev1api.ObjectReference{ + Kind: restore.Kind, + Namespace: restore.Namespace, + Name: restore.Name, + UID: restore.UID, + APIVersion: restore.APIVersion, + } + targetPVCObj := &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Namespace: "fake-ns", Name: "fake-target-pvc"}, + Spec: corev1api.PersistentVolumeClaimSpec{StorageClassName: &scName}, + } + storageClass := &storagev1api.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "fake-sc"}} + daemonSet := &appsv1api.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Namespace: "velero", Name: "node-agent"}, + TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: appsv1api.SchemeGroupVersion.String()}, + Spec: appsv1api.DaemonSetSpec{ + Template: corev1api.PodTemplateSpec{ + Spec: corev1api.PodSpec{Containers: []corev1api.Container{{Image: "fake-image"}}}, + }, + }, + } + + t.Run("copies secret and configmap from target namespace", func(t *testing.T) { + srcSecret := &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "kms-token", Namespace: "fake-ns"}, + Data: map[string][]byte{"token": []byte("vault-token")}, + Type: corev1api.SecretTypeOpaque, + } + srcCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "kms-config", Namespace: "fake-ns"}, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + } + fakeKubeClient := fake.NewSimpleClientset(targetPVCObj, storageClass, daemonSet, srcSecret, srcCM) + exposer := genericRestoreExposer{kubeClient: fakeKubeClient, log: velerotest.NewLogger()} + + err := exposer.Expose(t.Context(), ownerObject, GenericRestoreExposeParam{ + TargetPVCName: "fake-target-pvc", + TargetNamespace: "fake-ns", + HostingPodLabels: map[string]string{}, + Resources: corev1api.ResourceRequirements{}, + ExposeTimeout: time.Millisecond, + RestorePVCConfig: velerotypes.RestorePVC{ + SecretNames: []string{"kms-token"}, + ConfigMapNames: []string{"kms-config"}, + }, + }) + require.NoError(t, err) + + copiedSecret, err := fakeKubeClient.CoreV1().Secrets(ownerObject.Namespace).Get(t.Context(), "kms-token", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, []byte("vault-token"), copiedSecret.Data["token"]) + assert.Equal(t, string(ownerObject.UID), copiedSecret.Labels[BackupPVCSecretLabel]) + + copiedCM, err := fakeKubeClient.CoreV1().ConfigMaps(ownerObject.Namespace).Get(t.Context(), "kms-config", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, "https://vault.example.com", copiedCM.Data["vaultAddress"]) + assert.Equal(t, string(ownerObject.UID), copiedCM.Labels[BackupPVCSecretLabel]) + }) + + t.Run("returns error when source secret missing", func(t *testing.T) { + fakeKubeClient := fake.NewSimpleClientset(targetPVCObj, storageClass, daemonSet) + exposer := genericRestoreExposer{kubeClient: fakeKubeClient, log: velerotest.NewLogger()} + + err := exposer.Expose(t.Context(), ownerObject, GenericRestoreExposeParam{ + TargetPVCName: "fake-target-pvc", + TargetNamespace: "fake-ns", + HostingPodLabels: map[string]string{}, + Resources: corev1api.ResourceRequirements{}, + ExposeTimeout: time.Millisecond, + RestorePVCConfig: velerotypes.RestorePVC{SecretNames: []string{"missing-secret"}}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "error copying secret") + }) +} + func TestRebindVolume(t *testing.T) { restore := &velerov1.Restore{ TypeMeta: metav1.TypeMeta{ diff --git a/pkg/exposer/host_path.go b/pkg/exposer/host_path.go index e511787117..db1dff9082 100644 --- a/pkg/exposer/host_path.go +++ b/pkg/exposer/host_path.go @@ -21,7 +21,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" diff --git a/pkg/exposer/host_path_test.go b/pkg/exposer/host_path_test.go index e751afe0dc..4c34aed020 100644 --- a/pkg/exposer/host_path_test.go +++ b/pkg/exposer/host_path_test.go @@ -21,7 +21,7 @@ import ( "fmt" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/pkg/exposer/image.go b/pkg/exposer/image.go index 58658e1b78..2157d81754 100644 --- a/pkg/exposer/image.go +++ b/pkg/exposer/image.go @@ -20,7 +20,7 @@ import ( "context" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" diff --git a/pkg/exposer/pod_volume.go b/pkg/exposer/pod_volume.go index aeb6f1903d..0526b2c5e8 100644 --- a/pkg/exposer/pod_volume.go +++ b/pkg/exposer/pod_volume.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/exposer/vgdp_counter.go b/pkg/exposer/vgdp_counter.go index cf6737c147..1f9850085d 100644 --- a/pkg/exposer/vgdp_counter.go +++ b/pkg/exposer/vgdp_counter.go @@ -4,7 +4,7 @@ import ( "context" "sync/atomic" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index 4ce4b5a4ff..12e19377e2 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -509,7 +509,7 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme if len(c.plugins) > 0 { for _, image := range c.plugins { - container := *builder.ForPluginContainer(image, pullPolicy).Result() + container := *builder.ForPluginContainer(image, pullPolicy, deployment.Spec.Template.Spec.InitContainers).Result() deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, container) } } diff --git a/pkg/install/install.go b/pkg/install/install.go index b60c4f9aa4..4584c39024 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -23,7 +23,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" diff --git a/pkg/itemblock/actions/pod_action.go b/pkg/itemblock/actions/pod_action.go index 2596e78a2e..6e9955d4e1 100644 --- a/pkg/itemblock/actions/pod_action.go +++ b/pkg/itemblock/actions/pod_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/itemblock/actions/pvc_action.go b/pkg/itemblock/actions/pvc_action.go index 6777ef5668..996fcf3d73 100644 --- a/pkg/itemblock/actions/pvc_action.go +++ b/pkg/itemblock/actions/pvc_action.go @@ -19,7 +19,7 @@ package actions import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/itemblock/actions/service_account_action.go b/pkg/itemblock/actions/service_account_action.go index 91cdbbe590..d94b33f15e 100644 --- a/pkg/itemblock/actions/service_account_action.go +++ b/pkg/itemblock/actions/service_account_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/itemoperationmap/backup_operation_map.go b/pkg/itemoperationmap/backup_operation_map.go index 47cdcac814..49dfbecc84 100644 --- a/pkg/itemoperationmap/backup_operation_map.go +++ b/pkg/itemoperationmap/backup_operation_map.go @@ -20,7 +20,7 @@ import ( "bytes" "sync" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/persistence" diff --git a/pkg/itemoperationmap/restore_operation_map.go b/pkg/itemoperationmap/restore_operation_map.go index 4256591bc5..2586d7bb32 100644 --- a/pkg/itemoperationmap/restore_operation_map.go +++ b/pkg/itemoperationmap/restore_operation_map.go @@ -20,7 +20,7 @@ import ( "bytes" "sync" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/persistence" diff --git a/pkg/nodeagent/node_agent.go b/pkg/nodeagent/node_agent.go index 87efb896a6..b449a91f45 100644 --- a/pkg/nodeagent/node_agent.go +++ b/pkg/nodeagent/node_agent.go @@ -21,7 +21,8 @@ import ( "encoding/json" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" + appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -80,6 +81,36 @@ func KbClientIsRunningInNode(ctx context.Context, namespace string, nodeName str return isRunningInNode(ctx, namespace, nodeName, nil, kubeClient) } +// IsReady checks whether the node-agent daemonset has at least one ready pod +// by inspecting the DaemonSet status. +func IsReady(ctx context.Context, namespace string, crClient ctrlclient.Client) error { + dsLinux := new(appsv1api.DaemonSet) + if err := crClient.Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: daemonSet}, dsLinux); err != nil { + dsLinux = nil + if !apierrors.IsNotFound(err) { + return errors.Wrap(err, "failed to get linux node-agent daemonset") + } + } + + dsWindows := new(appsv1api.DaemonSet) + if err := crClient.Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: daemonsetWindows}, dsWindows); err != nil { + dsWindows = nil + if !apierrors.IsNotFound(err) { + return errors.Wrap(err, "failed to get windows node-agent daemonset") + } + } + + if dsLinux != nil && dsLinux.Status.NumberReady > 0 { + return nil + } + + if dsWindows != nil && dsWindows.Status.NumberReady > 0 { + return nil + } + + return errors.New("node-agent is not ready: no ready pods found") +} + // IsRunningInNode checks if the node agent pod is running properly in a specified node through controller client. If not, return the error found func IsRunningInNode(ctx context.Context, namespace string, nodeName string, crClient ctrlclient.Client) error { return isRunningInNode(ctx, namespace, nodeName, crClient, nil) diff --git a/pkg/nodeagent/node_agent_test.go b/pkg/nodeagent/node_agent_test.go index 168e91de10..9bba67ec47 100644 --- a/pkg/nodeagent/node_agent_test.go +++ b/pkg/nodeagent/node_agent_test.go @@ -17,9 +17,10 @@ limitations under the License. package nodeagent import ( + "context" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" @@ -28,7 +29,9 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" clientTesting "k8s.io/client-go/testing" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "github.com/vmware-tanzu/velero/pkg/builder" velerotypes "github.com/vmware-tanzu/velero/pkg/types" @@ -213,6 +216,152 @@ func TestIsRunningInNode(t *testing.T) { } } +func TestIsReady(t *testing.T) { + scheme := runtime.NewScheme() + appsv1api.AddToScheme(scheme) + + dsLinuxNotReady := &appsv1api.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Namespace: "fake-ns", Name: "node-agent"}, + Status: appsv1api.DaemonSetStatus{NumberReady: 0}, + } + dsLinuxReady := &appsv1api.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Namespace: "fake-ns", Name: "node-agent"}, + Status: appsv1api.DaemonSetStatus{NumberReady: 3}, + } + dsWindowsNotReady := &appsv1api.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Namespace: "fake-ns", Name: "node-agent-windows"}, + Status: appsv1api.DaemonSetStatus{NumberReady: 0}, + } + dsWindowsReady := &appsv1api.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Namespace: "fake-ns", Name: "node-agent-windows"}, + Status: appsv1api.DaemonSetStatus{NumberReady: 2}, + } + + tests := []struct { + name string + kubeClientObj []runtime.Object + namespace string + interceptor *interceptor.Funcs + expectErr string + }{ + { + name: "both daemonsets not found", + namespace: "fake-ns", + expectErr: "node-agent is not ready: no ready pods found", + }, + { + name: "linux daemonset get error", + namespace: "fake-ns", + interceptor: &interceptor.Funcs{ + Get: func(ctx context.Context, c ctrlclient.WithWatch, key ctrlclient.ObjectKey, obj ctrlclient.Object, opts ...ctrlclient.GetOption) error { + if key.Name == "node-agent" { + return errors.New("fake-get-error") + } + return c.Get(ctx, key, obj, opts...) + }, + }, + expectErr: "failed to get linux node-agent daemonset: fake-get-error", + }, + { + name: "windows daemonset get error", + namespace: "fake-ns", + interceptor: &interceptor.Funcs{ + Get: func(ctx context.Context, c ctrlclient.WithWatch, key ctrlclient.ObjectKey, obj ctrlclient.Object, opts ...ctrlclient.GetOption) error { + if key.Name == "node-agent-windows" { + return errors.New("fake-get-error") + } + return c.Get(ctx, key, obj, opts...) + }, + }, + expectErr: "failed to get windows node-agent daemonset: fake-get-error", + }, + { + name: "linux ds exist but no ready pods", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + dsLinuxNotReady, + }, + expectErr: "node-agent is not ready: no ready pods found", + }, + { + name: "linux ds with ready pods", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + dsLinuxReady, + }, + }, + { + name: "windows ds exist but no ready pods", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + dsWindowsNotReady, + }, + expectErr: "node-agent is not ready: no ready pods found", + }, + { + name: "windows ds with ready pods", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + dsWindowsReady, + }, + }, + { + name: "both daemonsets exist but no ready pods", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + dsLinuxNotReady, + dsWindowsNotReady, + }, + expectErr: "node-agent is not ready: no ready pods found", + }, + { + name: "both daemonsets exist, linux ready", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + dsLinuxReady, + dsWindowsNotReady, + }, + }, + { + name: "both daemonsets exist, windows ready", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + dsLinuxNotReady, + dsWindowsReady, + }, + }, + { + name: "both daemonsets exist, both ready", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + dsLinuxReady, + dsWindowsReady, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + builder := clientFake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(test.kubeClientObj...) + + if test.interceptor != nil { + builder = builder.WithInterceptorFuncs(*test.interceptor) + } + + fakeClient := builder.Build() + + err := IsReady(t.Context(), test.namespace, fakeClient) + if test.expectErr == "" { + assert.NoError(t, err) + } else { + assert.ErrorContains(t, err, test.expectErr) + } + }) + } +} + func TestGetPodSpec(t *testing.T) { podSpec := corev1api.PodSpec{ NodeName: "fake-node", diff --git a/pkg/persistence/object_store.go b/pkg/persistence/object_store.go index eaf983819f..c441d4a039 100644 --- a/pkg/persistence/object_store.go +++ b/pkg/persistence/object_store.go @@ -25,7 +25,7 @@ import ( snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/runtime/serializer" kerrors "k8s.io/apimachinery/pkg/util/errors" diff --git a/pkg/plugin/clientmgmt/backupitemaction/v1/restartable_backup_item_action.go b/pkg/plugin/clientmgmt/backupitemaction/v1/restartable_backup_item_action.go index 4bc28e487d..0fe8fb5831 100644 --- a/pkg/plugin/clientmgmt/backupitemaction/v1/restartable_backup_item_action.go +++ b/pkg/plugin/clientmgmt/backupitemaction/v1/restartable_backup_item_action.go @@ -17,7 +17,7 @@ limitations under the License. package v1 import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/runtime" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/pkg/plugin/clientmgmt/backupitemaction/v1/restartable_backup_item_action_test.go b/pkg/plugin/clientmgmt/backupitemaction/v1/restartable_backup_item_action_test.go index 8dc113df55..c1abcc4710 100644 --- a/pkg/plugin/clientmgmt/backupitemaction/v1/restartable_backup_item_action_test.go +++ b/pkg/plugin/clientmgmt/backupitemaction/v1/restartable_backup_item_action_test.go @@ -19,7 +19,7 @@ package v1 import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action.go b/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action.go index 84ddb54b2b..890aef75ad 100644 --- a/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action.go +++ b/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action.go @@ -17,7 +17,7 @@ limitations under the License. package v2 import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/runtime" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action_test.go b/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action_test.go index bd1ee0ec26..d0400f760f 100644 --- a/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action_test.go +++ b/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action_test.go @@ -19,7 +19,7 @@ package v2 import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/clientmgmt/itemblockaction/v1/restartable_item_block_action.go b/pkg/plugin/clientmgmt/itemblockaction/v1/restartable_item_block_action.go index 83742978f1..49d72251fc 100644 --- a/pkg/plugin/clientmgmt/itemblockaction/v1/restartable_item_block_action.go +++ b/pkg/plugin/clientmgmt/itemblockaction/v1/restartable_item_block_action.go @@ -17,7 +17,7 @@ limitations under the License. package v1 import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/runtime" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/pkg/plugin/clientmgmt/itemblockaction/v1/restartable_item_block_action_test.go b/pkg/plugin/clientmgmt/itemblockaction/v1/restartable_item_block_action_test.go index 04dd606526..99ac00016b 100644 --- a/pkg/plugin/clientmgmt/itemblockaction/v1/restartable_item_block_action_test.go +++ b/pkg/plugin/clientmgmt/itemblockaction/v1/restartable_item_block_action_test.go @@ -19,7 +19,7 @@ package v1 import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/clientmgmt/manager_test.go b/pkg/plugin/clientmgmt/manager_test.go index 7576c42c5b..95f911ffc8 100644 --- a/pkg/plugin/clientmgmt/manager_test.go +++ b/pkg/plugin/clientmgmt/manager_test.go @@ -20,7 +20,7 @@ import ( "fmt" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/plugin/clientmgmt/process/process.go b/pkg/plugin/clientmgmt/process/process.go index 4d20a17847..8d496cd7db 100644 --- a/pkg/plugin/clientmgmt/process/process.go +++ b/pkg/plugin/clientmgmt/process/process.go @@ -17,8 +17,8 @@ limitations under the License. package process import ( + "github.com/cockroachdb/errors" plugin "github.com/hashicorp/go-plugin" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" diff --git a/pkg/plugin/clientmgmt/process/process_test.go b/pkg/plugin/clientmgmt/process/process_test.go index e67de8db9c..455ad954e9 100644 --- a/pkg/plugin/clientmgmt/process/process_test.go +++ b/pkg/plugin/clientmgmt/process/process_test.go @@ -18,7 +18,7 @@ package process import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" diff --git a/pkg/plugin/clientmgmt/process/registry.go b/pkg/plugin/clientmgmt/process/registry.go index 744048690a..f667fa5d3d 100644 --- a/pkg/plugin/clientmgmt/process/registry.go +++ b/pkg/plugin/clientmgmt/process/registry.go @@ -22,7 +22,7 @@ import ( "path/filepath" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/plugin/framework" diff --git a/pkg/plugin/clientmgmt/process/restartable_process.go b/pkg/plugin/clientmgmt/process/restartable_process.go index e285f82daa..7f3053fa0e 100644 --- a/pkg/plugin/clientmgmt/process/restartable_process.go +++ b/pkg/plugin/clientmgmt/process/restartable_process.go @@ -19,7 +19,7 @@ package process import ( "sync" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" ) diff --git a/pkg/plugin/clientmgmt/restartable_delete_item_action.go b/pkg/plugin/clientmgmt/restartable_delete_item_action.go index b566ede6f3..7d98972402 100644 --- a/pkg/plugin/clientmgmt/restartable_delete_item_action.go +++ b/pkg/plugin/clientmgmt/restartable_delete_item_action.go @@ -17,7 +17,7 @@ limitations under the License. package clientmgmt import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" diff --git a/pkg/plugin/clientmgmt/restartable_delete_item_action_test.go b/pkg/plugin/clientmgmt/restartable_delete_item_action_test.go index 52edf7ffb6..e41dc43feb 100644 --- a/pkg/plugin/clientmgmt/restartable_delete_item_action_test.go +++ b/pkg/plugin/clientmgmt/restartable_delete_item_action_test.go @@ -19,7 +19,7 @@ package clientmgmt import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/clientmgmt/restartable_object_store.go b/pkg/plugin/clientmgmt/restartable_object_store.go index 6e66d4b3e8..c5eeeb60dc 100644 --- a/pkg/plugin/clientmgmt/restartable_object_store.go +++ b/pkg/plugin/clientmgmt/restartable_object_store.go @@ -20,7 +20,7 @@ import ( "io" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" diff --git a/pkg/plugin/clientmgmt/restartable_object_store_test.go b/pkg/plugin/clientmgmt/restartable_object_store_test.go index 0b25e02d80..e1f07fe9ba 100644 --- a/pkg/plugin/clientmgmt/restartable_object_store_test.go +++ b/pkg/plugin/clientmgmt/restartable_object_store_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/pkg/plugin/clientmgmt/restoreitemaction/v1/restartable_restore_item_action.go b/pkg/plugin/clientmgmt/restoreitemaction/v1/restartable_restore_item_action.go index a03ccecd2a..e5cefceb4e 100644 --- a/pkg/plugin/clientmgmt/restoreitemaction/v1/restartable_restore_item_action.go +++ b/pkg/plugin/clientmgmt/restoreitemaction/v1/restartable_restore_item_action.go @@ -17,7 +17,7 @@ limitations under the License. package v1 import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" diff --git a/pkg/plugin/clientmgmt/restoreitemaction/v1/restartable_restore_item_action_test.go b/pkg/plugin/clientmgmt/restoreitemaction/v1/restartable_restore_item_action_test.go index 08239fc931..1fefdbddd4 100644 --- a/pkg/plugin/clientmgmt/restoreitemaction/v1/restartable_restore_item_action_test.go +++ b/pkg/plugin/clientmgmt/restoreitemaction/v1/restartable_restore_item_action_test.go @@ -19,7 +19,7 @@ package v1 import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/clientmgmt/restoreitemaction/v2/restartable_restore_item_action.go b/pkg/plugin/clientmgmt/restoreitemaction/v2/restartable_restore_item_action.go index c23787cfb7..75b3e19379 100644 --- a/pkg/plugin/clientmgmt/restoreitemaction/v2/restartable_restore_item_action.go +++ b/pkg/plugin/clientmgmt/restoreitemaction/v2/restartable_restore_item_action.go @@ -17,7 +17,7 @@ limitations under the License. package v2 import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" diff --git a/pkg/plugin/clientmgmt/restoreitemaction/v2/restartable_restore_item_action_test.go b/pkg/plugin/clientmgmt/restoreitemaction/v2/restartable_restore_item_action_test.go index af6521e438..685835dfc4 100644 --- a/pkg/plugin/clientmgmt/restoreitemaction/v2/restartable_restore_item_action_test.go +++ b/pkg/plugin/clientmgmt/restoreitemaction/v2/restartable_restore_item_action_test.go @@ -19,7 +19,7 @@ package v2 import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/clientmgmt/volumesnapshotter/v1/restartable_volume_snapshotter.go b/pkg/plugin/clientmgmt/volumesnapshotter/v1/restartable_volume_snapshotter.go index 7aec39872c..c2afdd747c 100644 --- a/pkg/plugin/clientmgmt/volumesnapshotter/v1/restartable_volume_snapshotter.go +++ b/pkg/plugin/clientmgmt/volumesnapshotter/v1/restartable_volume_snapshotter.go @@ -17,7 +17,7 @@ limitations under the License. package v1 import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" diff --git a/pkg/plugin/clientmgmt/volumesnapshotter/v1/restartable_volume_snapshotter_test.go b/pkg/plugin/clientmgmt/volumesnapshotter/v1/restartable_volume_snapshotter_test.go index 8a2efe04cf..6675651e78 100644 --- a/pkg/plugin/clientmgmt/volumesnapshotter/v1/restartable_volume_snapshotter_test.go +++ b/pkg/plugin/clientmgmt/volumesnapshotter/v1/restartable_volume_snapshotter_test.go @@ -20,7 +20,7 @@ import ( "testing" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/framework/action_resolver.go b/pkg/plugin/framework/action_resolver.go index ac8a0b1d09..f2b883afed 100644 --- a/pkg/plugin/framework/action_resolver.go +++ b/pkg/plugin/framework/action_resolver.go @@ -17,7 +17,7 @@ limitations under the License. package framework import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/plugin/framework/backup_item_action_client.go b/pkg/plugin/framework/backup_item_action_client.go index 724737d01f..a0ac831ab6 100644 --- a/pkg/plugin/framework/backup_item_action_client.go +++ b/pkg/plugin/framework/backup_item_action_client.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/plugin/framework/backup_item_action_server.go b/pkg/plugin/framework/backup_item_action_server.go index 7c18b4ef6b..c3d8c59803 100644 --- a/pkg/plugin/framework/backup_item_action_server.go +++ b/pkg/plugin/framework/backup_item_action_server.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/pkg/plugin/framework/backup_item_action_test.go b/pkg/plugin/framework/backup_item_action_test.go index 1472eb1150..32fda1e787 100644 --- a/pkg/plugin/framework/backup_item_action_test.go +++ b/pkg/plugin/framework/backup_item_action_test.go @@ -20,7 +20,7 @@ import ( "encoding/json" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go index 64695dbe3e..cff46aee29 100644 --- a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go +++ b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go index f8c894ebad..1106bbebb5 100644 --- a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go +++ b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/protobuf/types/known/emptypb" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go index 502c375026..a60adb26f1 100644 --- a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go +++ b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go @@ -20,7 +20,7 @@ import ( "encoding/json" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/framework/common/handle_panic.go b/pkg/plugin/framework/common/handle_panic.go index 697ff588c0..2470875cea 100644 --- a/pkg/plugin/framework/common/handle_panic.go +++ b/pkg/plugin/framework/common/handle_panic.go @@ -19,7 +19,7 @@ package common import ( "runtime/debug" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/grpc/codes" ) @@ -37,7 +37,7 @@ func HandlePanic(p any) error { if panicErr, ok := p.(error); !ok { err = errors.Errorf("plugin panicked: %v", p) } else { - if _, ok := panicErr.(StackTracer); ok { + if errors.GetReportableStackTrace(panicErr) != nil { err = panicErr } else { errWithStacktrace := errors.Errorf("%v, stack trace: %s", panicErr, debug.Stack()) diff --git a/pkg/plugin/framework/common/plugin_config.go b/pkg/plugin/framework/common/plugin_config.go index 82b914352c..b248334e08 100644 --- a/pkg/plugin/framework/common/plugin_config.go +++ b/pkg/plugin/framework/common/plugin_config.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" diff --git a/pkg/plugin/framework/common/server_errors.go b/pkg/plugin/framework/common/server_errors.go index 60eff50f43..6d74d8add1 100644 --- a/pkg/plugin/framework/common/server_errors.go +++ b/pkg/plugin/framework/common/server_errors.go @@ -17,13 +17,12 @@ limitations under the License. package common import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/protoadapt" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" - "github.com/vmware-tanzu/velero/pkg/util/logging" ) // NewGRPCErrorWithCode wraps err in a gRPC status error with the error's stack trace @@ -61,25 +60,19 @@ func NewGRPCError(err error, details ...protoadapt.MessageV1) error { // ErrorStack gets a stack trace, if it exists, from the provided error, and // returns it as a *proto.Stack. func ErrorStack(err error) *proto.Stack { - stackTracer, ok := err.(StackTracer) - if !ok { + stack := errors.GetReportableStackTrace(err) + if stack == nil { return nil } stackTrace := new(proto.Stack) - for _, frame := range stackTracer.StackTrace() { - location := logging.GetFrameLocationInfo(frame) - + for _, frame := range stack.Frames { stackTrace.Frames = append(stackTrace.Frames, &proto.StackFrame{ - File: location.File, - Line: int32(location.Line), - Function: location.Function, + File: frame.Filename, + Line: int32(frame.Lineno), + Function: frame.Function, }) } return stackTrace } - -type StackTracer interface { - StackTrace() errors.StackTrace -} diff --git a/pkg/plugin/framework/common/server_mux.go b/pkg/plugin/framework/common/server_mux.go index 4eecdb8d21..ab13bf5aa4 100644 --- a/pkg/plugin/framework/common/server_mux.go +++ b/pkg/plugin/framework/common/server_mux.go @@ -19,7 +19,7 @@ package common import ( "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" diff --git a/pkg/plugin/framework/delete_item_action_client.go b/pkg/plugin/framework/delete_item_action_client.go index bec5088db6..90822fb3c7 100644 --- a/pkg/plugin/framework/delete_item_action_client.go +++ b/pkg/plugin/framework/delete_item_action_client.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/grpc" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" diff --git a/pkg/plugin/framework/delete_item_action_server.go b/pkg/plugin/framework/delete_item_action_server.go index 01abe8dc3e..fbdf624865 100644 --- a/pkg/plugin/framework/delete_item_action_server.go +++ b/pkg/plugin/framework/delete_item_action_server.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/pkg/plugin/framework/itemblockaction/v1/item_block_action_client.go b/pkg/plugin/framework/itemblockaction/v1/item_block_action_client.go index aa597c4afa..34d92e1e3c 100644 --- a/pkg/plugin/framework/itemblockaction/v1/item_block_action_client.go +++ b/pkg/plugin/framework/itemblockaction/v1/item_block_action_client.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/plugin/framework/itemblockaction/v1/item_block_action_server.go b/pkg/plugin/framework/itemblockaction/v1/item_block_action_server.go index 2d940550c3..fc6de76948 100644 --- a/pkg/plugin/framework/itemblockaction/v1/item_block_action_server.go +++ b/pkg/plugin/framework/itemblockaction/v1/item_block_action_server.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/pkg/plugin/framework/itemblockaction/v1/item_block_action_test.go b/pkg/plugin/framework/itemblockaction/v1/item_block_action_test.go index 6e2a0e4d5b..5a09f4e470 100644 --- a/pkg/plugin/framework/itemblockaction/v1/item_block_action_test.go +++ b/pkg/plugin/framework/itemblockaction/v1/item_block_action_test.go @@ -20,7 +20,7 @@ import ( "encoding/json" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/framework/object_store_client.go b/pkg/plugin/framework/object_store_client.go index b59f3d1b0c..474a0173c9 100644 --- a/pkg/plugin/framework/object_store_client.go +++ b/pkg/plugin/framework/object_store_client.go @@ -22,7 +22,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/grpc" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" diff --git a/pkg/plugin/framework/object_store_server.go b/pkg/plugin/framework/object_store_server.go index fbed21ecfb..ae79d7cf1f 100644 --- a/pkg/plugin/framework/object_store_server.go +++ b/pkg/plugin/framework/object_store_server.go @@ -22,7 +22,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" diff --git a/pkg/plugin/framework/plugin_lister.go b/pkg/plugin/framework/plugin_lister.go index 6db81c66d4..c3d6b89e95 100644 --- a/pkg/plugin/framework/plugin_lister.go +++ b/pkg/plugin/framework/plugin_lister.go @@ -19,8 +19,8 @@ package framework import ( "context" + "github.com/cockroachdb/errors" plugin "github.com/hashicorp/go-plugin" - "github.com/pkg/errors" "google.golang.org/grpc" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" diff --git a/pkg/plugin/framework/restore_item_action_client.go b/pkg/plugin/framework/restore_item_action_client.go index 3a5a633f3a..ea08fb0d70 100644 --- a/pkg/plugin/framework/restore_item_action_client.go +++ b/pkg/plugin/framework/restore_item_action_client.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/plugin/framework/restore_item_action_server.go b/pkg/plugin/framework/restore_item_action_server.go index 175a941bd1..94dd2a9115 100644 --- a/pkg/plugin/framework/restore_item_action_server.go +++ b/pkg/plugin/framework/restore_item_action_server.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/pkg/plugin/framework/restoreitemaction/v2/restore_item_action_client.go b/pkg/plugin/framework/restoreitemaction/v2/restore_item_action_client.go index 5e2f01c371..cc36b950ff 100644 --- a/pkg/plugin/framework/restoreitemaction/v2/restore_item_action_client.go +++ b/pkg/plugin/framework/restoreitemaction/v2/restore_item_action_client.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/plugin/framework/restoreitemaction/v2/restore_item_action_server.go b/pkg/plugin/framework/restoreitemaction/v2/restore_item_action_server.go index 1159616566..ace2a33e14 100644 --- a/pkg/plugin/framework/restoreitemaction/v2/restore_item_action_server.go +++ b/pkg/plugin/framework/restoreitemaction/v2/restore_item_action_server.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/emptypb" "google.golang.org/protobuf/types/known/timestamppb" diff --git a/pkg/plugin/framework/validation.go b/pkg/plugin/framework/validation.go index ba8f39be15..bbebf3ca33 100644 --- a/pkg/plugin/framework/validation.go +++ b/pkg/plugin/framework/validation.go @@ -17,7 +17,7 @@ limitations under the License. package framework import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/util/sets" ) diff --git a/pkg/plugin/framework/volume_snapshotter_client.go b/pkg/plugin/framework/volume_snapshotter_client.go index f7af07ce49..78df708632 100644 --- a/pkg/plugin/framework/volume_snapshotter_client.go +++ b/pkg/plugin/framework/volume_snapshotter_client.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/plugin/framework/volume_snapshotter_server.go b/pkg/plugin/framework/volume_snapshotter_server.go index de30c823ff..152f9451e5 100644 --- a/pkg/plugin/framework/volume_snapshotter_server.go +++ b/pkg/plugin/framework/volume_snapshotter_server.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" diff --git a/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go b/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go index 3c23802f22..323c8ef92b 100644 --- a/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go +++ b/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go @@ -21,7 +21,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" diff --git a/pkg/plugin/velero/restoreitemaction/v2/restore_item_action.go b/pkg/plugin/velero/restoreitemaction/v2/restore_item_action.go index dfc35428f8..1b2cff3186 100644 --- a/pkg/plugin/velero/restoreitemaction/v2/restore_item_action.go +++ b/pkg/plugin/velero/restoreitemaction/v2/restore_item_action.go @@ -19,7 +19,7 @@ package v2 import ( "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" diff --git a/pkg/podexec/pod_command_executor.go b/pkg/podexec/pod_command_executor.go index b1d7fbf59a..9ada2743c4 100644 --- a/pkg/podexec/pod_command_executor.go +++ b/pkg/podexec/pod_command_executor.go @@ -22,7 +22,7 @@ import ( "net/url" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/podexec/pod_command_executor_test.go b/pkg/podexec/pod_command_executor_test.go index e28eb04587..7dd2e9342d 100644 --- a/pkg/podexec/pod_command_executor_test.go +++ b/pkg/podexec/pod_command_executor_test.go @@ -25,7 +25,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" diff --git a/pkg/podvolume/backup_micro_service.go b/pkg/podvolume/backup_micro_service.go index 11ca66676b..3dbe1dc708 100644 --- a/pkg/podvolume/backup_micro_service.go +++ b/pkg/podvolume/backup_micro_service.go @@ -21,7 +21,7 @@ import ( "encoding/json" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" diff --git a/pkg/podvolume/backup_micro_service_test.go b/pkg/podvolume/backup_micro_service_test.go index a5b6cd1f42..9242c4165a 100644 --- a/pkg/podvolume/backup_micro_service_test.go +++ b/pkg/podvolume/backup_micro_service_test.go @@ -23,7 +23,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/podvolume/backupper.go b/pkg/podvolume/backupper.go index 1747f1b331..1db36291d3 100644 --- a/pkg/podvolume/backupper.go +++ b/pkg/podvolume/backupper.go @@ -20,12 +20,15 @@ import ( "context" "fmt" "sync" + "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/cache" ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -181,7 +184,7 @@ func newBackupper( // the PVB in the indexer is already in final status, no need to call WaitGroup.Done() if ok && (existPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseCompleted || existPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed || - pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled) { + existPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled) { statusChangedToFinal = false } } @@ -416,24 +419,71 @@ func (b *backupper) WaitAllPodVolumesProcessed(log logrus.FieldLogger) []*velero select { case <-b.ctx.Done(): log.Error("timed out waiting for all PodVolumeBackups to complete") - case <-done: + for _, obj := range b.pvbIndexer.List() { pvb, ok := obj.(*velerov1api.PodVolumeBackup) if !ok { - log.Errorf("expected PodVolumeBackup, but got %T", obj) + log.Errorf("expected PVB, but got %T", obj) continue } - podVolumeBackups = append(podVolumeBackups, pvb) - if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed { - log.Errorf("pod volume backup failed: %s", pvb.Status.Message) - } else if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled { - log.Errorf("pod volume backup canceled: %s", pvb.Status.Message) + + if pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseCompleted && + pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseFailed && + pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseCanceled { + log.Infof("Setting cancel flag for ongoing PVB %s/%s", pvb.Namespace, pvb.Name) + if err := updatePVBWithRetry(context.Background(), b.crClient, pvb.Namespace, pvb.Name); err != nil { + log.WithError(err).Errorf("Failed to set cancel flag for PVB %s/%s", pvb.Namespace, pvb.Name) + } } } + <-done + case <-done: + } + + // Collect tracked PVBs regardless of whether we timed out or completed normally. + // On timeout, already-completed PVBs must still be persisted so their data remains restorable. + for _, obj := range b.pvbIndexer.List() { + pvb, ok := obj.(*velerov1api.PodVolumeBackup) + if !ok { + log.Errorf("expected PodVolumeBackup, but got %T", obj) + continue + } + podVolumeBackups = append(podVolumeBackups, pvb) + if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed { + log.Errorf("pod volume backup failed: %s", pvb.Status.Message) + } else if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled { + log.Errorf("pod volume backup canceled: %s", pvb.Status.Message) + } } return podVolumeBackups } +func updatePVBWithRetry(ctx context.Context, client ctrlclient.Client, namespace, name string) error { + return wait.PollUntilContextCancel(ctx, 100*time.Millisecond, true, func(ctx context.Context) (bool, error) { + pvb := &velerov1api.PodVolumeBackup{} + if err := client.Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: name}, pvb); err != nil { + return false, errors.Wrap(err, "getting PVB") + } + + if pvb.Spec.Cancel { + return true, nil + } + + pvb.Spec.Cancel = true + pvb.Status.Message = "Cancel PVB on pod volume timeout" + + err := client.Update(ctx, pvb) + if err != nil { + if apierrors.IsConflict(err) { + return false, nil + } + return false, errors.Wrapf(err, "error updating PVB %s/%s", pvb.Namespace, pvb.Name) + } + + return true, nil + }) +} + func (b *backupper) GetPodVolumeBackupByPodAndVolume(podNamespace, podName, volume string) (*velerov1api.PodVolumeBackup, error) { obj, exist, err := b.pvbIndexer.GetByKey(fmt.Sprintf(pvbKeyPattern, podNamespace, podName, volume)) if err != nil { diff --git a/pkg/podvolume/backupper_factory.go b/pkg/podvolume/backupper_factory.go index f75f1d30b6..0f70fe8c02 100644 --- a/pkg/podvolume/backupper_factory.go +++ b/pkg/podvolume/backupper_factory.go @@ -19,7 +19,7 @@ package podvolume import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/client-go/tools/cache" ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache" diff --git a/pkg/podvolume/backupper_test.go b/pkg/podvolume/backupper_test.go index 846f65796e..853949c1a0 100644 --- a/pkg/podvolume/backupper_test.go +++ b/pkg/podvolume/backupper_test.go @@ -693,14 +693,14 @@ func TestListPodVolumeBackupsByPodp(t *testing.T) { } type logHook struct { - entry *logrus.Entry + entries []*logrus.Entry } func (l *logHook) Levels() []logrus.Level { return []logrus.Level{logrus.ErrorLevel} } func (l *logHook) Fire(entry *logrus.Entry) error { - l.entry = entry + l.entries = append(l.entries, entry) return nil } @@ -717,16 +717,18 @@ func TestWaitAllPodVolumesProcessed(t *testing.T) { statusToBeUpdated *velerov1api.PodVolumeBackupStatus expectedErr string expectedPVBPhase velerov1api.PodVolumeBackupPhase + expectedPVBCount int }{ { name: "contains no pvb should report no error", ctx: timeoutCtx, }, { - name: "context canceled", - ctx: timeoutCtx, - pvb: pvb, - expectedErr: "timed out waiting for all PodVolumeBackups to complete", + name: "context canceled should still return tracked pvbs", + ctx: timeoutCtx, + pvb: pvb, + expectedErr: "timed out waiting for all PodVolumeBackups to complete", + expectedPVBCount: 1, }, { name: "failed pvbs", @@ -776,12 +778,31 @@ func TestWaitAllPodVolumesProcessed(t *testing.T) { logHook := &logHook{} logger.Hooks.Add(logHook) - backuper := newBackupper(c.ctx, log, nil, nil, informer, nil, "", &velerov1api.Backup{}) + backuper := newBackupper(c.ctx, log, nil, nil, informer, client, "", &velerov1api.Backup{}) if c.pvb != nil { require.NoError(t, backuper.pvbIndexer.Add(c.pvb)) backuper.wg.Add(1) } + if c.ctx == timeoutCtx && c.pvb != nil { + // Start a goroutine to simulate the controller's cancellation behavior + go func() { + // Wait a short time for the cancel flag to be set + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for range ticker.C { + pvb := &velerov1api.PodVolumeBackup{} + err := client.Get(t.Context(), ctrlclient.ObjectKey{Namespace: c.pvb.Namespace, Name: c.pvb.Name}, pvb) + if err == nil && pvb.Spec.Cancel { + pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseCanceled + pvb.Status.Message = "canceled" + _ = client.Update(t.Context(), pvb) + return + } + } + }() + } + if c.statusToBeUpdated != nil { pvb := &velerov1api.PodVolumeBackup{} err := client.Get(t.Context(), ctrlclient.ObjectKey{Namespace: c.pvb.Namespace, Name: c.pvb.Name}, pvb) @@ -795,9 +816,22 @@ func TestWaitAllPodVolumesProcessed(t *testing.T) { pvbs := backuper.WaitAllPodVolumesProcessed(logger) if c.expectedErr != "" { - assert.Equal(t, c.expectedErr, logHook.entry.Message) + found := false + var loggedMsgs []string + for _, entry := range logHook.entries { + loggedMsgs = append(loggedMsgs, entry.Message) + if entry.Message == c.expectedErr { + found = true + break + } + } + assert.True(t, found, "Expected error %q to be logged, but got %v", c.expectedErr, loggedMsgs) } else { - assert.Nil(t, logHook.entry) + assert.Empty(t, logHook.entries) + } + + if c.expectedPVBCount > 0 { + require.Len(t, pvbs, c.expectedPVBCount) } if c.expectedPVBPhase != "" { diff --git a/pkg/podvolume/restore_micro_service.go b/pkg/podvolume/restore_micro_service.go index decf94b1bb..72b71b4669 100644 --- a/pkg/podvolume/restore_micro_service.go +++ b/pkg/podvolume/restore_micro_service.go @@ -23,7 +23,7 @@ import ( "path/filepath" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/podvolume/restore_micro_service_test.go b/pkg/podvolume/restore_micro_service_test.go index 2c94ddf467..e5f8a513fd 100644 --- a/pkg/podvolume/restore_micro_service_test.go +++ b/pkg/podvolume/restore_micro_service_test.go @@ -24,7 +24,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/podvolume/restorer.go b/pkg/podvolume/restorer.go index 47219ae990..589c431c2c 100644 --- a/pkg/podvolume/restorer.go +++ b/pkg/podvolume/restorer.go @@ -23,7 +23,7 @@ import ( "github.com/vmware-tanzu/velero/internal/volume" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -106,9 +106,9 @@ func newRestorer( if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseCompleted || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseFailed || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseCanceled { r.resultsLock.Lock() - defer r.resultsLock.Unlock() - resChan, ok := r.results[resultsKey(pvr.Spec.Pod.Namespace, pvr.Spec.Pod.Name)] + r.resultsLock.Unlock() + if !ok { log.Errorf("No results channel found for pod %s/%s to send pod volume restore %s/%s on", pvr.Spec.Pod.Namespace, pvr.Spec.Pod.Name, pvr.Namespace, pvr.Name) return @@ -147,7 +147,7 @@ func (r *restorer) RestorePodVolumes(data RestoreData, tracker *volume.RestoreVo r.repoLocker.Lock(repo.Name) defer r.repoLocker.Unlock(repo.Name) - resultsChan := make(chan *velerov1api.PodVolumeRestore) + resultsChan := make(chan *velerov1api.PodVolumeRestore, len(volumesToRestore)) r.resultsLock.Lock() r.results[resultsKey(data.Pod.Namespace, data.Pod.Name)] = resultsChan diff --git a/pkg/podvolume/restorer_factory.go b/pkg/podvolume/restorer_factory.go index 178d720c87..6a037b9926 100644 --- a/pkg/podvolume/restorer_factory.go +++ b/pkg/podvolume/restorer_factory.go @@ -19,7 +19,7 @@ package podvolume import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" diff --git a/pkg/repository/backup_repo_op.go b/pkg/repository/backup_repo_op.go index 36356e7aff..146418d0e9 100644 --- a/pkg/repository/backup_repo_op.go +++ b/pkg/repository/backup_repo_op.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/repository/config/aws.go b/pkg/repository/config/aws.go index 76a2829a5e..3c8e75b22f 100644 --- a/pkg/repository/config/aws.go +++ b/pkg/repository/config/aws.go @@ -31,7 +31,7 @@ import ( awsconfig "github.com/aws/aws-sdk-go-v2/config" s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" ) // getS3CredentialsFunc is used to make testing more convenient diff --git a/pkg/repository/config/azure.go b/pkg/repository/config/azure.go index 6662d13c6d..28724dda17 100644 --- a/pkg/repository/config/azure.go +++ b/pkg/repository/config/azure.go @@ -17,7 +17,7 @@ limitations under the License. package config import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/util/azure" ) diff --git a/pkg/repository/config/config.go b/pkg/repository/config/config.go index 46a5478e67..3d3c77b8d9 100644 --- a/pkg/repository/config/config.go +++ b/pkg/repository/config/config.go @@ -21,7 +21,7 @@ import ( "path" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/persistence" diff --git a/pkg/repository/config/config_test.go b/pkg/repository/config/config_test.go index aac5fc9bce..4195732d2c 100644 --- a/pkg/repository/config/config_test.go +++ b/pkg/repository/config/config_test.go @@ -19,7 +19,7 @@ package config import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/pkg/repository/ensurer.go b/pkg/repository/ensurer.go index 91cdb0d9e3..e3c20fdd85 100644 --- a/pkg/repository/ensurer.go +++ b/pkg/repository/ensurer.go @@ -21,7 +21,7 @@ import ( "sync" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/wait" diff --git a/pkg/repository/keys/keys.go b/pkg/repository/keys/keys.go index 21423afe09..a077d41537 100644 --- a/pkg/repository/keys/keys.go +++ b/pkg/repository/keys/keys.go @@ -20,7 +20,7 @@ package keys import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/repository/maintenance/maintenance.go b/pkg/repository/maintenance/maintenance.go index 747d89f524..1a74660e66 100644 --- a/pkg/repository/maintenance/maintenance.go +++ b/pkg/repository/maintenance/maintenance.go @@ -25,7 +25,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" appsv1api "k8s.io/api/apps/v1" batchv1api "k8s.io/api/batch/v1" @@ -350,7 +350,7 @@ func WaitJobComplete(cli client.Client, ctx context.Context, jobName, ns string, if maintenanceJob.Status.Failed > 0 { if r, err := getResultFromJob(cli, maintenanceJob); err != nil { log.WithError(err).Warn("Failed to get maintenance job result") - result = "Repo maintenance failed but result is not retrieveable" + result = "Repo maintenance failed but result is not retrievable" } else { result = r } @@ -413,7 +413,7 @@ func WaitAllJobsComplete(ctx context.Context, cli client.Client, repo *velerov1a if job.Status.Failed > 0 { if msg, err := getResultFromJob(cli, job); err != nil { log.WithError(err).Warnf("Failed to get result of maintenance job %s", job.Name) - message = fmt.Sprintf("Repo maintenance failed but result is not retrieveable, err: %v", err) + message = fmt.Sprintf("Repo maintenance failed but result is not retrievable, err: %v", err) } else { message = msg } diff --git a/pkg/repository/maintenance/maintenance_test.go b/pkg/repository/maintenance/maintenance_test.go index 97eee1148e..c7e225ddc9 100644 --- a/pkg/repository/maintenance/maintenance_test.go +++ b/pkg/repository/maintenance/maintenance_test.go @@ -789,7 +789,7 @@ func TestWaitAllJobsComplete(t *testing.T) { { Result: velerov1api.BackupRepositoryMaintenanceFailed, StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, - Message: "Repo maintenance failed but result is not retrieveable, err: no pod found for job job2", + Message: "Repo maintenance failed but result is not retrievable, err: no pod found for job job2", }, }, }, diff --git a/pkg/repository/manager/manager.go b/pkg/repository/manager/manager.go index abe76299fa..ed35dcb758 100644 --- a/pkg/repository/manager/manager.go +++ b/pkg/repository/manager/manager.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/repository/provider/unified_repo.go b/pkg/repository/provider/unified_repo.go index 2af59f1903..bfe1a2bd9d 100644 --- a/pkg/repository/provider/unified_repo.go +++ b/pkg/repository/provider/unified_repo.go @@ -27,7 +27,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/internal/credentials" diff --git a/pkg/repository/restic/repository.go b/pkg/repository/restic/repository.go index 7260542e10..00ac120736 100644 --- a/pkg/repository/restic/repository.go +++ b/pkg/repository/restic/repository.go @@ -20,7 +20,7 @@ import ( "os" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/internal/credentials" diff --git a/pkg/repository/udmrepo/kopialib/backend/file_system.go b/pkg/repository/udmrepo/kopialib/backend/file_system.go index f0999e832a..e3bf9e0c4a 100644 --- a/pkg/repository/udmrepo/kopialib/backend/file_system.go +++ b/pkg/repository/udmrepo/kopialib/backend/file_system.go @@ -23,9 +23,9 @@ import ( "github.com/sirupsen/logrus" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/blob/filesystem" - "github.com/pkg/errors" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/logging" diff --git a/pkg/repository/udmrepo/kopialib/backend/utils.go b/pkg/repository/udmrepo/kopialib/backend/utils.go index 62ba4c3228..d61a074101 100644 --- a/pkg/repository/udmrepo/kopialib/backend/utils.go +++ b/pkg/repository/udmrepo/kopialib/backend/utils.go @@ -22,8 +22,8 @@ import ( "strconv" "time" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/repo/logging" - "github.com/pkg/errors" ) func mustHaveString(key string, flags map[string]string) (string, error) { diff --git a/pkg/repository/udmrepo/kopialib/lib_repo.go b/pkg/repository/udmrepo/kopialib/lib_repo.go index e6c46ae66f..d9f185b728 100644 --- a/pkg/repository/udmrepo/kopialib/lib_repo.go +++ b/pkg/repository/udmrepo/kopialib/lib_repo.go @@ -25,6 +25,7 @@ import ( "sync/atomic" "time" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/compression" "github.com/kopia/kopia/repo/content/index" @@ -32,7 +33,6 @@ import ( "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" "github.com/kopia/kopia/snapshot/snapshotmaintenance" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/kopia" diff --git a/pkg/repository/udmrepo/kopialib/lib_repo_test.go b/pkg/repository/udmrepo/kopialib/lib_repo_test.go index 2feabaeca3..d254680d95 100644 --- a/pkg/repository/udmrepo/kopialib/lib_repo_test.go +++ b/pkg/repository/udmrepo/kopialib/lib_repo_test.go @@ -23,10 +23,10 @@ import ( "testing" "time" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/repository/udmrepo/kopialib/repo_init.go b/pkg/repository/udmrepo/kopialib/repo_init.go index ade9039d73..5c272298c5 100644 --- a/pkg/repository/udmrepo/kopialib/repo_init.go +++ b/pkg/repository/udmrepo/kopialib/repo_init.go @@ -26,11 +26,11 @@ import ( "github.com/sirupsen/logrus" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/maintenance" - "github.com/pkg/errors" "github.com/vmware-tanzu/velero/pkg/kopia" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" @@ -43,12 +43,18 @@ type kopiaBackendStore struct { store backend.Store } +type kopiaBackendStoreFactory struct { + name string + description string + newStore func() backend.Store +} + // backendStores lists the supported backend storages at present -var backendStores = []kopiaBackendStore{ - {udmrepo.StorageTypeAzure, "an Azure blob storage", &backend.AzureBackend{}}, - {udmrepo.StorageTypeFs, "a filesystem", &backend.FsBackend{}}, - {udmrepo.StorageTypeGcs, "a Google Cloud Storage bucket", &backend.GCSBackend{}}, - {udmrepo.StorageTypeS3, "an S3 bucket", &backend.S3Backend{}}, +var backendStores = []kopiaBackendStoreFactory{ + {udmrepo.StorageTypeAzure, "an Azure blob storage", func() backend.Store { return &backend.AzureBackend{} }}, + {udmrepo.StorageTypeFs, "a filesystem", func() backend.Store { return &backend.FsBackend{} }}, + {udmrepo.StorageTypeGcs, "a Google Cloud Storage bucket", func() backend.Store { return &backend.GCSBackend{} }}, + {udmrepo.StorageTypeS3, "an S3 bucket", func() backend.Store { return &backend.S3Backend{} }}, } const udmRepoBlobID = "udmrepo.Repository" @@ -226,7 +232,11 @@ func connectStore(ctx context.Context, repoOption udmrepo.RepoOptions, logger lo func findBackendStore(storage string) *kopiaBackendStore { for _, options := range backendStores { if strings.EqualFold(options.name, storage) { - return &options + return &kopiaBackendStore{ + name: options.name, + description: options.description, + store: options.newStore(), + } } } diff --git a/pkg/repository/udmrepo/kopialib/repo_init_test.go b/pkg/repository/udmrepo/kopialib/repo_init_test.go index 3b8a52be28..130bb7b4d4 100644 --- a/pkg/repository/udmrepo/kopialib/repo_init_test.go +++ b/pkg/repository/udmrepo/kopialib/repo_init_test.go @@ -38,9 +38,32 @@ import ( repomocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/mocks" storagemocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/mocks" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" ) +func TestFindBackendStore(t *testing.T) { + // findBackendStore should return a unique instance on each call + // so that concurrently executing controllers do not overwrite each other's credentials/options. + t.Run("returns distinct instances", func(t *testing.T) { + store1 := findBackendStore(udmrepo.StorageTypeS3) + require.NotNil(t, store1) + + store2 := findBackendStore(udmrepo.StorageTypeS3) + require.NotNil(t, store2) + + // The pointers to the wrapper struct must be different + assert.NotSame(t, store1, store2, "findBackendStore should return different kopiaBackendStore instances") + + // The pointers to the actual underlying store must be different + assert.NotSame(t, store1.store, store2.store, "findBackendStore should return different backend.Store instances") + }) + + t.Run("returns nil for unknown storage type", func(t *testing.T) { + store := findBackendStore("unknown-type") + assert.Nil(t, store) + }) +} + type comparableError struct { message string } @@ -133,11 +156,11 @@ func TestCreateBackupRepo(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { logger := velerotest.NewLogger() - backendStores = []kopiaBackendStore{ - {udmrepo.StorageTypeAzure, "fake store", tc.backendStore}, - {udmrepo.StorageTypeFs, "fake store", tc.backendStore}, - {udmrepo.StorageTypeGcs, "fake store", tc.backendStore}, - {udmrepo.StorageTypeS3, "fake store", tc.backendStore}, + backendStores = []kopiaBackendStoreFactory{ + {udmrepo.StorageTypeAzure, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeFs, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeGcs, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeS3, "fake store", func() backend.Store { return tc.backendStore }}, } if tc.backendStore != nil { @@ -219,11 +242,11 @@ func TestConnectBackupRepo(t *testing.T) { logger := velerotest.NewLogger() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - backendStores = []kopiaBackendStore{ - {udmrepo.StorageTypeAzure, "fake store", tc.backendStore}, - {udmrepo.StorageTypeFs, "fake store", tc.backendStore}, - {udmrepo.StorageTypeGcs, "fake store", tc.backendStore}, - {udmrepo.StorageTypeS3, "fake store", tc.backendStore}, + backendStores = []kopiaBackendStoreFactory{ + {udmrepo.StorageTypeAzure, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeFs, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeGcs, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeS3, "fake store", func() backend.Store { return tc.backendStore }}, } if tc.backendStore != nil { @@ -441,11 +464,11 @@ func TestGetRepositoryStatus(t *testing.T) { logger := velerotest.NewLogger() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - backendStores = []kopiaBackendStore{ - {udmrepo.StorageTypeAzure, "fake store", tc.backendStore}, - {udmrepo.StorageTypeFs, "fake store", tc.backendStore}, - {udmrepo.StorageTypeGcs, "fake store", tc.backendStore}, - {udmrepo.StorageTypeS3, "fake store", tc.backendStore}, + backendStores = []kopiaBackendStoreFactory{ + {udmrepo.StorageTypeAzure, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeFs, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeGcs, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeS3, "fake store", func() backend.Store { return tc.backendStore }}, } if tc.backendStore != nil { diff --git a/pkg/restic/common.go b/pkg/restic/common.go index a5bf05c447..4bef3c0589 100644 --- a/pkg/restic/common.go +++ b/pkg/restic/common.go @@ -23,7 +23,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/internal/credentials" diff --git a/pkg/restic/exec_commands.go b/pkg/restic/exec_commands.go index 94c17c04a4..02bd9aedc5 100644 --- a/pkg/restic/exec_commands.go +++ b/pkg/restic/exec_commands.go @@ -23,7 +23,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/uploader" diff --git a/pkg/restore/actions/add_pvc_from_pod_action.go b/pkg/restore/actions/add_pvc_from_pod_action.go index 3e88f796a2..3dd92424e6 100644 --- a/pkg/restore/actions/add_pvc_from_pod_action.go +++ b/pkg/restore/actions/add_pvc_from_pod_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/restore/actions/admissionwebhook_config_action.go b/pkg/restore/actions/admissionwebhook_config_action.go index 82599dc62f..3291fff71f 100644 --- a/pkg/restore/actions/admissionwebhook_config_action.go +++ b/pkg/restore/actions/admissionwebhook_config_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/restore/actions/change_image_name_action.go b/pkg/restore/actions/change_image_name_action.go index 828da40d6a..69e9e5f33b 100644 --- a/pkg/restore/actions/change_image_name_action.go +++ b/pkg/restore/actions/change_image_name_action.go @@ -21,7 +21,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/restore/actions/change_storageclass_action.go b/pkg/restore/actions/change_storageclass_action.go index f9f031fe32..bdee2aa13c 100644 --- a/pkg/restore/actions/change_storageclass_action.go +++ b/pkg/restore/actions/change_storageclass_action.go @@ -19,7 +19,7 @@ package actions import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" diff --git a/pkg/restore/actions/change_storageclass_action_test.go b/pkg/restore/actions/change_storageclass_action_test.go index 13bbcdcc46..72cab80e70 100644 --- a/pkg/restore/actions/change_storageclass_action_test.go +++ b/pkg/restore/actions/change_storageclass_action_test.go @@ -19,7 +19,7 @@ package actions import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/pkg/restore/actions/clusterrolebinding_action.go b/pkg/restore/actions/clusterrolebinding_action.go index a11665c1ae..edc2ed9618 100644 --- a/pkg/restore/actions/clusterrolebinding_action.go +++ b/pkg/restore/actions/clusterrolebinding_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/actions/crd_v1_preserve_unknown_fields_action.go b/pkg/restore/actions/crd_v1_preserve_unknown_fields_action.go index 9e4cf7e2fa..fda389de4e 100644 --- a/pkg/restore/actions/crd_v1_preserve_unknown_fields_action.go +++ b/pkg/restore/actions/crd_v1_preserve_unknown_fields_action.go @@ -19,7 +19,7 @@ package actions import ( "encoding/json" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/actions/csi/pvc_action.go b/pkg/restore/actions/csi/pvc_action.go index dee23cf707..e91be77067 100644 --- a/pkg/restore/actions/csi/pvc_action.go +++ b/pkg/restore/actions/csi/pvc_action.go @@ -23,7 +23,7 @@ import ( snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -176,6 +176,15 @@ func (p *pvcRestoreItemAction) Execute( Name: vsName, Namespace: pvc.Namespace, }) + + // Force-restore the VolumeSnapshot even when restore resource filters + // would otherwise exclude it (mirrors backup-side must-include). + annotations := pvc.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + pvc.SetAnnotations(annotations) } } diff --git a/pkg/restore/actions/csi/pvc_action_test.go b/pkg/restore/actions/csi/pvc_action_test.go index bc7f66d891..7be22f27bb 100644 --- a/pkg/restore/actions/csi/pvc_action_test.go +++ b/pkg/restore/actions/csi/pvc_action_test.go @@ -371,6 +371,7 @@ func TestExecute(t *testing.T) { backup *velerov1api.Backup restore *velerov1api.Restore pvc *corev1api.PersistentVolumeClaim + pvcFromBackup *corev1api.PersistentVolumeClaim vs *snapshotv1api.VolumeSnapshot dataUploadResult *corev1api.ConfigMap expectedErr string @@ -402,15 +403,40 @@ func TestExecute(t *testing.T) { vs: builder.ForVolumeSnapshot("velero", vsName).ObjectMeta( builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi"), ).Result(), - expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName")).Result(), + expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations( + velerov1api.VolumeSnapshotLabel, "vsName", + velerov1api.MustIncludeAdditionalItemRestoreAnnotation, "true", + )).Result(), }, { - name: "Restore from VolumeSnapshot without volume-snapshot-name annotation", - backup: builder.ForBackup("velero", "testBackup").Result(), - restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").Result(), - pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(AnnSelectedNode, "node1")).Result(), - vs: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi")).Result(), - expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(AnnSelectedNode, "node1")).Result(), + name: "Restore from VolumeSnapshot with nil PVC annotations", + backup: builder.ForBackup("velero", "testBackup").Result(), + restore: builder.ForRestore("velero", "testRestore").ObjectMeta(builder.WithUID("restoreUID")).Backup("testBackup").Result(), + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPVC", + Namespace: "velero", + }, + }, + pvcFromBackup: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName")).Result(), + vs: builder.ForVolumeSnapshot("velero", vsName).ObjectMeta( + builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi"), + ).Result(), + expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations( + velerov1api.MustIncludeAdditionalItemRestoreAnnotation, "true", + )).Result(), + }, + { + name: "Restore from VolumeSnapshot without volume-snapshot-name annotation", + backup: builder.ForBackup("velero", "testBackup").Result(), + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName", AnnSelectedNode, "node1")).Result(), + vs: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi")).Result(), + expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations( + velerov1api.VolumeSnapshotLabel, "vsName", + AnnSelectedNode, "node1", + velerov1api.MustIncludeAdditionalItemRestoreAnnotation, "true", + )).Result(), }, { name: "DataUploadResult cannot be found", @@ -480,7 +506,13 @@ func TestExecute(t *testing.T) { require.NoError(t, err) input.Item = &unstructured.Unstructured{Object: pvcMap} - input.ItemFromBackup = &unstructured.Unstructured{Object: pvcMap} + if tc.pvcFromBackup != nil { + pvcFromBackupMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pvcFromBackup) + require.NoError(t, err) + input.ItemFromBackup = &unstructured.Unstructured{Object: pvcFromBackupMap} + } else { + input.ItemFromBackup = &unstructured.Unstructured{Object: pvcMap} + } input.Restore = tc.restore } if tc.preCreatePVC { @@ -508,6 +540,12 @@ func TestExecute(t *testing.T) { err := runtime.DefaultUnstructuredConverter.FromUnstructured(output.UpdatedItem.UnstructuredContent(), pvc) require.NoError(t, err) require.Equal(t, tc.expectedPVC.GetObjectMeta(), pvc.GetObjectMeta()) + if tc.name == "Restore from VolumeSnapshot" { + require.Equal(t, "true", pvc.GetAnnotations()[velerov1api.MustIncludeAdditionalItemRestoreAnnotation]) + require.Len(t, output.AdditionalItems, 1) + require.Equal(t, "volumesnapshots.snapshot.storage.k8s.io", output.AdditionalItems[0].GroupResource.String()) + require.Equal(t, "vsName", output.AdditionalItems[0].Name) + } if pvc.Spec.Selector != nil && pvc.Spec.Selector.MatchLabels != nil { // This is used for long name and namespace case. if len(tc.pvc.Namespace+"."+tc.pvc.Name) >= validation.DNS1035LabelMaxLength { diff --git a/pkg/restore/actions/csi/volumesnapshot_action.go b/pkg/restore/actions/csi/volumesnapshot_action.go index dec33d4efe..13b7cb2464 100644 --- a/pkg/restore/actions/csi/volumesnapshot_action.go +++ b/pkg/restore/actions/csi/volumesnapshot_action.go @@ -20,9 +20,9 @@ import ( "context" "fmt" + "github.com/cockroachdb/errors" volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -66,6 +66,9 @@ func resetVolumeSnapshotSpecForRestore(vs *snapshotv1api.VolumeSnapshot, vscName } func resetVolumeSnapshotAnnotation(vs *snapshotv1api.VolumeSnapshot) { + if vs.ObjectMeta.Annotations == nil { + vs.ObjectMeta.Annotations = make(map[string]string) + } vs.ObjectMeta.Annotations[velerov1api.VSCDeletionPolicyAnnotation] = string(snapshotv1api.VolumeSnapshotContentRetain) } @@ -282,12 +285,6 @@ func (p *volumeSnapshotRestoreItemAction) Execute( vs.Namespace, vs.Name) } - vsMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&vs) - if err != nil { - p.log.Errorf("Fail to convert VS %s to unstructured", vs.Namespace+"/"+vs.Name) - return nil, errors.WithStack(err) - } - if vsFromBackup.Status == nil || vsFromBackup.Status.BoundVolumeSnapshotContentName == nil { p.log.Errorf("VS %s doesn't have bound VSC", vsFromBackup.Name) @@ -299,6 +296,21 @@ func (p *volumeSnapshotRestoreItemAction) Execute( Name: *vsFromBackup.Status.BoundVolumeSnapshotContentName, } + // Force-restore the bound VSC even when restore resource filters would + // otherwise exclude it (mirrors backup-side must-include for CSI deps). + annotations := vs.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + vs.SetAnnotations(annotations) + + vsMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&vs) + if err != nil { + p.log.Errorf("Fail to convert VS %s to unstructured", vs.Namespace+"/"+vs.Name) + return nil, errors.WithStack(err) + } + p.log.Infof(`Returning from VolumeSnapshotRestoreItemAction with VolumeSnapshotContent in additionalItems`) diff --git a/pkg/restore/actions/csi/volumesnapshot_action_test.go b/pkg/restore/actions/csi/volumesnapshot_action_test.go index de3e592c03..9d72971d05 100644 --- a/pkg/restore/actions/csi/volumesnapshot_action_test.go +++ b/pkg/restore/actions/csi/volumesnapshot_action_test.go @@ -103,6 +103,26 @@ func TestResetVolumeSnapshotSpecForRestore(t *testing.T) { } } +func TestResetVolumeSnapshotAnnotation(t *testing.T) { + t.Run("should set deletion policy annotation when annotations is nil", func(t *testing.T) { + vs := snapshotv1api.VolumeSnapshot{} + resetVolumeSnapshotAnnotation(&vs) + assert.NotNil(t, vs.ObjectMeta.Annotations) + assert.Equal(t, string(snapshotv1api.VolumeSnapshotContentRetain), vs.ObjectMeta.Annotations[velerov1api.VSCDeletionPolicyAnnotation]) + }) + + t.Run("should preserve existing annotations and set deletion policy annotation", func(t *testing.T) { + vs := snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"foo": "bar"}, + }, + } + resetVolumeSnapshotAnnotation(&vs) + assert.Equal(t, "bar", vs.ObjectMeta.Annotations["foo"]) + assert.Equal(t, string(snapshotv1api.VolumeSnapshotContentRetain), vs.ObjectMeta.Annotations[velerov1api.VSCDeletionPolicyAnnotation]) + }) +} + func TestVSExecute(t *testing.T) { newVscName := util.GenerateSha256FromRestoreUIDAndVsName("restoreUID", "vsName") tests := []struct { @@ -145,6 +165,18 @@ func TestVSExecute(t *testing.T) { expectErr: false, expectedVS: builder.ForVolumeSnapshot("ns", "test").SourceVolumeSnapshotContentName(newVscName).Result(), }, + { + name: "Normal case with nil VS annotations, VSC should be created", + vs: builder.ForVolumeSnapshot("ns", "vsName"). + SourceVolumeSnapshotContentName(newVscName). + VolumeSnapshotClass("vscClass"). + Status(). + BoundVolumeSnapshotContentName("vscName"). + Result(), + restore: builder.ForRestore("velero", "restore").ObjectMeta(builder.WithUID("restoreUID")).Result(), + expectErr: false, + expectedVS: builder.ForVolumeSnapshot("ns", "test").SourceVolumeSnapshotContentName(newVscName).Result(), + }, } for _, test := range tests { @@ -184,6 +216,10 @@ func TestVSExecute(t *testing.T) { require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured( result.UpdatedItem.UnstructuredContent(), &vs)) require.Equal(t, test.expectedVS.Spec, vs.Spec) + require.Equal(t, "true", vs.GetAnnotations()[velerov1api.MustIncludeAdditionalItemRestoreAnnotation]) + require.Len(t, result.AdditionalItems, 1) + require.Equal(t, "volumesnapshotcontents.snapshot.storage.k8s.io", result.AdditionalItems[0].GroupResource.String()) + require.Equal(t, "vscName", result.AdditionalItems[0].Name) } }) } diff --git a/pkg/restore/actions/csi/volumesnapshotclass_action.go b/pkg/restore/actions/csi/volumesnapshotclass_action.go index c906a04b22..595e39b31f 100644 --- a/pkg/restore/actions/csi/volumesnapshotclass_action.go +++ b/pkg/restore/actions/csi/volumesnapshotclass_action.go @@ -17,8 +17,8 @@ limitations under the License. package csi import ( + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/restore/actions/csi/volumesnapshotcontent_action.go b/pkg/restore/actions/csi/volumesnapshotcontent_action.go index 00a25c86f8..dc18b9bb36 100644 --- a/pkg/restore/actions/csi/volumesnapshotcontent_action.go +++ b/pkg/restore/actions/csi/volumesnapshotcontent_action.go @@ -19,8 +19,8 @@ package csi import ( "context" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/restore/actions/dataupload_retrieve_action.go b/pkg/restore/actions/dataupload_retrieve_action.go index a7efdc5f72..4d750d0551 100644 --- a/pkg/restore/actions/dataupload_retrieve_action.go +++ b/pkg/restore/actions/dataupload_retrieve_action.go @@ -20,7 +20,7 @@ import ( "context" "encoding/json" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/restore/actions/init_restorehook_pod_action.go b/pkg/restore/actions/init_restorehook_pod_action.go index 7614ef0850..f6fee4eeb8 100644 --- a/pkg/restore/actions/init_restorehook_pod_action.go +++ b/pkg/restore/actions/init_restorehook_pod_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/actions/job_action.go b/pkg/restore/actions/job_action.go index 1eabc208c3..5ee4b8c6a9 100644 --- a/pkg/restore/actions/job_action.go +++ b/pkg/restore/actions/job_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" batchv1api "k8s.io/api/batch/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/actions/pod_action.go b/pkg/restore/actions/pod_action.go index a9db3ed7e5..ca12c9031a 100644 --- a/pkg/restore/actions/pod_action.go +++ b/pkg/restore/actions/pod_action.go @@ -19,7 +19,7 @@ package actions import ( "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/actions/pod_volume_restore_action.go b/pkg/restore/actions/pod_volume_restore_action.go index e26a530347..cbfcbfb35c 100644 --- a/pkg/restore/actions/pod_volume_restore_action.go +++ b/pkg/restore/actions/pod_volume_restore_action.go @@ -23,7 +23,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/util/boolptr" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" @@ -198,6 +198,25 @@ func (a *PodVolumeRestoreAction) Execute(input *velero.RestoreItemActionExecuteI securityContext = *pod.Spec.Containers[0].SecurityContext.DeepCopy() securityContextSet = true } + // if no configmap or container-level securityContext is set, fall back to the pod-level + // spec.securityContext runAsUser/runAsGroup: the workload's own identity is the one that + // wrote the restored files, so it's the one that can read them back + if !securityContextSet && pod.Spec.SecurityContext != nil && + (pod.Spec.SecurityContext.RunAsUser != nil || pod.Spec.SecurityContext.RunAsGroup != nil) { + securityContext = defaultSecurityCtx() + if pod.Spec.SecurityContext.RunAsUser != nil { + securityContext.RunAsUser = pod.Spec.SecurityContext.RunAsUser + // defaultSecurityCtx() hardcodes RunAsNonRoot: true, which contradicts a pod-level + // RunAsUser of 0 (root); defer to the pod's own RunAsNonRoot setting in that case + if *pod.Spec.SecurityContext.RunAsUser == 0 { + securityContext.RunAsNonRoot = pod.Spec.SecurityContext.RunAsNonRoot + } + } + if pod.Spec.SecurityContext.RunAsGroup != nil { + securityContext.RunAsGroup = pod.Spec.SecurityContext.RunAsGroup + } + securityContextSet = true + } if !securityContextSet { securityContext = defaultSecurityCtx() } diff --git a/pkg/restore/actions/pod_volume_restore_action_test.go b/pkg/restore/actions/pod_volume_restore_action_test.go index bc9662ab7d..614a5d1bec 100644 --- a/pkg/restore/actions/pod_volume_restore_action_test.go +++ b/pkg/restore/actions/pod_volume_restore_action_test.go @@ -156,6 +156,155 @@ func TestPodVolumeRestoreActionExecute(t *testing.T) { defaultRestoreHelperImage := "velero/velero:v1.0" + podLevelUID := int64(999) + podLevelGID := int64(999) + podLevelSecurityContext := corev1api.SecurityContext{ + AllowPrivilegeEscalation: boolptr.False(), + Capabilities: &corev1api.Capabilities{ + Drop: []corev1api.Capability{"ALL"}, + }, + SeccompProfile: &corev1api.SeccompProfile{ + Type: corev1api.SeccompProfileTypeRuntimeDefault, + }, + RunAsUser: &podLevelUID, + RunAsGroup: &podLevelGID, + RunAsNonRoot: boolptr.True(), + } + + podWithPodLevelSecurityContext := builder.ForPod("ns-1", "my-pod"). + ObjectMeta(builder.WithAnnotations("snapshot.velero.io/myvol", "")). + Volumes( + builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), + ). + Result() + podWithPodLevelSecurityContext.Spec.SecurityContext = &corev1api.PodSecurityContext{RunAsUser: &podLevelUID, RunAsGroup: &podLevelGID} + + wantPodWithPodLevelSecurityContext := builder.ForPod("ns-1", "my-pod"). + ObjectMeta(builder.WithAnnotations("snapshot.velero.io/myvol", "")). + Volumes( + builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), + ). + InitContainers( + newRestoreInitContainerBuilder(defaultRestoreHelperImage, ""). + Resources(&resourceReqs). + SecurityContext(&podLevelSecurityContext). + VolumeMounts(builder.ForVolumeMount("myvol", "/restores/myvol").Result()). + Command([]string{"/velero-restore-helper"}).Result()). + Result() + wantPodWithPodLevelSecurityContext.Spec.SecurityContext = &corev1api.PodSecurityContext{RunAsUser: &podLevelUID, RunAsGroup: &podLevelGID} + + podLevelRootUID := int64(0) + podLevelRootSecurityContext := corev1api.SecurityContext{ + AllowPrivilegeEscalation: boolptr.False(), + Capabilities: &corev1api.Capabilities{ + Drop: []corev1api.Capability{"ALL"}, + }, + SeccompProfile: &corev1api.SeccompProfile{ + Type: corev1api.SeccompProfileTypeRuntimeDefault, + }, + RunAsUser: &podLevelRootUID, + } + + podWithPodLevelRootSecurityContext := builder.ForPod("ns-1", "my-pod"). + ObjectMeta(builder.WithAnnotations("snapshot.velero.io/myvol", "")). + Volumes( + builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), + ). + Result() + podWithPodLevelRootSecurityContext.Spec.SecurityContext = &corev1api.PodSecurityContext{RunAsUser: &podLevelRootUID} + + wantPodWithPodLevelRootSecurityContext := builder.ForPod("ns-1", "my-pod"). + ObjectMeta(builder.WithAnnotations("snapshot.velero.io/myvol", "")). + Volumes( + builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), + ). + InitContainers( + newRestoreInitContainerBuilder(defaultRestoreHelperImage, ""). + Resources(&resourceReqs). + SecurityContext(&podLevelRootSecurityContext). + VolumeMounts(builder.ForVolumeMount("myvol", "/restores/myvol").Result()). + Command([]string{"/velero-restore-helper"}).Result()). + Result() + wantPodWithPodLevelRootSecurityContext.Spec.SecurityContext = &corev1api.PodSecurityContext{RunAsUser: &podLevelRootUID} + + podLevelGroupOnlyGID := int64(777) + podLevelGroupOnlySecurityContext := corev1api.SecurityContext{ + AllowPrivilegeEscalation: boolptr.False(), + Capabilities: &corev1api.Capabilities{ + Drop: []corev1api.Capability{"ALL"}, + }, + SeccompProfile: &corev1api.SeccompProfile{ + Type: corev1api.SeccompProfileTypeRuntimeDefault, + }, + RunAsUser: &id, + RunAsGroup: &podLevelGroupOnlyGID, + RunAsNonRoot: boolptr.True(), + } + + podWithPodLevelGroupOnlySecurityContext := builder.ForPod("ns-1", "my-pod"). + ObjectMeta(builder.WithAnnotations("snapshot.velero.io/myvol", "")). + Volumes( + builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), + ). + Result() + podWithPodLevelGroupOnlySecurityContext.Spec.SecurityContext = &corev1api.PodSecurityContext{RunAsGroup: &podLevelGroupOnlyGID} + + wantPodWithPodLevelGroupOnlySecurityContext := builder.ForPod("ns-1", "my-pod"). + ObjectMeta(builder.WithAnnotations("snapshot.velero.io/myvol", "")). + Volumes( + builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), + ). + InitContainers( + newRestoreInitContainerBuilder(defaultRestoreHelperImage, ""). + Resources(&resourceReqs). + SecurityContext(&podLevelGroupOnlySecurityContext). + VolumeMounts(builder.ForVolumeMount("myvol", "/restores/myvol").Result()). + Command([]string{"/velero-restore-helper"}).Result()). + Result() + wantPodWithPodLevelGroupOnlySecurityContext.Spec.SecurityContext = &corev1api.PodSecurityContext{RunAsGroup: &podLevelGroupOnlyGID} + + bothLevelsPodUID := int64(500) + bothLevelsContainerUID := int64(999) + bothLevelsContainerSecurityContext := corev1api.SecurityContext{ + AllowPrivilegeEscalation: boolptr.False(), + Capabilities: &corev1api.Capabilities{ + Drop: []corev1api.Capability{"ALL"}, + }, + SeccompProfile: &corev1api.SeccompProfile{ + Type: corev1api.SeccompProfileTypeRuntimeDefault, + }, + RunAsUser: &bothLevelsContainerUID, + RunAsNonRoot: boolptr.True(), + } + + podWithBothLevelsSecurityContext := builder.ForPod("ns-1", "my-pod"). + ObjectMeta(builder.WithAnnotations("snapshot.velero.io/myvol", "")). + Volumes( + builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), + ). + Containers( + builder.ForContainer("app-container", "app-image"). + SecurityContext(&bothLevelsContainerSecurityContext).Result()). + Result() + podWithBothLevelsSecurityContext.Spec.SecurityContext = &corev1api.PodSecurityContext{RunAsUser: &bothLevelsPodUID} + + wantPodWithBothLevelsSecurityContext := builder.ForPod("ns-1", "my-pod"). + ObjectMeta(builder.WithAnnotations("snapshot.velero.io/myvol", "")). + Volumes( + builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), + ). + Containers( + builder.ForContainer("app-container", "app-image"). + SecurityContext(&bothLevelsContainerSecurityContext).Result()). + InitContainers( + newRestoreInitContainerBuilder(defaultRestoreHelperImage, ""). + Resources(&resourceReqs). + SecurityContext(&bothLevelsContainerSecurityContext). + VolumeMounts(builder.ForVolumeMount("myvol", "/restores/myvol").Result()). + Command([]string{"/velero-restore-helper"}).Result()). + Result() + wantPodWithBothLevelsSecurityContext.Spec.SecurityContext = &corev1api.PodSecurityContext{RunAsUser: &bothLevelsPodUID} + tests := []struct { name string pod *corev1api.Pod @@ -350,6 +499,62 @@ func TestPodVolumeRestoreActionExecute(t *testing.T) { VolumeMounts(builder.ForVolumeMount("myvol", "/restores/myvol").Result()). Command([]string{"/velero-restore-helper"}).Result()).Result(), }, + { + name: "Restoring pod with pod-level securityContext (no container-level SecurityContext) uses pod-level runAsUser/runAsGroup for the restore initContainer", + pod: podWithPodLevelSecurityContext, + podVolumeBackups: []runtime.Object{ + builder.ForPodVolumeBackup(veleroNs, "pvb-1"). + PodName("my-pod"). + PodNamespace("ns-1"). + Volume("myvol"). + ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, backupName)). + SnapshotID("foo"). + Result(), + }, + want: wantPodWithPodLevelSecurityContext, + }, + { + name: "Restoring pod with pod-level securityContext.runAsUser=0 does not force RunAsNonRoot on the restore initContainer", + pod: podWithPodLevelRootSecurityContext, + podVolumeBackups: []runtime.Object{ + builder.ForPodVolumeBackup(veleroNs, "pvb-1"). + PodName("my-pod"). + PodNamespace("ns-1"). + Volume("myvol"). + ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, backupName)). + SnapshotID("foo"). + Result(), + }, + want: wantPodWithPodLevelRootSecurityContext, + }, + { + name: "Restoring pod with pod-level securityContext.runAsGroup only (no runAsUser) still applies the group to the restore initContainer", + pod: podWithPodLevelGroupOnlySecurityContext, + podVolumeBackups: []runtime.Object{ + builder.ForPodVolumeBackup(veleroNs, "pvb-1"). + PodName("my-pod"). + PodNamespace("ns-1"). + Volume("myvol"). + ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, backupName)). + SnapshotID("foo"). + Result(), + }, + want: wantPodWithPodLevelGroupOnlySecurityContext, + }, + { + name: "Restoring pod with both container-level and pod-level SecurityContext set uses the container-level SecurityContext for the restore initContainer (container-level takes priority)", + pod: podWithBothLevelsSecurityContext, + podVolumeBackups: []runtime.Object{ + builder.ForPodVolumeBackup(veleroNs, "pvb-1"). + PodName("my-pod"). + PodNamespace("ns-1"). + Volume("myvol"). + ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, backupName)). + SnapshotID("foo"). + Result(), + }, + want: wantPodWithBothLevelsSecurityContext, + }, { name: "pod volume backups in a different namespace are ignored when looking for matches due to namespace scoping", pod: builder.ForPod("ns-1", "my-pod"). diff --git a/pkg/restore/actions/pvc_action.go b/pkg/restore/actions/pvc_action.go index a4a63374dc..b9422d20f3 100644 --- a/pkg/restore/actions/pvc_action.go +++ b/pkg/restore/actions/pvc_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/actions/rolebinding_action.go b/pkg/restore/actions/rolebinding_action.go index 05e4635877..ff63f30221 100644 --- a/pkg/restore/actions/rolebinding_action.go +++ b/pkg/restore/actions/rolebinding_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/actions/secret_action.go b/pkg/restore/actions/secret_action.go index 2ec9fb4ff8..6517045ca6 100644 --- a/pkg/restore/actions/secret_action.go +++ b/pkg/restore/actions/secret_action.go @@ -21,7 +21,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/actions/service_account_action.go b/pkg/restore/actions/service_account_action.go index 429c21949b..fb410d384d 100644 --- a/pkg/restore/actions/service_account_action.go +++ b/pkg/restore/actions/service_account_action.go @@ -19,7 +19,7 @@ package actions import ( "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/actions/service_action.go b/pkg/restore/actions/service_action.go index 9de75228e7..afa3dd8a19 100644 --- a/pkg/restore/actions/service_action.go +++ b/pkg/restore/actions/service_action.go @@ -21,7 +21,7 @@ import ( "fmt" "strconv" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/merge_service_account.go b/pkg/restore/merge_service_account.go index 7abaa7ee20..6d6ea38cdd 100644 --- a/pkg/restore/merge_service_account.go +++ b/pkg/restore/merge_service_account.go @@ -19,8 +19,8 @@ package restore import ( "encoding/json" + "github.com/cockroachdb/errors" jsonpatch "github.com/evanphx/json-patch/v5" - "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/prioritize_group_version.go b/pkg/restore/prioritize_group_version.go index 5d7ab15d53..8801a3b0df 100644 --- a/pkg/restore/prioritize_group_version.go +++ b/pkg/restore/prioritize_group_version.go @@ -21,7 +21,7 @@ import ( "sort" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/restore/pv_restorer.go b/pkg/restore/pv_restorer.go index 53fbd0126a..cc851f72e7 100644 --- a/pkg/restore/pv_restorer.go +++ b/pkg/restore/pv_restorer.go @@ -19,7 +19,7 @@ package restore import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/restore/pv_restorer_test.go b/pkg/restore/pv_restorer_test.go index 09c6dd0ad0..2f40a9a931 100644 --- a/pkg/restore/pv_restorer_test.go +++ b/pkg/restore/pv_restorer_test.go @@ -21,7 +21,7 @@ import ( "github.com/sirupsen/logrus" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" diff --git a/pkg/restore/request.go b/pkg/restore/request.go index 239d65df92..57ab6f1196 100644 --- a/pkg/restore/request.go +++ b/pkg/restore/request.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/internal/resourcemodifiers" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/itemoperation" @@ -61,6 +62,7 @@ type Request struct { RestoredItems map[itemKey]restoredItemStatus itemOperationsList *[]*itemoperation.RestoreOperation ResourceModifiers *resourcemodifiers.ResourceModifiers + ResPolicies *resourcepolicies.Policies DisableInformerCache bool CSIVolumeSnapshots []*snapshotv1api.VolumeSnapshot BackupVolumeInfoMap map[string]volume.BackupVolumeInfo diff --git a/pkg/restore/restore.go b/pkg/restore/restore.go index 0a75fd89f8..3d2620912a 100644 --- a/pkg/restore/restore.go +++ b/pkg/restore/restore.go @@ -31,9 +31,10 @@ import ( "sync" "time" + "github.com/cockroachdb/errors" + "github.com/gobwas/glob" "github.com/google/uuid" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -55,6 +56,7 @@ import ( "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/internal/resourcemodifiers" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/archive" @@ -85,7 +87,6 @@ const ObjectStatusRestoreAnnotationKey = "velero.io/restore-status" var resourceMustHave = []string{ "datauploads.velero.io", - "volumesnapshotcontents.snapshot.storage.k8s.io", } type VolumeSnapshotterGetter interface { @@ -237,6 +238,43 @@ func (kr *kubernetesRestorer) RestoreWithResolvers( Includes(req.Restore.Spec.IncludedNamespaces...). Excludes(req.Restore.Spec.ExcludedNamespaces...) + var clusterScopedFilterMap map[string]*resolvedResourceFilter + var namespacedFilterMap map[string]*resolvedNamespaceFilter + var namespacedFilterPatterns []namespacedFilterPattern + + if req.ResPolicies != nil { + if kr.discoveryHelper == nil { + return results.Result{}, results.Result{Velero: []string{"failed to resolve namespace filter policies: discovery client unavailable"}} + } + + // Resolve clusterScopedFilterPolicy + csPolicy := req.ResPolicies.GetClusterScopedFilterPolicy() + if csPolicy != nil { + clusterScopedFilterMap, err = resolveRestoreClusterScopedFilterPolicy( + csPolicy, + kr.discoveryHelper, + req.Log, + ) + if err != nil { + return results.Result{}, results.Result{Velero: []string{err.Error()}} + } + } + + // Resolve namespacedFilterPolicies + nfPolicies := req.ResPolicies.GetNamespacedFilterPolicies() + if len(nfPolicies) > 0 { + namespacedFilterMap, namespacedFilterPatterns, err = resolveRestoreNamespacedFilterPolicies( + nfPolicies, + req.Restore.Spec.ExcludedResources, + kr.discoveryHelper, + req.Log, + ) + if err != nil { + return results.Result{}, results.Result{Velero: []string{err.Error()}} + } + } + } + resolvedActions, err := restoreItemActionResolver.ResolveActions(kr.discoveryHelper, kr.logger) if err != nil { return results.Result{}, results.Result{Velero: []string{err.Error()}} @@ -333,6 +371,10 @@ func (kr *kubernetesRestorer) RestoreWithResolvers( restoreVolumeInfoTracker: req.RestoreVolumeInfoTracker, hooksWaitExecutor: hooksWaitExecutor, resourceDeletionStatusTracker: req.ResourceDeletionStatusTracker, + clusterScopedFilterMap: clusterScopedFilterMap, + namespacedFilterMap: namespacedFilterMap, + namespacedFilterPatterns: namespacedFilterPatterns, + namespaceFilterCache: make(map[string]*resolvedNamespaceFilter), } return restoreCtx.execute() @@ -382,6 +424,249 @@ type restoreContext struct { restoreVolumeInfoTracker *volume.RestoreVolumeInfoTracker hooksWaitExecutor *hooksWaitExecutor resourceDeletionStatusTracker kube.ResourceDeletionStatusTracker + + // clusterScopedFilterMap holds resolved per-kind filters for cluster-scoped resources. + // Key is the resolved group-resource string. + clusterScopedFilterMap map[string]*resolvedResourceFilter + + // namespacedFilterMap holds resolved per-namespace filters. + // Key is either an exact namespace name or a glob pattern string. + namespacedFilterMap map[string]*resolvedNamespaceFilter + + // namespacedFilterPatterns preserves the order of patterns for first-match + // semantics and caches pre-compiled globs to avoid repeated compilation. + namespacedFilterPatterns []namespacedFilterPattern + + // namespaceFilterCache memoizes the resolved filter for a given namespace + // to avoid re-evaluating glob patterns on every call. + namespaceFilterCache map[string]*resolvedNamespaceFilter +} + +type resolvedResourceFilter struct { + labelSelector labels.Selector + orLabelSelectors []labels.Selector + nameIE *collections.IncludesExcludes + originalKinds []string +} + +type resolvedNamespaceFilter struct { + // resourceFilterMap is keyed by the resolved group-resource string + resourceFilterMap map[string]*resolvedResourceFilter + // catchAllFilter holds the resolved filter for a catch-all entry (empty kinds or ["*"]). + // nil when no catch-all entry is defined. + catchAllFilter *resolvedResourceFilter + // hasUnresolvedKinds is true if any kind in the policy failed discovery. + // This is used to bypass the fast-path skip so the peek-and-map fallback can run. + hasUnresolvedKinds bool +} + +// namespacedFilterPattern pairs a namespace pattern string with its pre-compiled +// glob so that getNamespaceFilter does not recompile on every call. +type namespacedFilterPattern struct { + pattern string + compiled glob.Glob // compiled once at restore start; nil for exact-match patterns +} + +func (ctx *restoreContext) getNamespaceFilter(namespace string) *resolvedNamespaceFilter { + if ctx.namespacedFilterMap == nil { + return nil + } + + // 1. Check the cache first + if filter, ok := ctx.namespaceFilterCache[namespace]; ok { + return filter + } + + // 2. Check for exact match first (O(1) map lookup) + // This ensures exact namespace matches take precedence over globs, + // regardless of where they are listed in the configuration. + if filter, ok := ctx.namespacedFilterMap[namespace]; ok { + ctx.namespaceFilterCache[namespace] = filter + return filter + } + + // 3. Walk patterns in definition order using pre-compiled globs + // Note: namespaceFilterCache is mutated below without synchronization. This is safe + // today because resource collection runs sequentially. If the restore loop is + // parallelized in the future, these map writes will need a lock to prevent data races. + for _, p := range ctx.namespacedFilterPatterns { + if p.compiled != nil { + if p.compiled.Match(namespace) { + filter := ctx.namespacedFilterMap[p.pattern] + ctx.namespaceFilterCache[namespace] = filter + return filter + } + } + } + + // 4. Cache the miss so we don't re-evaluate failed matches + ctx.namespaceFilterCache[namespace] = nil + return nil +} + +// resolveRestoreClusterScopedFilterPolicy resolves the cluster-scoped filter policy +// into a map keyed by group-resource string. Note: catch-all entries (empty or ["*"] kinds) +// are NOT supported in clusterScopedFilterPolicy — validation rejects them earlier. +// Cluster-scoped filtering is a refinement overlay; unlisted kinds fall back to global +// filters via the existing pipeline, so there is no catchAllFilter field on this map. +func resolveRestoreClusterScopedFilterPolicy( + policy *resourcepolicies.ClusterScopedFilterPolicy, + helper discovery.Helper, + log logrus.FieldLogger, +) (map[string]*resolvedResourceFilter, error) { + result := make(map[string]*resolvedResourceFilter) + for _, rf := range policy.ResourceFilters { + resolved, err := resolveResourceFilter(rf) + if err != nil { + return nil, err + } + for _, kind := range resolved.originalKinds { + gr, resource, err := helper.ResourceFor(schema.ParseGroupResource(kind).WithVersion("")) + + key := kind + if err != nil { + log.WithField("kind", kind).Warnf("Cannot resolve kind via discovery, using as-is") + } else { + if resource.Namespaced { + log.Warnf("kind %q in clusterScopedFilterPolicy is a namespace-scoped resource; it will never match in a cluster-scoped filter — did you mean namespacedFilterPolicies?", kind) + } + key = gr.GroupResource().String() + } + + if _, exists := result[key]; exists { + return nil, fmt.Errorf("ambiguous policy: duplicate kind %q detected", key) + } + + result[key] = resolved + } + } + return result, nil +} + +func resolveRestoreNamespacedFilterPolicies( + policies []resourcepolicies.NamespacedFilterPolicy, + excludedResources []string, + helper discovery.Helper, + log logrus.FieldLogger, +) (map[string]*resolvedNamespaceFilter, []namespacedFilterPattern, error) { + result := make(map[string]*resolvedNamespaceFilter) + var patternOrder []namespacedFilterPattern + + // Build a quick lookup map for globally excluded resources + globalExcludes := make(map[string]bool) + for _, ex := range excludedResources { + // We lowercase the excluded resources here because the kinds in the resource filters + // are lowercased during resolution, and we want to ensure case-insensitive matching. + globalExcludes[strings.ToLower(ex)] = true + } + + for _, policy := range policies { + rfMap := make(map[string]*resolvedResourceFilter) + var catchAll *resolvedResourceFilter + hasUnresolvedKinds := false + + for _, rf := range policy.ResourceFilters { + resolved, err := resolveResourceFilter(rf) + if err != nil { + return nil, nil, err + } + + if rf.IsCatchAll() { + catchAll = resolved + continue + } + + for _, kind := range resolved.originalKinds { + gr, resource, err := helper.ResourceFor( + schema.ParseGroupResource(kind).WithVersion(""), + ) + + key := kind + if err != nil { + log.WithField("kind", kind).Warnf( + "Cannot resolve kind via discovery, using as-is") + hasUnresolvedKinds = true + } else { + if !resource.Namespaced { + log.Warnf("kind %q in namespacedFilterPolicies is a cluster-scoped resource; it will never match in a namespace-scoped filter — did you mean clusterScopedFilterPolicy?", kind) + } + + if globalExcludes[kind] || globalExcludes[gr.GroupResource().String()] { + log.WithFields(logrus.Fields{ + "kind": kind, + "namespacePattern": strings.Join(policy.Namespaces, ","), + }).Warn("namespacedFilterPolicies entry lists a kind that is globally excluded by RestoreSpec.ExcludedResources; the per-namespace filter entry has no effect") + } + key = gr.GroupResource().String() + } + + if _, exists := rfMap[key]; exists { + return nil, nil, fmt.Errorf("ambiguous policy: duplicate kind %q detected", key) + } + + rfMap[key] = resolved + } + } + + nsFilter := &resolvedNamespaceFilter{ + resourceFilterMap: rfMap, + catchAllFilter: catchAll, + hasUnresolvedKinds: hasUnresolvedKinds, + } + for _, nsPattern := range policy.Namespaces { + result[nsPattern] = nsFilter + var compiled glob.Glob + if strings.ContainsAny(nsPattern, "*?[") { + var err error + compiled, err = glob.Compile(nsPattern) + if err != nil { + log.WithError(err).Warnf("Failed to compile namespace glob pattern %q, falling back to exact match", nsPattern) + } + } + patternOrder = append(patternOrder, namespacedFilterPattern{ + pattern: nsPattern, + compiled: compiled, + }) + } + } + return result, patternOrder, nil +} + +// resolveResourceFilter converts a ResourceFilter's label selectors and name patterns +// into their runtime representations. +func resolveResourceFilter( + rf resourcepolicies.ResourceFilter, +) (*resolvedResourceFilter, error) { + selector, err := resourcepolicies.SelectorFromPolicyLabelSelector(rf.LabelSelector) + if err != nil { + return nil, fmt.Errorf("invalid label selector in resource filter: %w", err) + } + var orSelectors []labels.Selector + for _, ols := range rf.OrLabelSelectors { + s, err := resourcepolicies.SelectorFromPolicyLabelSelector(ols) + if err != nil { + return nil, fmt.Errorf("invalid OR label selector in resource filter: %w", err) + } + if s != nil { + orSelectors = append(orSelectors, s) + } + } + var nameIE *collections.IncludesExcludes + if len(rf.Names) > 0 || len(rf.ExcludedNames) > 0 { + nameIE = collections.NewIncludesExcludes().Includes(rf.Names...).Excludes(rf.ExcludedNames...) + } + + normalizedKinds := make([]string, len(rf.Kinds)) + for i, k := range rf.Kinds { + normalizedKinds[i] = strings.ToLower(k) + } + + return &resolvedResourceFilter{ + labelSelector: selector, + orLabelSelectors: orSelectors, + nameIE: nameIE, + originalKinds: normalizedKinds, + }, nil } type resourceClientKey struct { @@ -467,7 +752,7 @@ func (ctx *restoreContext) execute() (results.Result, results.Result) { backupResources, err := archive.NewParser(ctx.log, ctx.fileSystem).Parse(ctx.restoreDir) // If ErrNotExist occurs, it implies that the backup to be restored includes zero items. // Need to add a warning about it and jump out of the function. - if errors.Cause(err) == archive.ErrNotExist { + if errors.Is(err, archive.ErrNotExist) { warnings.AddVeleroError(errors.Wrap(err, "zero items to be restored")) return warnings, errs } @@ -774,7 +1059,7 @@ func (ctx *restoreContext) processSelectedResource( continue } - w, e, _ := ctx.restoreItem(obj, groupResource, targetNS) + w, e, _ := ctx.restoreItem(obj, groupResource, targetNS, false) warnings.Merge(&w) errs.Merge(&e) processedItems++ @@ -1100,7 +1385,7 @@ func (ctx *restoreContext) getResource(groupResource schema.GroupResource, obj * return u, nil } -func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupResource schema.GroupResource, namespace string) (results.Result, results.Result, bool) { +func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupResource schema.GroupResource, namespace string, mustInclude bool) (results.Result, results.Result, bool) { warnings, errs := results.Result{}, results.Result{} // itemExists bool is used to determine whether to include this item in the "wait for additional items" list itemExists := false @@ -1117,23 +1402,41 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso // Check if group/resource should be restored. We need to do this here since // this method may be getting called for an additional item which is a group/resource // that's excluded. - if !ctx.resourceIncludesExcludes.ShouldInclude(groupResource.String()) && !ctx.resourceMustHave.Has(groupResource.String()) { - restoreLogger.Info("Not restoring item because resource is excluded") - return warnings, errs, itemExists - } - - // Check if namespace/cluster-scoped resource should be restored. We need - // to do this here since this method may be getting called for an additional - // item which is in a namespace that's excluded, or which is cluster-scoped - // and should be excluded. Note that we're checking the object's namespace ( - // via obj.GetNamespace()) instead of the namespace parameter, because we want - // to check the *original* namespace, not the remapped one if it's been remapped. - if namespace != "" { - if !ctx.namespaceIncludesExcludes.ShouldInclude(obj.GetNamespace()) && !ctx.resourceMustHave.Has(groupResource.String()) { - restoreLogger.Info("Not restoring item because namespace is excluded") + // + // Note: Additional items intentionally bypass fine-grained resource filter policies + // (like per-namespace label/name selectors) to avoid breaking semantic dependencies, + // but they must still pass the global exclusions enforced below unless mustInclude is set. + if mustInclude { + restoreLogger.Info("Skipping the resource/namespace exclusion checks because the item is marked as must-include") + } else { + if !ctx.resourceIncludesExcludes.ShouldInclude(groupResource.String()) && !ctx.resourceMustHave.Has(groupResource.String()) { + restoreLogger.Info("Not restoring item because resource is excluded") return warnings, errs, itemExists } + // Check if namespace/cluster-scoped resource should be restored. We need + // to do this here since this method may be getting called for an additional + // item which is in a namespace that's excluded, or which is cluster-scoped + // and should be excluded. Note that we're checking the object's namespace ( + // via obj.GetNamespace()) instead of the namespace parameter, because we want + // to check the *original* namespace, not the remapped one if it's been remapped. + if namespace != "" { + if !ctx.namespaceIncludesExcludes.ShouldInclude(obj.GetNamespace()) && !ctx.resourceMustHave.Has(groupResource.String()) { + restoreLogger.Info("Not restoring item because namespace is excluded") + return warnings, errs, itemExists + } + } else { + if boolptr.IsSetToFalse(ctx.restore.Spec.IncludeClusterResources) { + restoreLogger.Info("Not restoring item because it's cluster-scoped") + return warnings, errs, itemExists + } + } + } + + // Namespace creation runs unconditionally when namespace != "", regardless of + // mustInclude. This ensures target namespaces exist for additional items that + // bypass the namespace-exclusion check above. + if namespace != "" { // If the namespace scoped resource should be restored, ensure that the // namespace into which the resource is being restored into exists. // This is the *remapped* namespace that we are ensuring exists. @@ -1152,11 +1455,6 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso } ctx.restoredItems[itemKey] = restoredItemStatus{action: ItemRestoreResultCreated, itemExists: true, createdName: nsToEnsure.Name} } - } else { - if boolptr.IsSetToFalse(ctx.restore.Spec.IncludeClusterResources) { - restoreLogger.Info("Not restoring item because it's cluster-scoped") - return warnings, errs, itemExists - } } // Make a copy of object retrieved from backup to make it available unchanged @@ -1378,6 +1676,21 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso obj = unstructuredObj + mustIncludeAdditionalItems := false + if annotations := obj.GetAnnotations(); annotations != nil { + if _, present := annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation]; present { + // Only the string value "true" enables the bypass. + if annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] == "true" { + mustIncludeAdditionalItems = true + restoreLogger.Info("RestoreItemAction marked additional items as must-include; bypassing resource/namespace exclusion checks for them") + } + // Always strip the annotation so it never lands on the cluster, + // regardless of whether the value enabled the bypass. + delete(annotations, velerov1api.MustIncludeAdditionalItemRestoreAnnotation) + obj.SetAnnotations(annotations) + } + } + var filteredAdditionalItems []velero.ResourceIdentifier for _, additionalItem := range executeOutput.AdditionalItems { itemPath := archive.GetItemFilePath(ctx.restoreDir, additionalItem.GroupResource.String(), additionalItem.Namespace, additionalItem.Name) @@ -1397,6 +1710,7 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso additionalObj, err := archive.Unmarshal(ctx.fileSystem, itemPath) if err != nil { errs.Add(namespace, errors.Wrapf(err, "error restoring additional item %s", additionalResourceID)) + continue } additionalItemNamespace := additionalItem.Namespace @@ -1406,7 +1720,7 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso } } - w, e, additionalItemExists := ctx.restoreItem(additionalObj, additionalItem.GroupResource, additionalItemNamespace) + w, e, additionalItemExists := ctx.restoreItem(additionalObj, additionalItem.GroupResource, additionalItemNamespace, mustIncludeAdditionalItems) if additionalItemExists { filteredAdditionalItems = append(filteredAdditionalItems, additionalItem) } @@ -1736,10 +2050,7 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso return warnings, errs, itemExists } - // Do not create podvolumerestore when current restore excludes pv/pvc - if ctx.resourceIncludesExcludes.ShouldInclude(kuberesource.PersistentVolumeClaims.String()) && - ctx.resourceIncludesExcludes.ShouldInclude(kuberesource.PersistentVolumes.String()) && - len(podvolume.GetVolumeBackupsForPod(ctx.podVolumeBackups, pod, originalNamespace)) > 0 { + if len(podvolume.GetVolumeBackupsForPod(ctx.podVolumeBackups, pod, originalNamespace)) > 0 { restorePodVolumeBackups(ctx, createdObj, originalNamespace) } } @@ -2059,6 +2370,9 @@ func hasPodVolumeBackup(unstructuredPV *unstructured.Unstructured, ctx *restoreC var found bool for _, pvb := range ctx.podVolumeBackups { + if pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseCompleted || pvb.Status.SnapshotID == "" { + continue + } if pvb.Spec.Pod.Namespace == pv.Spec.ClaimRef.Namespace && pvb.GetAnnotations()[configs.PVCNameAnnotation] == pv.Spec.ClaimRef.Name { found = true break @@ -2277,6 +2591,18 @@ func (ctx *restoreContext) getOrderedResourceCollection( continue } + // Per-namespace resource type check from restore filter policy + if namespace != "" && !ctx.resourceMustHave.Has(groupResource.String()) { + if nsFilter := ctx.getNamespaceFilter(namespace); nsFilter != nil { + _, kindListed := nsFilter.resourceFilterMap[groupResource.String()] + if !kindListed && nsFilter.catchAllFilter == nil && !nsFilter.hasUnresolvedKinds { + ctx.log.Infof("Skipping resource %s in namespace %s: not in resourceFilters", + resource, namespace) + continue + } + } + } + res, w, e := ctx.getSelectedRestoreableItems(groupResource.String(), namespace, items) warnings.Merge(&w) errs.Merge(&e) @@ -2330,6 +2656,88 @@ func (ctx *restoreContext) getSelectedRestoreableItems(resource string, original resourceForPath = filepath.Join(resource, cgv.Dir) } + var rf *resolvedResourceFilter + var useFilterPolicy bool + + if !ctx.resourceMustHave.Has(resource) { + if originalNamespace != "" { + // Namespace-scoped path + if nsFilter := ctx.getNamespaceFilter(originalNamespace); nsFilter != nil { + // Resolve effective filter: kind-specific takes precedence over catch-all + rf = nsFilter.resourceFilterMap[resource] + + // Peek-and-map logic for unresolvable kinds + if rf == nil && len(items) > 0 { + peekPath := archive.GetItemFilePath(ctx.restoreDir, resourceForPath, originalNamespace, items[0]) + // Ignore unmarshal errors during peek; the main restore loop will catch and report them + if obj, err := archive.Unmarshal(ctx.fileSystem, peekPath); err == nil { + actualKind := obj.GroupVersionKind().Kind + for _, filter := range nsFilter.resourceFilterMap { + for _, k := range filter.originalKinds { + if strings.EqualFold(k, actualKind) { + rf = filter + // Cache it for future lookups of this resource + // Note: resourceFilterMap is mutated in place without synchronization. + // This is safe today because resource collection runs sequentially. + // If parallelized in the future, this will need a lock to prevent data races. + nsFilter.resourceFilterMap[resource] = rf + break + } + } + if rf != nil { + break + } + } + } + } + + if rf == nil { + rf = nsFilter.catchAllFilter // may be nil if no catch-all + } + useFilterPolicy = true + + if rf == nil { + ctx.log.Infof("Skipping resource %s in namespace %s: not in resourceFilters", resource, originalNamespace) + return restorable, warnings, errs + } + } + } else if ctx.clusterScopedFilterMap != nil { + // Cluster-scoped path: only applies if kind is listed (refinement overlay) + if listedRF, ok := ctx.clusterScopedFilterMap[resource]; ok { + rf = listedRF + useFilterPolicy = true + } else if len(items) > 0 { + // Peek-and-map logic for unresolvable kinds. + // Note: Unlike the namespaced path, this fallback is always reachable + // because the main restore loop does not have a fast-path skip for + // unlisted cluster-scoped resources. + peekPath := archive.GetItemFilePath(ctx.restoreDir, resourceForPath, originalNamespace, items[0]) + // Ignore unmarshal errors during peek; the main restore loop will catch and report them + if obj, err := archive.Unmarshal(ctx.fileSystem, peekPath); err == nil { + actualKind := obj.GroupVersionKind().Kind + for _, filter := range ctx.clusterScopedFilterMap { + for _, k := range filter.originalKinds { + if strings.EqualFold(k, actualKind) { + rf = filter + // Cache it for future lookups of this resource + // Note: clusterScopedFilterMap is mutated in place without synchronization. + // This is safe today because resource collection runs sequentially. + // If parallelized in the future, this will need a lock to prevent data races. + ctx.clusterScopedFilterMap[resource] = rf + useFilterPolicy = true + break + } + } + if rf != nil { + break + } + } + } + } + // If kind not listed, fall through to global selectors below + } + } + for _, item := range items { itemPath := archive.GetItemFilePath(ctx.restoreDir, resourceForPath, originalNamespace, item) @@ -2347,29 +2755,58 @@ func (ctx *restoreContext) getSelectedRestoreableItems(resource string, original } if !ctx.resourceMustHave.Has(resource) { - if !ctx.selector.Matches(labels.Set(obj.GetLabels())) { - continue - } - - // Processing OrLabelSelectors when specified in the restore request. LabelSelectors as well as OrLabelSelectors - // cannot co-exist, only one of them can be specified - var skipItem = false - var skip = 0 - ctx.log.Debugf("orSelectors specified: %s for item: %s", ctx.OrSelectors, item) - for _, s := range ctx.OrSelectors { - if !s.Matches(labels.Set(obj.GetLabels())) { - skip++ + if useFilterPolicy { + if rf != nil { + // Per-kind label selector + if rf.labelSelector != nil && !rf.labelSelector.Matches(labels.Set(obj.GetLabels())) { + continue + } + // Per-kind OR label selectors + if len(rf.orLabelSelectors) > 0 { + matched := false + for _, s := range rf.orLabelSelectors { + if s.Matches(labels.Set(obj.GetLabels())) { + matched = true + break + } + } + if !matched { + ctx.log.Infof("Excluding item %s: no OR label selector matched (restore filter policy)", item) + continue + } + } + // Per-kind name filter + if rf.nameIE != nil && !rf.nameIE.ShouldInclude(obj.GetName()) { + ctx.log.Infof("Excluding item %s: name does not match restore filter policy", obj.GetName()) + continue + } + } + } else { + // Existing global selector logic + if !ctx.selector.Matches(labels.Set(obj.GetLabels())) { + continue } - if len(ctx.OrSelectors) == skip && skip > 0 { - ctx.log.Infof("setting skip flag to true for item: %s", item) - skipItem = true + // Processing OrLabelSelectors when specified in the restore request. LabelSelectors as well as OrLabelSelectors + // cannot co-exist, only one of them can be specified + var skipItem = false + var skip = 0 + ctx.log.Debugf("orSelectors specified: %s for item: %s", ctx.OrSelectors, item) + for _, s := range ctx.OrSelectors { + if !s.Matches(labels.Set(obj.GetLabels())) { + skip++ + } + + if len(ctx.OrSelectors) == skip && skip > 0 { + ctx.log.Infof("setting skip flag to true for item: %s", item) + skipItem = true + } } - } - if skipItem { - ctx.log.Infof("restore orSelector labels did not match, skipping restore of item: %s", skipItem, item) - continue + if skipItem { + ctx.log.Infof("restore orSelector labels did not match, skipping restore of item: %s", item) + continue + } } } diff --git a/pkg/restore/restore_policies_test.go b/pkg/restore/restore_policies_test.go new file mode 100644 index 0000000000..569d8923db --- /dev/null +++ b/pkg/restore/restore_policies_test.go @@ -0,0 +1,369 @@ +package restore + +import ( + "io" + "strings" + "testing" + + "github.com/sirupsen/logrus" + logrustest "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/require" + corev1api "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/vmware-tanzu/velero/internal/resourcepolicies" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" + "github.com/vmware-tanzu/velero/pkg/test" +) + +func TestRestoreResourcePoliciesFiltering(t *testing.T) { + customKindRes := &test.APIResource{Group: "mygroup.io", Version: "v1", Name: "mycustomkinds", Kind: "MyCustomKind", Namespaced: true} + clusterCustomKindRes := &test.APIResource{Group: "mygroup.io", Version: "v1", Name: "myclustercustomkinds", Kind: "MyClusterCustomKind", Namespaced: false} + + tests := []struct { + name string + restore *velerov1api.Restore + backup *velerov1api.Backup + policyYAML string + apiResources []*test.APIResource + tarball io.Reader + want map[*test.APIResource][]string + }{ + { + name: "namespaced filter policy with exact namespace match", + restore: defaultRestore().Result(), + backup: defaultBackup().Result(), + policyYAML: `version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-1 + resourceFilters: + - kinds: + - pods + names: + - pod-1 +`, + tarball: test.NewTarWriter(t). + AddItems("pods", + builder.ForPod("ns-1", "pod-1").Result(), + builder.ForPod("ns-1", "pod-2").Result(), + builder.ForPod("ns-2", "pod-1").Result(), + ). + Done(), + apiResources: []*test.APIResource{ + test.Pods(), + }, + want: map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1", "ns-2/pod-1"}, // ns-2 is not filtered, ns-1 only includes pod-1 + }, + }, + { + name: "namespaced filter policy with exact match priority over glob (glob listed first)", + restore: defaultRestore().Result(), + backup: defaultBackup().Result(), + policyYAML: `version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-* + resourceFilters: + - kinds: + - pods + names: + - pod-1 + - namespaces: + - ns-1 + resourceFilters: + - kinds: + - pods + names: + - pod-2 +`, + tarball: test.NewTarWriter(t). + AddItems("pods", + builder.ForPod("ns-1", "pod-1").Result(), + builder.ForPod("ns-1", "pod-2").Result(), + builder.ForPod("ns-2", "pod-1").Result(), + builder.ForPod("ns-2", "pod-2").Result(), + ). + Done(), + apiResources: []*test.APIResource{ + test.Pods(), + }, + want: map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-2", "ns-2/pod-1"}, + }, + }, + { + name: "namespaced filter policy with exact match priority over glob (exact listed first)", + restore: defaultRestore().Result(), + backup: defaultBackup().Result(), + policyYAML: `version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-1 + resourceFilters: + - kinds: + - pods + names: + - pod-2 + - namespaces: + - ns-* + resourceFilters: + - kinds: + - pods + names: + - pod-1 +`, + tarball: test.NewTarWriter(t). + AddItems("pods", + builder.ForPod("ns-1", "pod-1").Result(), + builder.ForPod("ns-1", "pod-2").Result(), + builder.ForPod("ns-2", "pod-1").Result(), + builder.ForPod("ns-2", "pod-2").Result(), + ). + Done(), + apiResources: []*test.APIResource{ + test.Pods(), + }, + want: map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-2", "ns-2/pod-1"}, + }, + }, + { + name: "cluster scoped filter policy", + restore: defaultRestore().Result(), + backup: defaultBackup().Result(), + policyYAML: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: + - persistentvolumes + names: + - pv-1 +`, + tarball: test.NewTarWriter(t). + AddItems("persistentvolumes", + builder.ForPersistentVolume("pv-1").Result(), + builder.ForPersistentVolume("pv-2").Result(), + ). + Done(), + apiResources: []*test.APIResource{ + test.PVs(), + }, + want: map[*test.APIResource][]string{ + test.PVs(): {"/pv-1"}, + }, + }, + { + name: "catch-all filter", + restore: defaultRestore().Result(), + backup: defaultBackup().Result(), + policyYAML: `version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-1 + resourceFilters: + - kinds: + - '*' + labelSelector: + matchLabels: + app: test +`, + tarball: test.NewTarWriter(t). + AddItems("pods", + builder.ForPod("ns-1", "pod-1").ObjectMeta(builder.WithLabels("app", "test")).Result(), + builder.ForPod("ns-1", "pod-2").Result(), + ). + AddItems("deployments.apps", + builder.ForDeployment("ns-1", "deploy-1").ObjectMeta(builder.WithLabels("app", "test")).Result(), + builder.ForDeployment("ns-1", "deploy-2").Result(), + ). + Done(), + apiResources: []*test.APIResource{ + test.Pods(), + test.Deployments(), + }, + want: map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1"}, + test.Deployments(): {"ns-1/deploy-1"}, + }, + }, + { + name: "unresolved kind in namespaced filter policy is still restored via peek-and-map", + restore: defaultRestore().Result(), + backup: defaultBackup().Result(), + policyYAML: `version: v1 +namespacedFilterPolicies: + - namespaces: ["ns-1"] + resourceFilters: + - kinds: ["MyCustomKind"] +`, + tarball: test.NewTarWriter(t).AddItems("mycustomkinds.mygroup.io", + &unstructured.Unstructured{Object: map[string]any{"apiVersion": "mygroup.io/v1", "kind": "MyCustomKind", "metadata": map[string]any{"namespace": "ns-1", "name": "my-cr"}}}, + ).Done(), + apiResources: []*test.APIResource{ + customKindRes, + }, + want: map[*test.APIResource][]string{ + customKindRes: {"ns-1/my-cr"}, + }, + }, + { + name: "unresolved kind in cluster-scoped filter policy is still restored via peek-and-map", + restore: defaultRestore().Result(), + backup: defaultBackup().Result(), + policyYAML: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["MyClusterCustomKind"] +`, + tarball: test.NewTarWriter(t).AddItems("myclustercustomkinds.mygroup.io", + &unstructured.Unstructured{Object: map[string]any{"apiVersion": "mygroup.io/v1", "kind": "MyClusterCustomKind", "metadata": map[string]any{"name": "my-cluster-cr"}}}, + ).Done(), + apiResources: []*test.APIResource{ + clusterCustomKindRes, + }, + want: map[*test.APIResource][]string{ + clusterCustomKindRes: {"/my-cluster-cr"}, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t) + + for _, r := range tc.apiResources { + h.DiscoveryClient.WithAPIResource(r) + } + require.NoError(t, h.restorer.discoveryHelper.Refresh()) + + var resPolicies *resourcepolicies.Policies + if tc.policyYAML != "" { + cm := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-policies", + Namespace: "velero", + }, + Data: map[string]string{ + "policy.yaml": tc.policyYAML, + }, + } + client := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(cm).Build() + restore := tc.restore.DeepCopy() + restore.Namespace = "velero" + restore.Spec.ResourcePolicy = &corev1api.TypedLocalObjectReference{ + Kind: "configmap", + Name: "test-policies", + } + var err error + resPolicies, err = resourcepolicies.GetResourcePoliciesFromRestore(t.Context(), restore, client, logrus.New()) + require.NoError(t, err) + } + + data := &Request{ + Log: h.log, + Restore: tc.restore, + Backup: tc.backup, + PodVolumeBackups: nil, + VolumeSnapshots: nil, + BackupReader: tc.tarball, + ResPolicies: resPolicies, + } + warnings, errs := h.restorer.Restore( + data, + nil, // restoreItemActions + nil, // volume snapshotter getter + ) + + assertEmptyResults(t, warnings, errs) + assertAPIContents(t, h, tc.want) + }) + } +} + +func TestResolveRestoreNamespacedFilterPolicies_Validation(t *testing.T) { + log := logrus.New() + helper := test.NewFakeDiscoveryHelper(true, nil) + + policies := []resourcepolicies.NamespacedFilterPolicy{ + { + Namespaces: []string{"ns-1"}, + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"MyKind", "mykind"}, + }, + }, + }, + } + + _, _, err := resolveRestoreNamespacedFilterPolicies(policies, nil, helper, log) + require.Error(t, err) + require.Contains(t, err.Error(), "ambiguous policy: duplicate kind") +} + +func TestResolveRestoreClusterScopedFilterPolicy_Validation(t *testing.T) { + log := logrus.New() + helper := test.NewFakeDiscoveryHelper(true, nil) + + policy := &resourcepolicies.ClusterScopedFilterPolicy{ + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"MyKind", "mykind"}, + }, + }, + } + + _, err := resolveRestoreClusterScopedFilterPolicy(policy, helper, log) + require.Error(t, err) + require.Contains(t, err.Error(), "ambiguous policy: duplicate kind") +} + +func TestResolveRestoreNamespacedFilterPolicies_GlobalExcludesWarning(t *testing.T) { + log, hook := logrustest.NewNullLogger() + helper := test.NewFakeDiscoveryHelper(true, nil) + + policies := []resourcepolicies.NamespacedFilterPolicy{ + { + Namespaces: []string{"ns-1"}, + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"ConfigMaps"}, + }, + }, + }, + } + + excludedResources := []string{"ConfigMaps"} // Same case + _, _, err := resolveRestoreNamespacedFilterPolicies(policies, excludedResources, helper, log) + require.NoError(t, err) + + // Check if a warning was emitted + found := false + for _, entry := range hook.Entries { + if entry.Level == logrus.WarnLevel && strings.Contains(entry.Message, "namespacedFilterPolicies entry lists a kind that is globally excluded") { + found = true + break + } + } + require.True(t, found, "expected warning about globally excluded resource") + + hook.Reset() + + excludedResourcesDiffCase := []string{"configmaps"} // Different case + _, _, err = resolveRestoreNamespacedFilterPolicies(policies, excludedResourcesDiffCase, helper, log) + require.NoError(t, err) + + found = false + for _, entry := range hook.Entries { + if entry.Level == logrus.WarnLevel && strings.Contains(entry.Message, "namespacedFilterPolicies entry lists a kind that is globally excluded") { + found = true + break + } + } + require.True(t, found, "expected warning about globally excluded resource even if case differs") +} diff --git a/pkg/restore/restore_test.go b/pkg/restore/restore_test.go index 95b283cbe0..935586e639 100644 --- a/pkg/restore/restore_test.go +++ b/pkg/restore/restore_test.go @@ -25,11 +25,12 @@ import ( "testing" "time" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/collections" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -753,6 +754,29 @@ func TestRestoreResourceFiltering(t *testing.T) { apiResources: []*test.APIResource{test.ServiceAccounts()}, want: map[*test.APIResource][]string{test.ServiceAccounts(): {"ns-1/sa-1"}}, }, + { + // Regression for #9957: VSC must not be force-included via resourceMustHave + // when the restore only selects unrelated resource types. + name: "volumesnapshotcontents are not force-included for selective resource restores", + restore: defaultRestore().IncludedResources("storageclasses").IncludeClusterResources(true).Result(), + backup: defaultBackup().Result(), + tarball: test.NewTarWriter(t). + AddItems("storageclasses.storage.k8s.io", + builder.ForStorageClass("sc-1").Result(), + ). + AddItems("volumesnapshotcontents.snapshot.storage.k8s.io", + builder.ForVolumeSnapshotContent("vsc-1").Result(), + ). + Done(), + apiResources: []*test.APIResource{ + test.StorageClasses(), + test.VolumeSnapshotContents(), + }, + want: map[*test.APIResource][]string{ + test.StorageClasses(): {"/sc-1"}, + test.VolumeSnapshotContents(): nil, + }, + }, } for _, tc := range tests { @@ -764,6 +788,10 @@ func TestRestoreResourceFiltering(t *testing.T) { } require.NoError(t, h.restorer.discoveryHelper.Refresh()) + // We need to fetch the policies using the actual function + resPolicies, err := resourcepolicies.GetResourcePoliciesFromRestore(t.Context(), tc.restore, h.restorer.kbClient, h.log) + require.NoError(t, err) + data := &Request{ Log: h.log, Restore: tc.restore, @@ -771,6 +799,7 @@ func TestRestoreResourceFiltering(t *testing.T) { PodVolumeBackups: nil, VolumeSnapshots: nil, BackupReader: tc.tarball, + ResPolicies: resPolicies, } warnings, errs := h.restorer.Restore( data, @@ -2144,6 +2173,102 @@ func TestRestoreActionAdditionalItems(t *testing.T) { test.PVs(): nil, }, }, + { + name: "must-include annotation bypasses resource exclusion for additional items", + restore: defaultRestore().IncludedResources("pods").Result(), + backup: defaultBackup().Result(), + tarball: test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). + AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()). + Done(), + apiResources: []*test.APIResource{test.Pods(), test.PVs()}, + actions: []riav2.RestoreItemAction{ + &pluggableAction{ + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, + }, + }, nil + }, + }, + }, + want: map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1"}, + test.PVs(): {"/pv-1"}, + }, + }, + { + name: "must-include annotation bypasses namespace exclusion for additional items", + restore: defaultRestore().IncludedNamespaces("ns-1").Result(), + backup: defaultBackup().Result(), + tarball: test.NewTarWriter(t).AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result()).Done(), + apiResources: []*test.APIResource{test.Pods()}, + actions: []riav2.RestoreItemAction{ + &pluggableAction{ + selector: velero.ResourceSelector{IncludedNamespaces: []string{"ns-1"}}, + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.Pods, Namespace: "ns-2", Name: "pod-2"}, + }, + }, nil + }, + }, + }, + want: map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1", "ns-2/pod-2"}, + }, + }, + { + name: "must-include annotation bypasses IncludeClusterResources=false for additional items", + restore: defaultRestore().IncludeClusterResources(false).Result(), + backup: defaultBackup().Result(), + tarball: test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). + AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()). + Done(), + apiResources: []*test.APIResource{test.Pods(), test.PVs()}, + actions: []riav2.RestoreItemAction{ + &pluggableAction{ + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, + }, + }, nil + }, + }, + }, + want: map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1"}, + test.PVs(): {"/pv-1"}, + }, + }, } for _, tc := range tests { @@ -2174,6 +2299,370 @@ func TestRestoreActionAdditionalItems(t *testing.T) { } } +// TestRestoreMustIncludeAdditionalItems covers restore must-include edge cases beyond the +// basic filter-bypass cases in TestRestoreActionAdditionalItems. +func TestRestoreMustIncludeAdditionalItems(t *testing.T) { + t.Run("must-include annotation is stripped from the restored item", func(t *testing.T) { + h := newHarness(t) + h.AddItems(t, test.Pods()) + + data := &Request{ + Log: h.log, + Restore: defaultRestore().Result(), + Backup: defaultBackup().Result(), + BackupReader: test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). + Done(), + } + warnings, errs := h.restorer.Restore( + data, + []riav2.RestoreItemAction{ + &pluggableAction{ + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + annotations["keep-me"] = "yes" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{UpdatedItem: item}, nil + }, + }, + }, + nil, + ) + + assertEmptyResults(t, warnings, errs) + + got, err := h.DynamicClient.Resource(test.Pods().GVR()).Namespace("ns-1").Get(t.Context(), "pod-1", metav1.GetOptions{}) + require.NoError(t, err) + annotations := got.GetAnnotations() + assert.NotContains(t, annotations, velerov1api.MustIncludeAdditionalItemRestoreAnnotation) + assert.Equal(t, "yes", annotations["keep-me"]) + }) + + t.Run("non-true must-include annotation is stripped without bypassing filters", func(t *testing.T) { + h := newHarness(t) + h.AddItems(t, test.Pods()) + h.AddItems(t, test.PVs()) + + data := &Request{ + Log: h.log, + Restore: defaultRestore().IncludedResources("pods").Result(), + Backup: defaultBackup().Result(), + BackupReader: test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). + AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()). + Done(), + } + warnings, errs := h.restorer.Restore( + data, + []riav2.RestoreItemAction{ + &pluggableAction{ + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "True" + annotations["keep-me"] = "yes" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, + }, + }, nil + }, + }, + }, + nil, + ) + + assertEmptyResults(t, warnings, errs) + assertAPIContents(t, h, map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1"}, + test.PVs(): nil, + }) + + got, err := h.DynamicClient.Resource(test.Pods().GVR()).Namespace("ns-1").Get(t.Context(), "pod-1", metav1.GetOptions{}) + require.NoError(t, err) + annotations := got.GetAnnotations() + assert.NotContains(t, annotations, velerov1api.MustIncludeAdditionalItemRestoreAnnotation) + assert.Equal(t, "yes", annotations["keep-me"]) + }) + + t.Run("SkipRestore supersedes must-include annotation and skips additional items", func(t *testing.T) { + h := newHarness(t) + h.AddItems(t, test.Pods()) + h.AddItems(t, test.PVs()) + + data := &Request{ + Log: h.log, + Restore: defaultRestore().IncludedResources("pods").Result(), + Backup: defaultBackup().Result(), + BackupReader: test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). + AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()). + Done(), + } + warnings, errs := h.restorer.Restore( + data, + []riav2.RestoreItemAction{ + &pluggableAction{ + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + SkipRestore: true, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, + }, + }, nil + }, + }, + }, + nil, + ) + + assertEmptyResults(t, warnings, errs) + assertAPIContents(t, h, map[*test.APIResource][]string{ + test.Pods(): nil, + test.PVs(): nil, + }) + }) + + t.Run("must-include does not restore additional items missing from the backup tarball", func(t *testing.T) { + h := newHarness(t) + h.AddItems(t, test.Pods()) + h.AddItems(t, test.PVs()) + + data := &Request{ + Log: h.log, + Restore: defaultRestore().IncludedResources("pods").Result(), + Backup: defaultBackup().Result(), + BackupReader: test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). + Done(), + } + warnings, errs := h.restorer.Restore( + data, + []riav2.RestoreItemAction{ + &pluggableAction{ + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumes, Name: "pv-missing"}, + }, + }, nil + }, + }, + }, + nil, + ) + + assertEmptyResults(t, errs) + assertNonEmptyResults(t, "warning", warnings) + assertAPIContents(t, h, map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1"}, + test.PVs(): nil, + }) + }) + + t.Run("transitive must-include requires each RIA level to re-set the annotation", func(t *testing.T) { + h := newHarness(t) + h.AddItems(t, test.Pods()) + h.AddItems(t, test.PVs()) + h.AddItems(t, test.PVCs()) + + data := &Request{ + Log: h.log, + Restore: defaultRestore().IncludedResources("pods").Result(), + Backup: defaultBackup().Result(), + BackupReader: test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). + AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()). + AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-2", "pvc-1").Result()). + Done(), + } + warnings, errs := h.restorer.Restore( + data, + []riav2.RestoreItemAction{ + // Parent pod RIA force-includes the excluded PV. + &pluggableAction{ + selector: velero.ResourceSelector{IncludedResources: []string{"pods"}}, + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, + }, + }, nil + }, + }, + // Child PV RIA also re-sets the annotation to force-include an excluded PVC. + &pluggableAction{ + selector: velero.ResourceSelector{IncludedResources: []string{"persistentvolumes"}}, + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumeClaims, Namespace: "ns-2", Name: "pvc-1"}, + }, + }, nil + }, + }, + }, + nil, + ) + + assertEmptyResults(t, warnings, errs) + assertAPIContents(t, h, map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1"}, + test.PVs(): {"/pv-1"}, + test.PVCs(): {"ns-2/pvc-1"}, + }) + }) + + t.Run("without re-annotating, transitive additional items still respect filters", func(t *testing.T) { + h := newHarness(t) + h.AddItems(t, test.Pods()) + h.AddItems(t, test.PVs()) + h.AddItems(t, test.PVCs()) + + data := &Request{ + Log: h.log, + Restore: defaultRestore().IncludedResources("pods").Result(), + Backup: defaultBackup().Result(), + BackupReader: test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). + AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()). + AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-2", "pvc-1").Result()). + Done(), + } + warnings, errs := h.restorer.Restore( + data, + []riav2.RestoreItemAction{ + &pluggableAction{ + selector: velero.ResourceSelector{IncludedResources: []string{"pods"}}, + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, + }, + }, nil + }, + }, + // Child PV RIA returns an additional PVC but does NOT set must-include. + &pluggableAction{ + selector: velero.ResourceSelector{IncludedResources: []string{"persistentvolumes"}}, + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: input.Item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumeClaims, Namespace: "ns-2", Name: "pvc-1"}, + }, + }, nil + }, + }, + }, + nil, + ) + + assertEmptyResults(t, warnings, errs) + assertAPIContents(t, h, map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1"}, + test.PVs(): {"/pv-1"}, + test.PVCs(): nil, + }) + }) + + t.Run("VS must-include restores excluded VolumeSnapshotContent additional item", func(t *testing.T) { + h := newHarness(t) + h.AddItems(t, test.VolumeSnapshots()) + h.AddItems(t, test.VolumeSnapshotContents()) + + data := &Request{ + Log: h.log, + Restore: defaultRestore().IncludedResources("volumesnapshots.snapshot.storage.k8s.io").IncludeClusterResources(true).Result(), + Backup: defaultBackup().Result(), + BackupReader: test.NewTarWriter(t). + AddItems("volumesnapshots.snapshot.storage.k8s.io", builder.ForVolumeSnapshot("ns-1", "vs-1").Result()). + AddItems("volumesnapshotcontents.snapshot.storage.k8s.io", builder.ForVolumeSnapshotContent("vsc-1").Result()). + Done(), + } + warnings, errs := h.restorer.Restore( + data, + []riav2.RestoreItemAction{ + &pluggableAction{ + selector: velero.ResourceSelector{IncludedResources: []string{"volumesnapshots.snapshot.storage.k8s.io"}}, + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.VolumeSnapshotContents, Name: "vsc-1"}, + }, + }, nil + }, + }, + }, + nil, + ) + + assertEmptyResults(t, warnings, errs) + assertAPIContents(t, h, map[*test.APIResource][]string{ + test.VolumeSnapshots(): {"ns-1/vs-1"}, + test.VolumeSnapshotContents(): {"/vsc-1"}, + }) + }) +} + // TestShouldRestore runs the ShouldRestore function for various permutations of // existing/nonexisting/being-deleted PVs, PVCs, and namespaces, and verifies the // result/error matches expectations. @@ -4240,3 +4729,84 @@ func TestDetermineRestoreStatus(t *testing.T) { }) } } + +func TestHasPodVolumeBackup(t *testing.T) { + pvUnstructured := func() *unstructured.Unstructured { + pv := &corev1api.PersistentVolume{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "PersistentVolume"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pv", + }, + Spec: corev1api.PersistentVolumeSpec{ + ClaimRef: &corev1api.ObjectReference{ + Namespace: "test-ns", + Name: "test-pvc", + }, + }, + } + obj, _ := runtime.DefaultUnstructuredConverter.ToUnstructured(pv) + return &unstructured.Unstructured{Object: obj} + } + + makePVB := func(phase velerov1api.PodVolumeBackupPhase, snapshotID string) *velerov1api.PodVolumeBackup { + return &velerov1api.PodVolumeBackup{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "velero.io/pvc-name": "test-pvc", + }, + }, + Spec: velerov1api.PodVolumeBackupSpec{ + Pod: corev1api.ObjectReference{ + Namespace: "test-ns", + }, + }, + Status: velerov1api.PodVolumeBackupStatus{ + Phase: phase, + SnapshotID: snapshotID, + }, + } + } + + tests := []struct { + name string + pvbs []*velerov1api.PodVolumeBackup + expected bool + }{ + { + name: "no pvbs", + pvbs: nil, + expected: false, + }, + { + name: "completed pvb with snapshot ID", + pvbs: []*velerov1api.PodVolumeBackup{makePVB(velerov1api.PodVolumeBackupPhaseCompleted, "snap-123")}, + expected: true, + }, + { + name: "in-progress pvb should not match", + pvbs: []*velerov1api.PodVolumeBackup{makePVB(velerov1api.PodVolumeBackupPhaseInProgress, "")}, + expected: false, + }, + { + name: "completed pvb with empty snapshot ID should not match", + pvbs: []*velerov1api.PodVolumeBackup{makePVB(velerov1api.PodVolumeBackupPhaseCompleted, "")}, + expected: false, + }, + { + name: "failed pvb should not match", + pvbs: []*velerov1api.PodVolumeBackup{makePVB(velerov1api.PodVolumeBackupPhaseFailed, "")}, + expected: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := &restoreContext{ + podVolumeBackups: tc.pvbs, + log: logrus.New(), + } + result := hasPodVolumeBackup(pvUnstructured(), ctx) + assert.Equal(t, tc.expected, result) + }) + } +} diff --git a/pkg/test/api_server.go b/pkg/test/api_server.go index dd5b0a07a6..c69dc5926a 100644 --- a/pkg/test/api_server.go +++ b/pkg/test/api_server.go @@ -56,6 +56,11 @@ func NewAPIServer(t *testing.T) *APIServer { {Group: "extensions", Version: "v1", Resource: "deployments"}: "ExtDeploymentsList", {Group: "velero.io", Version: "v1", Resource: "deployments"}: "VeleroDeploymentsList", {Group: "velero.io", Version: "v2alpha1", Resource: "datauploads"}: "DataUploadsList", + {Group: "mygroup.io", Version: "v1", Resource: "mycustomkinds"}: "MyCustomKindList", + {Group: "mygroup.io", Version: "v1", Resource: "myclustercustomkinds"}: "MyClusterCustomKindList", + {Group: "storage.k8s.io", Version: "v1", Resource: "storageclasses"}: "StorageClassList", + {Group: "snapshot.storage.k8s.io", Version: "v1", Resource: "volumesnapshots"}: "VolumeSnapshotList", + {Group: "snapshot.storage.k8s.io", Version: "v1", Resource: "volumesnapshotcontents"}: "VolumeSnapshotContentList", }) discoveryClient = &DiscoveryClient{FakeDiscovery: kubeClient.Discovery().(*discoveryfake.FakeDiscovery)} ) diff --git a/pkg/test/fake_mapper.go b/pkg/test/fake_mapper.go index 1686af8156..529d989d53 100644 --- a/pkg/test/fake_mapper.go +++ b/pkg/test/fake_mapper.go @@ -17,7 +17,7 @@ limitations under the License. package test import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/test/resources.go b/pkg/test/resources.go index fe2ad6352b..975359d47c 100644 --- a/pkg/test/resources.go +++ b/pkg/test/resources.go @@ -220,3 +220,37 @@ func DataUploads(items ...metav1.Object) *APIResource { Items: items, } } + +func StorageClasses(items ...metav1.Object) *APIResource { + return &APIResource{ + Group: "storage.k8s.io", + Version: "v1", + Name: "storageclasses", + ShortName: "sc", + Kind: "StorageClass", + Namespaced: false, + Items: items, + } +} + +func VolumeSnapshotContents(items ...metav1.Object) *APIResource { + return &APIResource{ + Group: "snapshot.storage.k8s.io", + Version: "v1", + Name: "volumesnapshotcontents", + Kind: "VolumeSnapshotContent", + Namespaced: false, + Items: items, + } +} + +func VolumeSnapshots(items ...metav1.Object) *APIResource { + return &APIResource{ + Group: "snapshot.storage.k8s.io", + Version: "v1", + Name: "volumesnapshots", + Kind: "VolumeSnapshot", + Namespaced: true, + Items: items, + } +} diff --git a/pkg/types/node_agent.go b/pkg/types/node_agent.go index f456bbf55c..694a629146 100644 --- a/pkg/types/node_agent.go +++ b/pkg/types/node_agent.go @@ -59,11 +59,36 @@ type BackupPVC struct { // Annotations permits setting annotations for the backupPVC Annotations map[string]string `json:"annotations,omitempty"` + + // SecretNames is a list of secret names to copy from the source PVC namespace + // to the Velero namespace before creating the backupPVC. The secrets are deleted + // after the DataUpload completes. This is needed for CSI drivers that require + // namespace-scoped secrets for volume provisioning (e.g., encrypted volumes). + SecretNames []string `json:"secretNames,omitempty"` + + // ConfigMapNames is a list of configmap names to copy from the source PVC namespace + // to the Velero namespace before creating the backupPVC. The configmaps are deleted + // after the DataUpload completes. This is needed for CSI drivers that require + // namespace-scoped configmaps for volume provisioning (e.g., tenant-specific + // Vault connection overrides for encrypted volumes). + ConfigMapNames []string `json:"configMapNames,omitempty"` } type RestorePVC struct { // IgnoreDelayBinding indicates to ignore delay binding the restorePVC when it is in WaitForFirstConsumer mode IgnoreDelayBinding bool `json:"ignoreDelayBinding,omitempty"` + + // SecretNames is a list of secret names to copy from the target namespace to the + // Velero namespace before creating the restorePVC. The secrets are deleted after the + // DataDownload completes. This is needed for CSI drivers that require namespace-scoped + // secrets for volume provisioning (e.g., encrypted volumes). + SecretNames []string `json:"secretNames,omitempty"` + + // ConfigMapNames is a list of configmap names to copy from the target namespace to the + // Velero namespace before creating the restorePVC. The configmaps are deleted after the + // DataDownload completes. This is needed for CSI drivers that require namespace-scoped + // configmaps for volume provisioning (e.g., tenant-specific Vault connection overrides). + ConfigMapNames []string `json:"configMapNames,omitempty"` } type CachePVC struct { diff --git a/pkg/uploader/kopia/block_backup.go b/pkg/uploader/kopia/block_backup.go index ad90b723fc..eb34358567 100644 --- a/pkg/uploader/kopia/block_backup.go +++ b/pkg/uploader/kopia/block_backup.go @@ -23,9 +23,9 @@ import ( "os" "syscall" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/fs/virtualfs" - "github.com/pkg/errors" ) const ErrNotPermitted = "operation not permitted" diff --git a/pkg/uploader/kopia/block_restore.go b/pkg/uploader/kopia/block_restore.go index 4f28a59de4..33f1b72e03 100644 --- a/pkg/uploader/kopia/block_restore.go +++ b/pkg/uploader/kopia/block_restore.go @@ -26,9 +26,9 @@ import ( "path/filepath" "syscall" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/snapshot/restore" - "github.com/pkg/errors" ) type BlockOutput struct { diff --git a/pkg/uploader/kopia/flush_volume_linux.go b/pkg/uploader/kopia/flush_volume_linux.go index 98234e1b91..d73091a24a 100644 --- a/pkg/uploader/kopia/flush_volume_linux.go +++ b/pkg/uploader/kopia/flush_volume_linux.go @@ -22,7 +22,7 @@ package kopia import ( "os" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "golang.org/x/sys/unix" ) diff --git a/pkg/uploader/kopia/progress_test.go b/pkg/uploader/kopia/progress_test.go index 8c18bb85bd..065d6e3956 100644 --- a/pkg/uploader/kopia/progress_test.go +++ b/pkg/uploader/kopia/progress_test.go @@ -20,7 +20,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/uploader" diff --git a/pkg/uploader/kopia/restore_output.go b/pkg/uploader/kopia/restore_output.go index 74311d38ab..986530d52c 100644 --- a/pkg/uploader/kopia/restore_output.go +++ b/pkg/uploader/kopia/restore_output.go @@ -17,8 +17,8 @@ limitations under the License. package kopia import ( + "github.com/cockroachdb/errors" "github.com/kopia/kopia/snapshot/restore" - "github.com/pkg/errors" ) var errFlushUnsupported = errors.New("flush is not supported") diff --git a/pkg/uploader/kopia/shim.go b/pkg/uploader/kopia/shim.go index 1b9812d487..482a8af7a3 100644 --- a/pkg/uploader/kopia/shim.go +++ b/pkg/uploader/kopia/shim.go @@ -21,7 +21,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" diff --git a/pkg/uploader/kopia/snapshot.go b/pkg/uploader/kopia/snapshot.go index 1924ed35bc..217ff531fd 100644 --- a/pkg/uploader/kopia/snapshot.go +++ b/pkg/uploader/kopia/snapshot.go @@ -28,6 +28,7 @@ import ( "github.com/sirupsen/logrus" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/fs/localfs" "github.com/kopia/kopia/repo" @@ -36,7 +37,6 @@ import ( "github.com/kopia/kopia/snapshot/policy" "github.com/kopia/kopia/snapshot/restore" "github.com/kopia/kopia/snapshot/snapshotfs" - "github.com/pkg/errors" "github.com/vmware-tanzu/velero/pkg/kopia" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" diff --git a/pkg/uploader/kopia/snapshot_test.go b/pkg/uploader/kopia/snapshot_test.go index 984b92af57..36f30d82c3 100644 --- a/pkg/uploader/kopia/snapshot_test.go +++ b/pkg/uploader/kopia/snapshot_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/fs/virtualfs" "github.com/kopia/kopia/repo" @@ -30,7 +31,6 @@ import ( "github.com/kopia/kopia/snapshot/policy" "github.com/kopia/kopia/snapshot/restore" "github.com/kopia/kopia/snapshot/snapshotfs" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/uploader/provider/kopia.go b/pkg/uploader/provider/kopia.go index 16d8aeb521..c87c5ffa56 100644 --- a/pkg/uploader/provider/kopia.go +++ b/pkg/uploader/provider/kopia.go @@ -22,8 +22,8 @@ import ( "strings" "sync/atomic" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/snapshot/upload" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/uploader" diff --git a/pkg/uploader/provider/kopia_test.go b/pkg/uploader/provider/kopia_test.go index 74eaa67f74..f85c3f2328 100644 --- a/pkg/uploader/provider/kopia_test.go +++ b/pkg/uploader/provider/kopia_test.go @@ -22,9 +22,9 @@ import ( "testing" "time" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot/upload" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/uploader/provider/provider.go b/pkg/uploader/provider/provider.go index fe1dd3091b..5167bbe606 100644 --- a/pkg/uploader/provider/provider.go +++ b/pkg/uploader/provider/provider.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" diff --git a/pkg/uploader/provider/restic.go b/pkg/uploader/provider/restic.go index 93b907be95..8ec0110f0f 100644 --- a/pkg/uploader/provider/restic.go +++ b/pkg/uploader/provider/restic.go @@ -22,7 +22,7 @@ import ( "os" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" diff --git a/pkg/uploader/util/uploader_config.go b/pkg/uploader/util/uploader_config.go index 5584ffbce9..c221741bf2 100644 --- a/pkg/uploader/util/uploader_config.go +++ b/pkg/uploader/util/uploader_config.go @@ -19,7 +19,7 @@ package util import ( "strconv" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) diff --git a/pkg/uploader/util/uploader_config_test.go b/pkg/uploader/util/uploader_config_test.go index 593bce4f09..46df8b7149 100644 --- a/pkg/uploader/util/uploader_config_test.go +++ b/pkg/uploader/util/uploader_config_test.go @@ -20,7 +20,7 @@ import ( "reflect" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) diff --git a/pkg/util/actionhelpers/rbac.go b/pkg/util/actionhelpers/rbac.go index 1ecd97da26..521a8042ae 100644 --- a/pkg/util/actionhelpers/rbac.go +++ b/pkg/util/actionhelpers/rbac.go @@ -19,7 +19,7 @@ package actionhelpers import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" rbacv1 "k8s.io/api/rbac/v1" rbacbeta "k8s.io/api/rbac/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/util/azure/credential.go b/pkg/util/azure/credential.go index b67b34f6c6..f36eb43a68 100644 --- a/pkg/util/azure/credential.go +++ b/pkg/util/azure/credential.go @@ -23,7 +23,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" ) // NewCredential constructs a Credential that tries the config credential, workload identity credential diff --git a/pkg/util/azure/storage.go b/pkg/util/azure/storage.go index 49943a3f92..9f701e80b8 100644 --- a/pkg/util/azure/storage.go +++ b/pkg/util/azure/storage.go @@ -27,7 +27,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" ) diff --git a/pkg/util/azure/util.go b/pkg/util/azure/util.go index e708d6ce33..5e30513363 100644 --- a/pkg/util/azure/util.go +++ b/pkg/util/azure/util.go @@ -29,8 +29,9 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" - "github.com/joho/godotenv" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" + + "github.com/vmware-tanzu/velero/pkg/util/dotenv" ) const ( @@ -68,7 +69,7 @@ func LoadCredentials(config map[string]string) (map[string]string, error) { } // put the credential file content into a map - creds, err := godotenv.Read(credFile) + creds, err := dotenv.Read(credFile) if err != nil { return nil, errors.Wrapf(err, "failed to read credentials from file %s", credFile) } diff --git a/pkg/util/collections/includes_excludes.go b/pkg/util/collections/includes_excludes.go index b5d6513165..8ca16eb738 100644 --- a/pkg/util/collections/includes_excludes.go +++ b/pkg/util/collections/includes_excludes.go @@ -22,8 +22,8 @@ import ( "github.com/vmware-tanzu/velero/internal/resourcepolicies" + "github.com/cockroachdb/errors" "github.com/gobwas/glob" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/validation" "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/util/collections/includes_excludes_test.go b/pkg/util/collections/includes_excludes_test.go index 241fc9fea6..55c51bd9dd 100644 --- a/pkg/util/collections/includes_excludes_test.go +++ b/pkg/util/collections/includes_excludes_test.go @@ -21,7 +21,7 @@ import ( "github.com/vmware-tanzu/velero/internal/resourcepolicies" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/pkg/util/csi/volume_snapshot.go b/pkg/util/csi/volume_snapshot.go index a7d9f055dc..a75763cc7e 100644 --- a/pkg/util/csi/volume_snapshot.go +++ b/pkg/util/csi/volume_snapshot.go @@ -25,10 +25,10 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" jsonpatch "github.com/evanphx/json-patch/v5" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/util/dotenv/dotenv.go b/pkg/util/dotenv/dotenv.go new file mode 100644 index 0000000000..23c0cda014 --- /dev/null +++ b/pkg/util/dotenv/dotenv.go @@ -0,0 +1,165 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dotenv + +import ( + "bufio" + "fmt" + "os" + "strings" +) + +// Read parses dotenv-style files and returns merged key/value pairs. +func Read(filenames ...string) (map[string]string, error) { + filenames = filenamesOrDefault(filenames) + envMap := make(map[string]string) + + for _, filename := range filenames { + fileMap, err := readFile(filename) + if err != nil { + return nil, err + } + + for key, value := range fileMap { + envMap[key] = value + } + } + + return envMap, nil +} + +// Overload loads dotenv-style files into process env vars, overriding existing values. +func Overload(filenames ...string) error { + filenames = filenamesOrDefault(filenames) + + for _, filename := range filenames { + envMap, err := readFile(filename) + if err != nil { + return err + } + + for key, value := range envMap { + if err := os.Setenv(key, value); err != nil { + return err + } + } + } + + return nil +} + +func filenamesOrDefault(filenames []string) []string { + if len(filenames) == 0 { + return []string{".env"} + } + return filenames +} + +func readFile(filename string) (map[string]string, error) { + file, err := os.Open(filename) + if err != nil { + return nil, err + } + defer file.Close() + + envMap := make(map[string]string) + scanner := bufio.NewScanner(file) + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + line = stripInlineComment(line) + key, value, err := parseLine(line) + if err != nil { + return nil, err + } + envMap[key] = value + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + return envMap, nil +} + +func stripInlineComment(line string) string { + inSingle := false + inDouble := false + for i, r := range line { + switch r { + case '\'': + if !inDouble { + inSingle = !inSingle + } + case '"': + if !inSingle { + inDouble = !inDouble + } + case '#': + if !inSingle && !inDouble { + return strings.TrimSpace(line[:i]) + } + } + } + return line +} + +func parseLine(line string) (string, string, error) { + if strings.HasPrefix(line, "export ") { + line = strings.TrimSpace(strings.TrimPrefix(line, "export ")) + } + + sep := strings.Index(line, "=") + colon := strings.Index(line, ":") + if sep == -1 || (colon != -1 && colon < sep) { + sep = colon + } + if sep == -1 { + return "", "", fmt.Errorf("invalid dotenv line: %q", line) + } + + key := strings.TrimSpace(line[:sep]) + rawValue := strings.TrimSpace(line[sep+1:]) + if key == "" { + return "", "", fmt.Errorf("invalid dotenv line: %q", line) + } + + value := parseValue(rawValue) + return key, value, nil +} + +func parseValue(value string) string { + if len(value) >= 2 { + if strings.HasPrefix(value, `"`) && strings.HasSuffix(value, `"`) { + unquoted := strings.TrimSuffix(strings.TrimPrefix(value, `"`), `"`) + unquoted = strings.ReplaceAll(unquoted, `\n`, "\n") + unquoted = strings.ReplaceAll(unquoted, `\r`, "\r") + unquoted = strings.ReplaceAll(unquoted, `\\`, `\`) + unquoted = strings.ReplaceAll(unquoted, `\"`, `"`) + return unquoted + } + if strings.HasPrefix(value, "'") && strings.HasSuffix(value, "'") { + return strings.TrimSuffix(strings.TrimPrefix(value, "'"), "'") + } + } + + return value +} diff --git a/pkg/util/encode/encode.go b/pkg/util/encode/encode.go index b7cbdc1c75..a213e458bc 100644 --- a/pkg/util/encode/encode.go +++ b/pkg/util/encode/encode.go @@ -23,7 +23,7 @@ import ( "fmt" "io" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" diff --git a/pkg/util/exec/exec.go b/pkg/util/exec/exec.go index 109118d582..bdcfef08f8 100644 --- a/pkg/util/exec/exec.go +++ b/pkg/util/exec/exec.go @@ -21,7 +21,7 @@ import ( "io" "os/exec" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" ) diff --git a/pkg/util/kube/node.go b/pkg/util/kube/node.go index ba68536242..3426e508fc 100644 --- a/pkg/util/kube/node.go +++ b/pkg/util/kube/node.go @@ -18,7 +18,7 @@ package kube import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/util/kube/node_test.go b/pkg/util/kube/node_test.go index 9f14c380b8..612b8f977e 100644 --- a/pkg/util/kube/node_test.go +++ b/pkg/util/kube/node_test.go @@ -19,7 +19,7 @@ package kube import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/util/kube/pod.go b/pkg/util/kube/pod.go index 9a59f926f9..e117c22341 100644 --- a/pkg/util/kube/pod.go +++ b/pkg/util/kube/pod.go @@ -23,7 +23,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -319,9 +319,26 @@ func ExitPodWithMessage(logger logrus.FieldLogger, succeed bool, message string, funcExit(exitCode) } +// deepCopy returns a deep copy of the LoadAffinity, so that the returned value +// can be safely modified without affecting the source. +func (a *LoadAffinity) deepCopy() *LoadAffinity { + if a == nil { + return nil + } + + result := &LoadAffinity{ + StorageClass: a.StorageClass, + } + a.NodeSelector.DeepCopyInto(&result.NodeSelector) + + return result +} + // GetLoadAffinityByStorageClass retrieves the LoadAffinity from the parameter affinityList. // The function first try to find by the scName. If there is no such LoadAffinity, // it will try to get the LoadAffinity whose StorageClass has no value. +// The returned LoadAffinity is a deep copy of the matched element, so that the +// callers can modify it without corrupting the shared node-agent configuration. func GetLoadAffinityByStorageClass( affinityList []*LoadAffinity, scName string, @@ -332,7 +349,7 @@ func GetLoadAffinityByStorageClass( for _, affinity := range affinityList { if affinity.StorageClass == scName { logger.WithField("StorageClass", scName).Info("Found pod's affinity setting per StorageClass.") - return affinity + return affinity.deepCopy() } if affinity.StorageClass == "" && globalAffinity == nil { @@ -346,5 +363,5 @@ func GetLoadAffinityByStorageClass( logger.Info("No Affinity is found for pod.") } - return globalAffinity + return globalAffinity.deepCopy() } diff --git a/pkg/util/kube/pod_test.go b/pkg/util/kube/pod_test.go index aa8d4db991..2591f5caaa 100644 --- a/pkg/util/kube/pod_test.go +++ b/pkg/util/kube/pod_test.go @@ -27,8 +27,8 @@ import ( "testing" "time" + "github.com/cockroachdb/errors" "github.com/google/uuid" - "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" @@ -1545,3 +1545,82 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { }) } } + +func TestGetLoadAffinityByStorageClassReturnsCopy(t *testing.T) { + newAffinityList := func() []*LoadAffinity { + return []*LoadAffinity{ + { + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"pool": "backup"}, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: corev1api.LabelArchStable, + Operator: metav1.LabelSelectorOpIn, + Values: []string{"amd64"}, + }, + }, + }, + }, + { + NodeSelector: metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: corev1api.LabelArchStable, + Operator: metav1.LabelSelectorOpIn, + Values: []string{"arm64"}, + }, + }, + }, + StorageClass: "storage-class-01", + }, + } + } + + tests := []struct { + name string + scName string + }{ + { + name: "global affinity", + scName: "no-such-storage-class", + }, + { + name: "affinity matched by StorageClass", + scName: "storage-class-01", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + affinityList := newAffinityList() + + // Simulate the exposers, which append an OS related term to the returned + // affinity on every expose call. The source list must not be affected. + for range 3 { + result := GetLoadAffinityByStorageClass(affinityList, test.scName, velerotest.NewLogger()) + require.NotNil(t, result) + + result.NodeSelector.MatchExpressions = append(result.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: NodeOSLabel, + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{NodeOSWindows}, + }) + + assert.Len(t, result.NodeSelector.MatchExpressions, 2) + } + + assert.Equal(t, newAffinityList(), affinityList) + + // The other fields must be copied as well. + result := GetLoadAffinityByStorageClass(affinityList, test.scName, velerotest.NewLogger()) + require.NotNil(t, result) + result.StorageClass = "modified" + result.NodeSelector.MatchExpressions[0].Values[0] = "modified" + if result.NodeSelector.MatchLabels != nil { + result.NodeSelector.MatchLabels["pool"] = "modified" + } + + assert.Equal(t, newAffinityList(), affinityList) + }) + } +} diff --git a/pkg/util/kube/pvc_pv.go b/pkg/util/kube/pvc_pv.go index fa886bf604..578b245dbc 100644 --- a/pkg/util/kube/pvc_pv.go +++ b/pkg/util/kube/pvc_pv.go @@ -23,8 +23,8 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" jsonpatch "github.com/evanphx/json-patch/v5" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/util/kube/pvc_pv_test.go b/pkg/util/kube/pvc_pv_test.go index 63b8e1edda..93831d3edf 100644 --- a/pkg/util/kube/pvc_pv_test.go +++ b/pkg/util/kube/pvc_pv_test.go @@ -20,7 +20,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/util/kube/resource_requirements.go b/pkg/util/kube/resource_requirements.go index 12cf7a79ca..5c2abecfaf 100644 --- a/pkg/util/kube/resource_requirements.go +++ b/pkg/util/kube/resource_requirements.go @@ -17,7 +17,7 @@ limitations under the License. package kube import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" diff --git a/pkg/util/kube/secrets.go b/pkg/util/kube/secrets.go index e9cb4c04c9..e949b0e978 100644 --- a/pkg/util/kube/secrets.go +++ b/pkg/util/kube/secrets.go @@ -18,9 +18,14 @@ package kube import ( "context" + "reflect" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" + "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" kbclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -49,3 +54,151 @@ func GetSecretKey(client kbclient.Client, namespace string, selector *corev1api. return key, nil } + +// ErrSecretCollision is returned when a secret or configmap with the same name but different +// data already exists in the target namespace, indicating another owner is using it. +var ErrSecretCollision = errors.New("secret collision: same name exists with different data") + +// labelsMatch reports whether all entries in want are present in have with matching values. +func labelsMatch(have, want map[string]string) bool { + for k, v := range want { + if have[k] != v { + return false + } + } + return true +} + +// CopySecret copies a secret from sourceNamespace to targetNamespace, applying the given labels. +// If a secret with the same name already exists in the target with identical data and matching +// labels, it is a no-op. If the data matches but the labels differ, or the data differs, it +// returns ErrSecretCollision. +func CopySecret(ctx context.Context, client corev1client.CoreV1Interface, secretName, sourceNamespace, targetNamespace string, labels map[string]string, log logrus.FieldLogger) error { + srcSecret, err := client.Secrets(sourceNamespace).Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "error getting secret %s/%s", sourceNamespace, secretName) + } + + newSecret := &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: targetNamespace, + Labels: labels, + }, + Type: srcSecret.Type, + Data: srcSecret.Data, + } + + _, err = client.Secrets(targetNamespace).Create(ctx, newSecret, metav1.CreateOptions{}) + if err == nil { + log.Infof("Copied secret %s from %s to %s", secretName, sourceNamespace, targetNamespace) + return nil + } + + if !apierrors.IsAlreadyExists(err) { + return errors.Wrapf(err, "error creating secret %s in %s", secretName, targetNamespace) + } + + existing, err := client.Secrets(targetNamespace).Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "error getting existing secret %s/%s", targetNamespace, secretName) + } + + if reflect.DeepEqual(existing.Data, srcSecret.Data) && labelsMatch(existing.Labels, labels) { + log.Infof("Secret %s already exists in %s with same data and labels, skipping copy", secretName, targetNamespace) + return nil + } + + log.Infof("Secret %s already exists in %s owned by a different owner, collision detected", secretName, targetNamespace) + return ErrSecretCollision +} + +// DeleteSecretsWithLabel deletes all secrets in a namespace matching a label key=value pair. +// Uses UID preconditions to avoid deleting a recreated object with the same name. +func DeleteSecretsWithLabel(ctx context.Context, client corev1client.CoreV1Interface, namespace, labelKey, labelValue string, log logrus.FieldLogger) { + secrets, err := client.Secrets(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: labelKey + "=" + labelValue, + }) + if err != nil { + log.WithError(err).Errorf("Failed to list secrets with label %s=%s in %s", labelKey, labelValue, namespace) + return + } + + for i := range secrets.Items { + uid := secrets.Items[i].UID + err := client.Secrets(namespace).Delete(ctx, secrets.Items[i].Name, metav1.DeleteOptions{ + Preconditions: &metav1.Preconditions{UID: &uid}, + }) + if err != nil && !apierrors.IsNotFound(err) { + log.WithError(err).Errorf("Failed to delete secret %s/%s", namespace, secrets.Items[i].Name) + } + } +} + +// CopyConfigMap copies a configmap from sourceNamespace to targetNamespace, applying the given +// labels. If a configmap with the same name already exists in the target with identical data and +// matching labels, it is a no-op. If the data matches but the labels differ, or the data differs, +// it returns ErrSecretCollision. +func CopyConfigMap(ctx context.Context, client corev1client.CoreV1Interface, cmName, sourceNamespace, targetNamespace string, labels map[string]string, log logrus.FieldLogger) error { + srcCM, err := client.ConfigMaps(sourceNamespace).Get(ctx, cmName, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "error getting configmap %s/%s", sourceNamespace, cmName) + } + + newCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: targetNamespace, + Labels: labels, + }, + Data: srcCM.Data, + BinaryData: srcCM.BinaryData, + } + + _, err = client.ConfigMaps(targetNamespace).Create(ctx, newCM, metav1.CreateOptions{}) + if err == nil { + log.Infof("Copied configmap %s from %s to %s", cmName, sourceNamespace, targetNamespace) + return nil + } + + if !apierrors.IsAlreadyExists(err) { + return errors.Wrapf(err, "error creating configmap %s in %s", cmName, targetNamespace) + } + + existing, err := client.ConfigMaps(targetNamespace).Get(ctx, cmName, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "error getting existing configmap %s/%s", targetNamespace, cmName) + } + + if reflect.DeepEqual(existing.Data, srcCM.Data) && + reflect.DeepEqual(existing.BinaryData, srcCM.BinaryData) && + labelsMatch(existing.Labels, labels) { + log.Infof("ConfigMap %s already exists in %s with same data and labels, skipping copy", cmName, targetNamespace) + return nil + } + + log.Infof("ConfigMap %s already exists in %s owned by a different owner, collision detected", cmName, targetNamespace) + return ErrSecretCollision +} + +// DeleteConfigMapsWithLabel deletes all configmaps in a namespace matching a label key=value pair. +// Uses UID preconditions to avoid deleting a recreated object with the same name. +func DeleteConfigMapsWithLabel(ctx context.Context, client corev1client.CoreV1Interface, namespace, labelKey, labelValue string, log logrus.FieldLogger) { + cms, err := client.ConfigMaps(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: labelKey + "=" + labelValue, + }) + if err != nil { + log.WithError(err).Errorf("Failed to list configmaps with label %s=%s in %s", labelKey, labelValue, namespace) + return + } + + for i := range cms.Items { + uid := cms.Items[i].UID + err := client.ConfigMaps(namespace).Delete(ctx, cms.Items[i].Name, metav1.DeleteOptions{ + Preconditions: &metav1.Preconditions{UID: &uid}, + }) + if err != nil && !apierrors.IsNotFound(err) { + log.WithError(err).Errorf("Failed to delete configmap %s/%s", namespace, cms.Items[i].Name) + } + } +} diff --git a/pkg/util/kube/secrets_copy_test.go b/pkg/util/kube/secrets_copy_test.go new file mode 100644 index 0000000000..ea294eb4e8 --- /dev/null +++ b/pkg/util/kube/secrets_copy_test.go @@ -0,0 +1,362 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kube + +import ( + "context" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1api "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" +) + +const testCopyLabel = "velero.io/backup-pvc-secret" + +func TestCopySecret(t *testing.T) { + log := logrus.New() + + tests := []struct { + name string + secretName string + sourceNS string + targetNS string + ownerName string + objects []k8sruntime.Object + expectErr bool + errContains string + }{ + { + name: "successfully copies secret to target namespace", + secretName: "ceph-csi-kms-token", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-token", Namespace: "app-ns"}, + Data: map[string][]byte{"token": []byte("vault-token-a")}, + Type: corev1api.SecretTypeOpaque, + }, + }, + }, + { + name: "returns error when source secret does not exist", + secretName: "missing-secret", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{}, + expectErr: true, + errContains: "error getting secret", + }, + { + name: "no-op when target already has secret with same data and same owner", + secretName: "ceph-csi-kms-token", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-token", Namespace: "app-ns"}, + Data: map[string][]byte{"token": []byte("same-token")}, + Type: corev1api.SecretTypeOpaque, + }, + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ceph-csi-kms-token", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-123"}, + }, + Data: map[string][]byte{"token": []byte("same-token")}, + Type: corev1api.SecretTypeOpaque, + }, + }, + }, + { + name: "returns collision when same data but different owner", + secretName: "ceph-csi-kms-token", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-456", + objects: []k8sruntime.Object{ + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-token", Namespace: "app-ns"}, + Data: map[string][]byte{"token": []byte("same-token")}, + Type: corev1api.SecretTypeOpaque, + }, + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ceph-csi-kms-token", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-123"}, + }, + Data: map[string][]byte{"token": []byte("same-token")}, + Type: corev1api.SecretTypeOpaque, + }, + }, + expectErr: true, + errContains: "collision", + }, + { + name: "returns collision error when target has secret with different data", + secretName: "ceph-csi-kms-token", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-token", Namespace: "app-ns"}, + Data: map[string][]byte{"token": []byte("token-a")}, + Type: corev1api.SecretTypeOpaque, + }, + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-token", Namespace: "velero"}, + Data: map[string][]byte{"token": []byte("token-b")}, + Type: corev1api.SecretTypeOpaque, + }, + }, + expectErr: true, + errContains: "secret collision", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeClient := fake.NewSimpleClientset(tt.objects...) + + err := CopySecret(context.Background(), fakeClient.CoreV1(), + tt.secretName, tt.sourceNS, tt.targetNS, + map[string]string{testCopyLabel: tt.ownerName}, log) + + if tt.expectErr { + require.Error(t, err) + if tt.errContains != "" { + assert.Contains(t, err.Error(), tt.errContains) + } + return + } + + require.NoError(t, err) + + copied, getErr := fakeClient.CoreV1().Secrets(tt.targetNS).Get( + context.Background(), tt.secretName, metav1.GetOptions{}) + require.NoError(t, getErr) + assert.NotNil(t, copied) + }) + } +} + +func TestDeleteSecretsWithLabel(t *testing.T) { + log := logrus.New() + + fakeClient := fake.NewSimpleClientset( + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-1", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-123"}, + }, + }, + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-2", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-456"}, + }, + }, + ) + + DeleteSecretsWithLabel(context.Background(), fakeClient.CoreV1(), "velero", + testCopyLabel, "du-123", log) + + _, err := fakeClient.CoreV1().Secrets("velero").Get( + context.Background(), "secret-1", metav1.GetOptions{}) + require.Error(t, err, "secret-1 should be deleted") + + _, err = fakeClient.CoreV1().Secrets("velero").Get( + context.Background(), "secret-2", metav1.GetOptions{}) + assert.NoError(t, err, "secret-2 should still exist") +} + +func TestCopyConfigMap(t *testing.T) { + log := logrus.New() + + tests := []struct { + name string + cmName string + sourceNS string + targetNS string + ownerName string + objects []k8sruntime.Object + expectErr bool + errContains string + }{ + { + name: "successfully copies configmap to target namespace", + cmName: "ceph-csi-kms-config", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-config", Namespace: "app-ns"}, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + }, + }, + }, + { + name: "returns error when source configmap does not exist", + cmName: "missing-cm", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{}, + expectErr: true, + errContains: "error getting configmap", + }, + { + name: "no-op when target already has configmap with same data and same owner", + cmName: "ceph-csi-kms-config", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-config", Namespace: "app-ns"}, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + }, + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ceph-csi-kms-config", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-123"}, + }, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + }, + }, + }, + { + name: "returns collision when same data but different owner", + cmName: "ceph-csi-kms-config", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-456", + objects: []k8sruntime.Object{ + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-config", Namespace: "app-ns"}, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + }, + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ceph-csi-kms-config", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-123"}, + }, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + }, + }, + expectErr: true, + errContains: "collision", + }, + { + name: "copies configmap with BinaryData", + cmName: "ceph-csi-kms-config", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-config", Namespace: "app-ns"}, + BinaryData: map[string][]byte{"ca.crt": []byte("binary-ca-bundle")}, + }, + }, + }, + { + name: "returns collision error when target has configmap with different data", + cmName: "ceph-csi-kms-config", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-config", Namespace: "app-ns"}, + Data: map[string]string{"vaultAddress": "https://vault-a.example.com"}, + }, + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-config", Namespace: "velero"}, + Data: map[string]string{"vaultAddress": "https://vault-b.example.com"}, + }, + }, + expectErr: true, + errContains: "secret collision", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeClient := fake.NewSimpleClientset(tt.objects...) + + err := CopyConfigMap(context.Background(), fakeClient.CoreV1(), + tt.cmName, tt.sourceNS, tt.targetNS, + map[string]string{testCopyLabel: tt.ownerName}, log) + + if tt.expectErr { + require.Error(t, err) + if tt.errContains != "" { + assert.Contains(t, err.Error(), tt.errContains) + } + return + } + + require.NoError(t, err) + + copied, getErr := fakeClient.CoreV1().ConfigMaps(tt.targetNS).Get( + context.Background(), tt.cmName, metav1.GetOptions{}) + require.NoError(t, getErr) + assert.NotNil(t, copied) + }) + } +} + +func TestDeleteConfigMapsWithLabel(t *testing.T) { + log := logrus.New() + + fakeClient := fake.NewSimpleClientset( + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cm-1", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-123"}, + }, + }, + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cm-2", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-456"}, + }, + }, + ) + + DeleteConfigMapsWithLabel(context.Background(), fakeClient.CoreV1(), "velero", + testCopyLabel, "du-123", log) + + _, err := fakeClient.CoreV1().ConfigMaps("velero").Get( + context.Background(), "cm-1", metav1.GetOptions{}) + require.Error(t, err, "cm-1 should be deleted") + + _, err = fakeClient.CoreV1().ConfigMaps("velero").Get( + context.Background(), "cm-2", metav1.GetOptions{}) + assert.NoError(t, err, "cm-2 should still exist") +} diff --git a/pkg/util/kube/security_context.go b/pkg/util/kube/security_context.go index 1fd911649a..9b62b32fbe 100644 --- a/pkg/util/kube/security_context.go +++ b/pkg/util/kube/security_context.go @@ -19,7 +19,7 @@ package kube import ( "strconv" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" "sigs.k8s.io/yaml" ) diff --git a/pkg/util/kube/utils.go b/pkg/util/kube/utils.go index 5e5e976033..44aaf9d494 100644 --- a/pkg/util/kube/utils.go +++ b/pkg/util/kube/utils.go @@ -23,7 +23,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" diff --git a/pkg/util/logging/dual_mode_logger.go b/pkg/util/logging/dual_mode_logger.go index f5533c8eb0..efcfceb3a3 100644 --- a/pkg/util/logging/dual_mode_logger.go +++ b/pkg/util/logging/dual_mode_logger.go @@ -21,7 +21,7 @@ import ( "io" "os" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" ) diff --git a/pkg/util/logging/error_location_hook.go b/pkg/util/logging/error_location_hook.go index 5246a8318c..affcefa108 100644 --- a/pkg/util/logging/error_location_hook.go +++ b/pkg/util/logging/error_location_hook.go @@ -21,7 +21,7 @@ import ( "strconv" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors/errbase" "github.com/sirupsen/logrus" ) @@ -32,7 +32,7 @@ const ( // ErrorLocationHook is a logrus hook that attaches error location information // to log entries if an error is being logged and it has stack-trace information -// (i.e. if it originates from or is wrapped by github.com/pkg/errors, or if it +// (i.e. if it carries a stack trace from wrapped errors), or if it // implements the errorLocationer interface, like errors returned from plugins // typically do). type ErrorLocationHook struct{} @@ -90,8 +90,8 @@ type LocationInfo struct { } // GetFrameLocationInfo returns the location of a frame. -func GetFrameLocationInfo(frame errors.Frame) LocationInfo { - // see https://godoc.org/github.com/pkg/errors#Frame.Format for +func GetFrameLocationInfo(frame errbase.StackFrame) LocationInfo { + // see https://pkg.go.dev/github.com/cockroachdb/errors#Frame.Format for // details on formatting verbs functionNameAndFileAndLine := fmt.Sprintf("%+v", frame) @@ -121,7 +121,7 @@ type errorLocationer interface { type stackTracer interface { error - StackTrace() errors.StackTrace + StackTrace() errbase.StackTrace } type causer interface { diff --git a/pkg/util/logging/error_location_hook_test.go b/pkg/util/logging/error_location_hook_test.go index f6f230c029..afd6430ee8 100644 --- a/pkg/util/logging/error_location_hook_test.go +++ b/pkg/util/logging/error_location_hook_test.go @@ -20,7 +20,7 @@ import ( "errors" "testing" - pkgerrs "github.com/pkg/errors" + pkgerrs "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -95,6 +95,7 @@ func TestFire(t *testing.T) { // test existence of this field only since testing the value // is fragile case errorFileField: + case errorFunctionField: case logrus.ErrorKey: if err, ok := expectedValue.(error); ok { assert.Equal(t, err.Error(), actualValue.(error).Error()) @@ -108,69 +109,3 @@ func TestFire(t *testing.T) { }) } } - -func TestGetInnermostTrace(t *testing.T) { - newError := func() error { - return errors.New("a normal error") - } - - tests := []struct { - name string - err error - expectedRes error - }{ - { - name: "normal error", - err: newError(), - expectedRes: nil, - }, - { - name: "pkg/errs error", - err: pkgerrs.New("a pkg/errs error"), - expectedRes: pkgerrs.New("a pkg/errs error"), - }, - { - name: "one level of stack-ing a normal error", - err: pkgerrs.WithStack(newError()), - expectedRes: pkgerrs.WithStack(newError()), - }, - { - name: "two levels of stack-ing a normal error", - err: pkgerrs.WithStack(pkgerrs.WithStack(newError())), - expectedRes: pkgerrs.WithStack(newError()), - }, - { - name: "one level of stack-ing a pkg/errors error", - err: pkgerrs.WithStack(pkgerrs.New("a pkg/errs error")), - expectedRes: pkgerrs.New("a pkg/errs error"), - }, - { - name: "two levels of stack-ing a pkg/errors error", - err: pkgerrs.WithStack(pkgerrs.WithStack(pkgerrs.New("a pkg/errs error"))), - expectedRes: pkgerrs.New("a pkg/errs error"), - }, - { - name: "two levels of wrapping a normal error", - err: pkgerrs.Wrap(pkgerrs.Wrap(newError(), "wrap 1"), "wrap 2"), - expectedRes: pkgerrs.Wrap(newError(), "wrap 1"), - }, - { - name: "two levels of wrapping a pkg/errors error", - err: pkgerrs.Wrap(pkgerrs.Wrap(pkgerrs.New("a pkg/errs error"), "wrap 1"), "wrap 2"), - expectedRes: pkgerrs.New("a pkg/errs error"), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - res := getInnermostTrace(test.err) - - if test.expectedRes == nil { - require.NoError(t, res) - return - } - - assert.Equal(t, test.expectedRes.Error(), res.Error()) - }) - } -} diff --git a/pkg/util/logging/log_merge_hook.go b/pkg/util/logging/log_merge_hook.go index b993cb38ab..3854539257 100644 --- a/pkg/util/logging/log_merge_hook.go +++ b/pkg/util/logging/log_merge_hook.go @@ -21,7 +21,7 @@ import ( "io" "os" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" ) diff --git a/pkg/util/logging/log_merge_hook_test.go b/pkg/util/logging/log_merge_hook_test.go index 43b5ae1cb8..d4e7870c44 100644 --- a/pkg/util/logging/log_merge_hook_test.go +++ b/pkg/util/logging/log_merge_hook_test.go @@ -21,7 +21,7 @@ import ( "os" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/pkg/util/podvolume/pod_volume.go b/pkg/util/podvolume/pod_volume.go index 7c7e0f9c45..3c9ad2127d 100644 --- a/pkg/util/podvolume/pod_volume.go +++ b/pkg/util/podvolume/pod_volume.go @@ -21,7 +21,7 @@ import ( "strings" "sync" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" crclient "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/util/results/result_test.go b/pkg/util/results/result_test.go index 26017c35fa..85f94364c3 100644 --- a/pkg/util/results/result_test.go +++ b/pkg/util/results/result_test.go @@ -19,7 +19,7 @@ package results import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" ) diff --git a/restic b/restic index 6accd09859..990719ca53 160000 --- a/restic +++ b/restic @@ -1 +1 @@ -Subproject commit 6accd09859c7bbb21309ff244750f43d3500ef05 +Subproject commit 990719ca53f488209d7fe834ad1c3e35ee7d4126 diff --git a/site/content/docs/main/custom-plugins.md b/site/content/docs/main/custom-plugins.md index 8e3cf3f37a..ce436bda97 100644 --- a/site/content/docs/main/custom-plugins.md +++ b/site/content/docs/main/custom-plugins.md @@ -65,6 +65,32 @@ order in which item action plugins are invoked. However, if a single binary impl they may be invoked in the order in which they are registered but it is best to not depend on this implementation. This is not guaranteed officially and the implementation can change at any time. +### Must-include additional items (Restore Item Actions) + +Restore Item Actions may return `AdditionalItems` that Velero restores as dependencies of the current item. +By default those additional items must still pass the restore's global resource and namespace include/exclude +filters (and `IncludeClusterResources=false` for cluster-scoped resources). + +To force-restore hard dependencies despite those filters, set the following annotation on the `UpdatedItem` +returned from `Execute()`: + +``` +restore.velero.io/must-include-additional-items: "true" +``` + +Behavior: +- Only the string value `"true"` enables the bypass. +- The annotation applies blanket to all `AdditionalItems` from that RIA invocation (not per-item). +- Velero strips the annotation before applying the item to the cluster. +- `SkipRestore: true` takes precedence: if set, the annotation is never inspected and `AdditionalItems` are not processed. +- Must-include only bypasses filters; the additional item must still exist in the backup tarball. +- When an additional item targets an excluded namespace, Velero may still create that target namespace so the item can be restored. +- Cluster-scoped additional items are restored even when `IncludeClusterResources=false`. +- Transitive force-include requires each RIA level to re-set the annotation on its own `UpdatedItem`. + +This mirrors the backup-side annotation `backup.velero.io/must-include-additional-items` used by Backup Item Actions. +Installing an RIA that sets this annotation is a trust decision: the plugin can restore resources outside the operator's restore filters. + ## Plugin Logging Velero provides a [logger][2] that can be used by plugins to log structured information to the main Velero server log or diff --git a/site/content/docs/main/resource-filtering.md b/site/content/docs/main/resource-filtering.md index cbfdb2816e..88584b362d 100644 --- a/site/content/docs/main/resource-filtering.md +++ b/site/content/docs/main/resource-filtering.md @@ -287,6 +287,11 @@ The policies YAML config file would look like this: # pvc matches specific phase(s) pvcPhase: - Pending + # pvc matches specific volume mode + pvcVolumeMode: Block + # pvc matches specific access mode(s) + pvcAccessModes: + - ReadWriteOnce action: type: skip - conditions: @@ -380,6 +385,8 @@ Currently, Velero supports the volume attributes listed below: - storageClass: matching volumes those with specified `storageClass`, such as `gp2`, `ebs-sc` in eks - volume sources: matching volumes that used specified volume sources. Currently we support nfs or csi backend volume source - pvcPhase: matching volumes based on the phase of their associated PVCs (Pending, Bound, Lost) +- pvcVolumeMode: matching volumes based on the volume mode of their associated PVCs (Filesystem, Block) +- pvcAccessModes: matching volumes based on the access modes of their associated PVCs (ReadWriteOnce, ReadOnlyMany, ReadWriteMany, ReadWriteOncePod). All configured access modes must be present on the PVC. Velero supported conditions and format listed below: - capacity @@ -521,6 +528,72 @@ Velero supported conditions and format listed below: type: skip ``` +- pvc VolumeMode + + This condition filters PVC-backed volumes based on the volume mode of their associated PVCs. The condition is specified as a single volume mode to match. The volume matches this condition if the PVC's volume mode exactly matches the configured value. Matching is case-sensitive, so `block` does not match `Block`. Supported volume modes are: `Filesystem` and `Block`. If `pvcVolumeMode` is omitted from a policy, volume mode is not restricted. Non-PVC volumes, such as `emptyDir`, `configMap`, or inline volumes without an associated PVC, do not match policies that require this condition. + ```yaml + pvcVolumeMode: Block + ``` + + Some examples: + - Skip Block PVCs: Skip backup of volumes whose associated PVC uses `Block` volume mode. + ```yaml + volumePolicies: + - conditions: + pvcVolumeMode: Block + action: + type: skip + ``` + - Combine with other conditions: You can combine PVC volume mode conditions with other conditions like PVC phase, storage class, or labels. + ```yaml + volumePolicies: + - conditions: + pvcVolumeMode: Block + pvcPhase: + - Bound + action: + type: snapshot + ``` + +- pvc AccessModes + + This condition filters PVC-backed volumes based on the access modes of their associated PVCs. The condition is specified as a list of access modes to match. The volume matches this condition only if the PVC has all of the access modes in the list. Matching is case-sensitive, so `readwriteonce` does not match `ReadWriteOnce`. Supported access modes are: `ReadWriteOnce`, `ReadOnlyMany`, `ReadWriteMany`, and `ReadWriteOncePod`. Non-PVC volumes, such as `emptyDir`, `configMap`, or inline volumes without an associated PVC, do not match policies that require this condition. + ```yaml + pvcAccessModes: + - ReadWriteOnce + ``` + + Some examples: + - Skip ReadWriteOnce PVCs: Skip backup of volumes whose associated PVC includes the `ReadWriteOnce` access mode. + ```yaml + volumePolicies: + - conditions: + pvcAccessModes: + - ReadWriteOnce + action: + type: skip + ``` + - Match multiple access modes: Apply an action to volumes whose associated PVC includes both `ReadOnlyMany` and `ReadWriteMany`. + ```yaml + volumePolicies: + - conditions: + pvcAccessModes: + - ReadOnlyMany + - ReadWriteMany + action: + type: snapshot + ``` + - Combine with other conditions: You can combine PVC access mode conditions with other conditions like PVC volume mode, PVC phase, storage class, or labels. + ```yaml + volumePolicies: + - conditions: + pvcAccessModes: + - ReadWriteOnce + pvcVolumeMode: Block + action: + type: snapshot + ``` + ### Resource policies rules @@ -631,3 +704,85 @@ volumePolicies: 3. The outcome would be that velero would perform `fs-backup` operation on both the volumes - `fs-backup` on `Volume 1` because `Volume 1` satisfies the criteria for `fs-backup` action. - Also, for Volume 2 as no matching action was found so legacy approach will be used as a fallback option for this volume (`fs-backup` operation will be done as `defaultVolumesToFSBackup: true` is specified by the user). + +### Global backup volume policies + +Resource policies (volume policies) are normally opt-in per backup via `--resource-policies-configmap`. An administrator can instead configure a cluster-wide baseline that applies to **every** backup by starting the Velero server with the `--global-backup-volume-policies-configmap` flag, pointing at a ConfigMap in the Velero install namespace: + +```bash +velero server --global-backup-volume-policies-configmap global-volume-policy +``` + +The ConfigMap uses the exact same format as a per-backup resource policies ConfigMap (a single data key holding a `ResourcePolicies` YAML document): + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: global-volume-policy + namespace: velero +data: + policies.yaml: | + version: v1 + volumePolicies: + - conditions: + storageClass: + - gp2 + action: + type: skip +``` + +#### Behavior + +- **Only `volumePolicies` apply globally.** If the global ConfigMap contains `includeExcludePolicy`, `clusterScopedFilterPolicy`, or `namespacedFilterPolicies`, those sections are ignored and a warning is logged. Those filters are tied to a specific backup use case, so they remain per-backup only. +- **Merge semantics.** When a backup runs, the effective `volumePolicies` list is the backup-level policies followed by the global policies: + + ``` + merged.volumePolicies = backup.volumePolicies ++ global.volumePolicies + ``` + + Because the first matching policy wins, a backup can override the global baseline for a specific volume while still inheriting every global rule it does not override. If a backup references no resource policy, the global policy applies on its own. +- **Validation.** The global ConfigMap is validated at server startup (the server fails to start if it is missing or invalid) and again on each backup (a backup whose global policy has become missing or invalid is moved to the `FailedValidation` phase). + +#### Example + +Global policy (`--global-backup-volume-policies-configmap=global-volume-policy`): skip `gp2` volumes. + +```yaml +version: v1 +volumePolicies: + - conditions: + storageClass: + - gp2 + action: + type: skip +``` + +Backup-level policy (`--resource-policies-configmap backup01`): `fs-backup` NFS volumes. + +```yaml +version: v1 +volumePolicies: + - conditions: + nfs: {} + action: + type: fs-backup +``` + +Effective (merged) policy used for the backup — backup rules first, then global: + +```yaml +version: v1 +volumePolicies: + - conditions: + nfs: {} + action: + type: fs-backup + - conditions: + storageClass: + - gp2 + action: + type: skip +``` + +When a global policy contributes to a backup, `velero backup describe` surfaces the contributing ConfigMap under a `Global volume policies` section. diff --git a/site/content/docs/v1.18/custom-plugins.md b/site/content/docs/v1.18/custom-plugins.md index 33c6c1ba86..e160cb6bb6 100644 --- a/site/content/docs/v1.18/custom-plugins.md +++ b/site/content/docs/v1.18/custom-plugins.md @@ -65,6 +65,32 @@ order in which item action plugins are invoked. However, if a single binary impl they may be invoked in the order in which they are registered but it is best to not depend on this implementation. This is not guaranteed officially and the implementation can change at any time. +### Must-include additional items (Restore Item Actions) + +Restore Item Actions may return `AdditionalItems` that Velero restores as dependencies of the current item. +By default those additional items must still pass the restore's global resource and namespace include/exclude +filters (and `IncludeClusterResources=false` for cluster-scoped resources). + +To force-restore hard dependencies despite those filters, set the following annotation on the `UpdatedItem` +returned from `Execute()`: + +``` +restore.velero.io/must-include-additional-items: "true" +``` + +Behavior: +- Only the string value `"true"` enables the bypass. +- The annotation applies blanket to all `AdditionalItems` from that RIA invocation (not per-item). +- Velero strips the annotation before applying the item to the cluster. +- `SkipRestore: true` takes precedence: if set, the annotation is never inspected and `AdditionalItems` are not processed. +- Must-include only bypasses filters; the additional item must still exist in the backup tarball. +- When an additional item targets an excluded namespace, Velero may still create that target namespace so the item can be restored. +- Cluster-scoped additional items are restored even when `IncludeClusterResources=false`. +- Transitive force-include requires each RIA level to re-set the annotation on its own `UpdatedItem`. + +This mirrors the backup-side annotation `backup.velero.io/must-include-additional-items` used by Backup Item Actions. +Installing an RIA that sets this annotation is a trust decision: the plugin can restore resources outside the operator's restore filters. + ## Plugin Logging Velero provides a [logger][2] that can be used by plugins to log structured information to the main Velero server log or diff --git a/site/content/docs/v1.18/fine-grained-backup-filters.md b/site/content/docs/v1.18/fine-grained-backup-filters.md new file mode 100644 index 0000000000..fe9d5bb3b6 --- /dev/null +++ b/site/content/docs/v1.18/fine-grained-backup-filters.md @@ -0,0 +1,856 @@ +--- +title: "Fine-Grained Backup Filters" +layout: docs +--- + +This guide explains how to use Velero's **fine-grained backup filters**: per-namespace, per-kind rules with independent label selectors and resource name patterns. Configuration lives in the same **ResourcePolicy ConfigMap** you may already use for volume policies. + +For architecture and pipeline details, see the [design document](https://github.com/velero-io/velero/blob/main/design/backup-filter-enhancement/fine-grained-backup-filters-design.md). + +--- + +## Introduction + +Velero's global backup filters apply the same namespace list, resource types, and label selector to every namespace in a backup. That works for many clusters, but common scenarios need more control: + +- **Different namespaces, different strategies** — back up everything in a database namespace, but only Deployments and ConfigMaps in a frontend namespace. +- **Filter by resource name** — back up `app-config` and `app-secret` without also capturing `monitoring-config`. +- **Different labels per kind** — Deployments labeled `app=workload-1` and StatefulSets labeled `app=workload-2` in the same namespace. + +Fine-grained filters add two optional sections to the ResourcePolicy ConfigMap: + +| Section | Scope | Behavior | +|---------|-------|----------| +| `namespacedFilterPolicies` | Namespaces you match (exact name or glob) | **Exclusive allowlist** — only resource kinds listed in `resourceFilters` (or covered by a catch-all) are backed up from those namespaces | +| `clusterScopedFilterPolicy` | Cluster-scoped resources globally | **Refinement overlay** — listed kinds get per-kind label and name rules; unlisted cluster-scoped kinds still use global BackupSpec filters | + +**No new BackupSpec CRD fields** are required. Reference the policy from `Backup.spec.resourcePolicy` or `velero backup create --resource-policies-configmap`. + +**Backward compatible:** if you omit both new sections, backups behave exactly as they do today. + +--- + +## Prerequisites and wiring + +### What you need + +- Velero installed with backup filters support (see your Velero release notes). +- A ResourcePolicy ConfigMap in the Velero namespace (`velero` by default). +- Permission to create Backups (or Schedules) that reference the ConfigMap. + +### End-to-end pattern + +Every example below follows the same three steps: + +1. **Create or update** a ConfigMap with `data.policy` containing `version: v1` and your filter rules. +2. **Create a Backup** (or Schedule) that includes the target namespaces and references the ConfigMap. +3. **Verify** with `velero backup describe` and inspect backup contents or logs. + +### Minimal skeleton + +Use this once; later examples show only the `policy:` body. + +**ResourcePolicy ConfigMap:** + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-backup-filter-policy + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + - namespaces: + - my-namespace + resourceFilters: + - kinds: [ConfigMap] + labelSelector: + matchLabels: + app: my-app +``` + +**Backup:** + +```yaml +apiVersion: velero.io/v1 +kind: Backup +metadata: + name: my-backup + namespace: velero +spec: + includedNamespaces: + - my-namespace + resourcePolicy: + kind: configmap + name: my-backup-filter-policy + storageLocation: default +``` + +**CLI equivalent:** + +```bash +velero backup create my-backup \ + --include-namespaces my-namespace \ + --resource-policies-configmap my-backup-filter-policy +``` + +**Verify:** + +```bash +velero backup describe my-backup +velero backup describe my-backup -o json | jq '.namespacedFilterPolicies' +``` + +### Important: do not mix old-style BackupSpec resource filters + +When `namespacedFilterPolicies` or `clusterScopedFilterPolicy` is present in the ResourcePolicy, **do not** set these on the Backup: + +- `spec.includedResources` / `spec.excludedResources` +- `spec.includeClusterResources` + +Use `includeExcludePolicy` inside the ResourcePolicy ConfigMap for global resource-type include/exclude instead. Velero rejects backups that combine the new policy sections with old-style fields. + +Schedules follow the same rule: configure filters in the ResourcePolicy ConfigMap, not via deprecated resource filter fields on the Schedule template. + +--- + +## Examples + +Each example includes: **goal**, **policy YAML**, **backup notes**, **expected outcome**, and **how to verify**. + +--- + +### Example 0 — Baseline (no new filters) + +**Goal:** Confirm that namespaces without a `namespacedFilterPolicies` entry still use global BackupSpec filters. + +**Policy:** Omit `namespacedFilterPolicies` and `clusterScopedFilterPolicy` entirely (or use a ConfigMap with only `volumePolicies` / `includeExcludePolicy`). + +**Backup:** + +```yaml +spec: + includedNamespaces: + - ns-a + - ns-b + - production + # No resourcePolicy — global filters only +``` + +**Expected outcome:** All resources in included namespaces follow `includedNamespaces`, `labelSelector`, `includedResources`, and related global fields — same as before this feature. + +**Verify:** `velero backup describe` shows no namespace-scoped filter policies section. + +--- + +### Example 1 — Per-namespace kinds and labels + +**Goal:** In `ns-a`, back up only ConfigMaps, Secrets, Deployments, and Pods with `app=my-app`. In `ns-b`, use global filters (no policy entry for that namespace). + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret, Deployment, Pod] + labelSelector: + matchLabels: + app: my-app +``` + +**Backup:** + +```yaml +spec: + includedNamespaces: + - ns-a + - ns-b + resourcePolicy: + kind: configmap + name: per-namespace-resource-filter-policy # or your ConfigMap name +``` + +**Expected outcome:** + +- **ns-a:** Only listed kinds with label `app=my-app` (e.g. `app-config`, `app-secret`, `app-deployment`). Resources like `monitoring-config` (different labels) are excluded. +- **ns-b:** Everything allowed by global filters (no namespace policy match). + +**Verify:** `velero backup describe` lists resolved filters for `ns-a`. + +--- + +### Example 2 — Exact resource names + +**Goal:** Back up only two ConfigMaps by exact name, optionally requiring a label. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - target-namespace + resourceFilters: + - kinds: [ConfigMap] + names: [vm-1, vm-2] + labelSelector: + matchLabels: + resource-type: VirtualMachine +``` + +**Backup:** `includedNamespaces: [target-namespace]` plus `resourcePolicy` reference. + +**Expected outcome:** Only `vm-1` and `vm-2` ConfigMaps with `resource-type=VirtualMachine`. `vm-3` and other ConfigMaps are excluded. + +**Verify:** Backup archive contains exactly those two ConfigMaps in `target-namespace`. + +--- + +### Example 3 — Glob name patterns with exclusions + +**Goal:** Back up `app-*` ConfigMaps and Secrets in `production`, but exclude temporary and debug names. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [ConfigMap, Secret] + names: ["app-*"] + excludedNames: ["*-tmp-*", "*-debug-*", "*-tmp", "*-debug"] +``` + +**Expected outcome:** + +- **Included:** `app-config`, `app-cache-config`, `app-secret`, `app-db-secret` +- **Excluded:** `app-tmp-config`, `app-debug-config` (excluded by `excludedNames`), and `monitoring-tmp-secret` (excluded because it does not match the `names: ["app-*"]` allowlist) + +`excludedNames` takes precedence over `names` when both match. + +**Verify:** Inspect backup item list. + +--- + +### Example 4 — Per-kind label selectors + +**Goal:** Apply different label rules to different resource types in the same namespace. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - target-namespace + resourceFilters: + - kinds: [ConfigMap] + orLabelSelectors: + - matchLabels: + app: production-workload-1 + component: vm-group + - matchLabels: + app: production-workload-2 + component: vm-service +``` + +**Expected outcome:** ConfigMaps matching either label combination are backed up; other ConfigMaps in the namespace are not (for this kind). + +**Note:** Prefer `matchExpressions` with `In` for value-OR on a single key (see next example). Use `orLabelSelectors` when you need OR across **independent multi-key groups**. `labelSelector` and `orLabelSelectors` cannot appear in the same `resourceFilters` entry. + +--- + +### Example 4b — Set-based label selectors (`matchExpressions`) + +**Goal:** Back up Deployments and Pods that are in `prod` or `staging`, belong to `app=my-app`, and do **not** carry a skip label. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [Deployment, Pod] + labelSelector: + matchLabels: + app: my-app + matchExpressions: + - key: environment + operator: In + values: [prod, staging] + - key: do-not-backup + operator: DoesNotExist +``` + +**Supported operators:** `In`, `NotIn`, `Exists`, `DoesNotExist` (same as Kubernetes / Velero global `--selector`). + +**Other useful patterns:** + +```yaml +# Exclude environments +matchExpressions: + - key: environment + operator: NotIn + values: [dev, test] + +# Require a label key to be present (any value) +matchExpressions: + - key: tier + operator: Exists +``` + +**Expected outcome:** Only Deployments/Pods with `app=my-app`, `environment` in `{prod, staging}`, and without `do-not-backup` are backed up. + +--- + +### Example 5 — OR label selectors across kinds + +**Goal:** Back up ConfigMaps, Secrets, or Deployments that match any of several label conditions. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret] + orLabelSelectors: + - matchLabels: + app: my-app + - matchLabels: + app: monitoring + - kinds: [Deployment] + orLabelSelectors: + - matchLabels: + app: my-app + - matchLabels: + app: monitoring + - matchLabels: + component: backend +``` + +**Expected outcome:** Resources included if they match **any** selector in `orLabelSelectors` for their kind (AND within each selector, OR across the list). + +--- + +### Example 6 — Multiple criteria on one kind + +**Goal:** Combine exact names with OR label selectors for a single kind. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - target-namespace + resourceFilters: + - kinds: [ConfigMap] + names: [vm-1, vm-2] + orLabelSelectors: + - matchLabels: + resource-type: VirtualMachine + - matchLabels: + component: vm-group + - matchLabels: + component: vm-service +``` + +**Expected outcome:** Only `vm-1` and `vm-2` that also satisfy one of the label OR branches. + +--- + +### Example 7 — One policy entry, multiple namespaces + +**Goal:** Apply the same rules to `ns-a`, `ns-b`, and `production` in a single policy block. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + - ns-b + - production + resourceFilters: + - kinds: [ConfigMap] + - kinds: [Deployment] + labelSelector: + matchLabels: + tier: web +``` + +**Expected outcome:** + +- All ConfigMaps in those namespaces (no label filter on that entry). +- Deployments with `tier=web` only. + +--- + +### Example 8 — Namespace glob patterns and ordering + +**Goal:** Different backup breadth for `team-frontend-prod`, `team-frontend-dev`, and `team-backend-test` using glob patterns. + +**Policy (correct order — most specific first):** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - "team-frontend-*" + resourceFilters: + - kinds: [Deployment, Service, ConfigMap] + - namespaces: + - "team-*" + resourceFilters: + - kinds: [Deployment, Service] + - namespaces: + - team-frontend-prod # exact match + resourceFilters: + - kinds: [Deployment, Service, ConfigMap, Secret, PersistentVolumeClaim] +``` + +**Expected outcome:** + +| Namespace | Matched policy | Kinds backed up | +|-----------|----------------|-----------------| +| `team-frontend-prod` | First entry (exact) | 5 kinds | +| `team-frontend-dev` | `team-frontend-*` | 3 kinds | +| `team-backend-test` | `team-*` | 2 kinds | + +**Wrong order (avoid):** If `team-*` is listed **before** `team-frontend-*`, then `team-frontend-dev` matches the broader `team-*` rule first and only Deployments and Services are backed up — the more specific `team-frontend-*` rule is never reached. + +Velero evaluates namespaces by looking for an **exact match** first, and then evaluates glob patterns in **definition order** (first-match wins). Because `team-frontend-prod` is an exact match in this policy, its evaluation is unaffected by glob ordering. However, for namespaces relying on glob patterns like `team-frontend-dev`, the order of the glob patterns is critical. + +**Backup:** Include all relevant namespaces in `includedNamespaces` (they must still pass the global namespace filter). + +--- + +### Example 9 — Catch-all by label + +**Goal:** Back up any resource kind that has a given label, without listing every kind. Kind-specific entries override the catch-all. + +**Policy (recommended explicit form):** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: ["*"] # catch-all + labelSelector: + matchLabels: + app: common-app + - kinds: [ConfigMap, Secret] # override for these kinds + labelSelector: + matchLabels: + app: specialized-app +``` + +**Equivalent:** `kinds: []` (empty) also denotes a catch-all; `kinds: ["*"]` is preferred for readability. + +**Rules:** + +- At most **one** catch-all per namespace policy entry. +- Catch-all entries **cannot** use `names` or `excludedNames` — use kind-specific entries for name filtering. +- Catch-all does **not** inherit `BackupSpec.labelSelector`; set `labelSelector` or `orLabelSelectors` on the catch-all entry explicitly. + +**Expected outcome:** ConfigMaps and Secrets use `app=specialized-app`; all other kinds listed only via catch-all use `app=common-app`. + +--- + +### Example 10 — Catch-all with per-kind name overrides + +**Goal:** Pin critical Deployments and Secrets by exact name; back up everything else with a label convention. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [Deployment] + names: [api-server, worker] + - kinds: [Secret] + names: [db-credentials, tls-cert] + - kinds: ["*"] + labelSelector: + matchLabels: + backup: "true" +``` + +**Expected outcome:** + +- Deployments: only `api-server` and `worker` +- Secrets: only `db-credentials` and `tls-cert` +- Other kinds (ConfigMap, Service, …): resources with `backup=true` only + +**Verify:** `other-deployment` and `no-backup-label-config` should be absent; `backup-labeled-config` and `catch-all-labeled-service` should be present. + +--- + +### Example 11 — Override-only catch-all (no label on catch-all) + +**Goal:** Apply a strict name filter to one kind while including all other kinds without listing them or adding labels. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [Secret] + names: [app-secret] + - kinds: ["*"] # no labelSelector — all other kinds included +``` + +**Expected outcome:** + +- Secrets: only `app-secret` +- Other kinds in `ns-a`: all instances included (subject to global filters and allowlist semantics for listed vs unlisted kinds via catch-all) + +Use this when you need a narrow exception for one type and broad inclusion for the rest of the namespace. + +--- + +### Example 12 — Cluster-scoped refinement + +**Goal:** Refine which cluster-scoped resources are backed up by name and label, without replacing global cluster-scoped inclusion. + +**Policy:** + +```yaml +version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: [StorageClass] + names: ["my-app-*"] + - kinds: [ClusterRole, ClusterRoleBinding] + labelSelector: + matchLabels: + app: my-app +``` + +**Backup (required):** You must still include cluster-scoped kinds on the Backup: + +```yaml +spec: + includedNamespaces: + - ns-a + includedClusterScopedResources: + - storageclasses + - clusterroles + - clusterrolebindings + resourcePolicy: + kind: configmap + name: cluster-scoped-filter-policy +``` + +**Expected outcome (full overlay):** + +- StorageClasses matching `my-app-*` only +- ClusterRoles and ClusterRoleBindings with `app=my-app` only +- Namespace-scoped resources in `ns-a`: global filters (no `namespacedFilterPolicies` in this example) + +**Partial overlay:** If `includedClusterScopedResources` lists only `clusterroles` and `clusterrolebindings`, StorageClasses are **not** backed up even if listed in `clusterScopedFilterPolicy` — global inclusion is evaluated first. + +**Differences from namespace policies:** + +- **Not** an allowlist — unlisted cluster-scoped kinds fall back to global filters. +- **No catch-all** — `kinds: []` or `kinds: ["*"]` is invalid and fails validation. + +--- + +### Example 13 — Global `includeExcludePolicy` and namespace filters + +**Goal:** Set a global resource-type baseline, then refine per namespace. Understand that global **exclusions** cannot be overridden per namespace. + +**Policy:** + +```yaml +version: v1 +includeExcludePolicy: + includedNamespaceScopedResources: + - configmaps + - secrets + - deployments + - services +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret] + labelSelector: + matchLabels: + app: my-app + - namespaces: + - production + resourceFilters: + - kinds: [ConfigMap] + names: ["app-*"] +``` + +**Expected outcome:** + +- **ns-a:** ConfigMaps and Secrets with `app=my-app` (within global allowlist) +- **production:** ConfigMaps matching `app-*` pattern +- **Other included namespaces:** Only kinds allowed by `includeExcludePolicy` (no per-namespace override) + +**Global exclusion wins (important):** + +```yaml +includeExcludePolicy: + excludedNamespaceScopedResources: + - secrets +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret, Deployment] + labelSelector: + matchLabels: + app: my-app +``` + +**Result:** No Secrets in the backup — the namespace policy cannot re-include a globally excluded kind. Velero logs a warning at backup start if you list an excluded kind in `namespacedFilterPolicies`. + +**Backup tip:** Do not set `includedResources` on the Backup; use `includeExcludePolicy` in the ConfigMap instead. + +--- + +### Example 14 — Volume policies and namespace filters together + +**Goal:** Use volume snapshot/fs-backup rules and namespace filters in one ConfigMap. + +**Policy:** + +```yaml +version: v1 +volumePolicies: + - conditions: + capacity: "0,10Gi" + storageClass: + - standard + action: + type: fs-backup + - conditions: + capacity: "10Gi,100Gi" + action: + type: snapshot +namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [ConfigMap] + names: ["app-*"] + excludedNames: ["*-tmp-*", "*-debug-*", "*-tmp", "*-debug"] + - kinds: [Secret] + labelSelector: + matchLabels: + workload: application +``` + +**Expected outcome:** Volume actions apply to PVCs per `volumePolicies`; resource inclusion follows `namespacedFilterPolicies`. The sections are independent. + +--- + +### Example 15 — `velero.io/exclude-from-backup=true` always wins + +**Goal:** Ensure explicitly excluded resources never appear in the backup, even when they match namespace filters or catch-all rules. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret] + labelSelector: + matchLabels: + app: my-app + - kinds: ["*"] + labelSelector: + matchLabels: + app: my-app +``` + +**On resources to exclude**, set: + +```yaml +metadata: + labels: + velero.io/exclude-from-backup: "true" +``` + +**Expected outcome:** Resources with `app=my-app` **and** `velero.io/exclude-from-backup=true` are excluded. Same rule applies to cluster-scoped resources refined by `clusterScopedFilterPolicy`. + +--- + +## Concepts reference + +### `resourceFilters` fields + +| Field | Description | +|-------|-------------| +| `kinds` | Resource type names (e.g. `ConfigMap`, `deployments`). Empty or `["*"]` = catch-all (namespace policies only). | +| `labelSelector` | Kubernetes-style selector with `matchLabels` and/or `matchExpressions` (`In`, `NotIn`, `Exists`, `DoesNotExist`). All requirements are AND-ed. | +| `orLabelSelectors` | List of selectors; match if **any** entry matches (AND within each, OR across the list). Use for OR of multi-key groups; prefer `In` for value-OR on one key. Mutually exclusive with `labelSelector`. | +| `names` | Exact names or glob patterns to include. | +| `excludedNames` | Patterns to exclude; wins over `names` when both match. | + +Only kinds listed in `resourceFilters` (or covered by catch-all) are collected from namespaces matched by `namespacedFilterPolicies`. + +### Glob pattern syntax + +Name and namespace patterns use the same glob style as elsewhere in Velero (`gobwas/glob`): + +- Supported: `*`, `?`, `[abc]`, `[a-z]` +- Not supported: `**`, regex, `|`, `()`, `!`, `{}`, `,` + +Examples: `app-*`, `team-frontend-*`, `*-tmp`. + +### Precedence cheat sheet + +**Namespaces** + +1. `BackupSpec.excludedNamespaces` — excluded namespaces are never backed up; namespace policies cannot override this. +2. `namespacedFilterPolicies` — first matching pattern (exact match checked before globs in pattern order). +3. No match — use global BackupSpec + `includeExcludePolicy`. + +**Namespace-scoped resources (when a namespace policy matches)** + +1. Global `includeExcludePolicy` exclusions (e.g. `excludedNamespaceScopedResources`) apply first. +2. Only kinds in `resourceFilters` (or catch-all) are allowlisted for collection. +3. Per-kind `labelSelector` / `orLabelSelectors` for API list calls. +4. Per-kind `names` / `excludedNames` at backup write time. +5. Label `velero.io/exclude-from-backup=true` always excludes. + +**Cluster-scoped resources** + +1. Must be allowed by `includedClusterScopedResources` / global cluster settings. +2. If `clusterScopedFilterPolicy` lists the kind, apply its label and name rules. +3. If not listed in `clusterScopedFilterPolicy`, use global BackupSpec filters. +4. `velero.io/exclude-from-backup=true` always excludes. + +```mermaid +flowchart TD + nsGlobal[BackupSpec namespace include/exclude] + nsPolicy{namespacedFilterPolicies match?} + nsAllow[Allowlist kinds + per-kind filters] + nsGlobalFallback[Global BackupSpec + includeExcludePolicy] + + nsGlobal --> nsPolicy + nsPolicy -->|yes| nsAllow + nsPolicy -->|no| nsGlobalFallback + + csInclude[includedClusterScopedResources] + csPolicy{kind in clusterScopedFilterPolicy?} + csRefine[Per-kind label and name rules] + csGlobal[Global cluster filters] + + csInclude --> csPolicy + csPolicy -->|yes| csRefine + csPolicy -->|no| csGlobal +``` + +### Catch-all summary + +| Rule | Detail | +|------|--------| +| Syntax | `kinds: ["*"]` or `kinds: []` | +| Count | At most one catch-all per `namespacedFilterPolicies` entry | +| Names | `names` / `excludedNames` not allowed on catch-all | +| Override | Kind-specific entries take precedence over catch-all | +| Label inheritance | Does not use `BackupSpec.labelSelector` | +| Cluster-scoped | Catch-all **not** supported in `clusterScopedFilterPolicy` | + +--- + +## Troubleshooting and validation + +### Verify a backup + +```bash +velero backup describe BACKUP_NAME +velero backup logs BACKUP_NAME +velero backup describe BACKUP_NAME -o json | jq '.namespacedFilterPolicies' +velero backup describe BACKUP_NAME -o json | jq '.clusterScopedFilterPolicy' +``` + +Catch-all entries appear as ` (all other kinds)` in text output, or `"isCatchAll": true` in JSON. + +### Common misconfigurations + +| Symptom | Likely cause | Fix | +|---------|----------------|-----| +| Fewer resources than expected in `team-frontend-prod` | Broad namespace pattern listed before specific one | Reorder policies: most specific `namespaces` first | +| Namespace policy lists Secrets but none in backup | `includeExcludePolicy` excludes `secrets` globally | Remove global exclusion or accept no Secrets | +| `ClusterRole` in namespace policy has no effect | Cluster-scoped kind in `namespacedFilterPolicies` | Move rule to `clusterScopedFilterPolicy`; check logs for warning | +| Backup fails at creation with filter message | Old-style `includedResources` with new policies | Move resource types to `includeExcludePolicy` in ConfigMap | +| Catch-all does not use backup-wide label | By design | Set `labelSelector` on the catch-all entry | +| Cluster-scoped policy validation error on `kinds: ["*"]` | Catch-all not allowed for cluster policy | List each cluster-scoped kind explicitly | + +### Velero logs + +```bash +kubectl logs -n velero deployment/velero | grep -i "namespacedFilterPolicies\|clusterScopedFilterPolicy" +kubectl logs -n velero deployment/velero | grep "globally excluded by includeExcludePolicy" +kubectl logs -n velero deployment/velero | grep "cluster-scoped" +``` + +### Validation errors (policy ConfigMap) + +Velero validates the ResourcePolicy when a backup starts. Common errors: + +| Error (summary) | Cause | +|-----------------|--------| +| `at least one namespace must be specified` | Empty `namespaces: []` | +| `at least one resourceFilter must be specified` | Empty `resourceFilters: []` | +| `names or excludedNames cannot be specified for catch-all filters` | Name patterns on catch-all entry | +| `only one catch-all resource filter is allowed` | Multiple catch-alls in one policy entry | +| `kind "X" appears in both resourceFilters[...]` | Same kind in two entries | +| `labelSelector and orLabelSelectors cannot co-exist` | Both set in one entry | +| `invalid label selector` | Bad operator, values, or label key/value syntax | +| `duplicate namespace pattern` | Same namespace string in two policy entries | +| `invalid glob pattern` | Bad characters in namespace or name pattern | +| `clusterScopedFilterPolicy... kinds must be specified (catch-all is not supported)` | Empty or `["*"]` kinds in cluster policy | +| `include-resources, exclude-resources... cannot be used with namespace-scoped or cluster-scoped global filter policies` | Old-style BackupSpec filters with new policy | + +### Silent edge cases (no error) + +- Namespace pattern matches no existing namespace — policy loaded but never applied. +- Kind listed but no instances in namespace — empty result, backup still succeeds. +- `excludedNames` narrows `names` — e.g. `names: ["app-*"]` + `excludedNames: ["app-config"]` excludes `app-config` only. + +--- + +## Restore behavior + +Restore is unchanged: it restores whatever is in the backup archive. Resources excluded by fine-grained filters are simply absent. Use `Restore.spec.includedNamespaces` (and existing restore filters) to limit what you restore from a partial backup. + +Fine-grained resource filtering is also available on the restore path using `namespacedFilterPolicies` and `clusterScopedFilterPolicy`. For details on the restore-side policies, see the [Fine-grained restore filters design](https://github.com/vmware-tanzu/velero/blob/main/design/restore-filter-enhancement/fine-grained-restore-filters-design.md). + +--- + +## Related links + +- [Fine-grained backup filters design](https://github.com/velero-io/velero/blob/main/design/backup-filter-enhancement/fine-grained-backup-filters-design.md) diff --git a/site/content/docs/v1.18/fine-grained-restore-filters.md b/site/content/docs/v1.18/fine-grained-restore-filters.md new file mode 100644 index 0000000000..0f7c52c263 --- /dev/null +++ b/site/content/docs/v1.18/fine-grained-restore-filters.md @@ -0,0 +1,726 @@ +--- +title: "Fine-Grained Restore Filters" +layout: docs +--- + +This guide explains how to use Velero's **fine-grained restore filters**: per-namespace, per-kind rules with independent label selectors and resource name patterns. Configuration lives in a **ResourcePolicy ConfigMap**, using the exact same format introduced for fine-grained backup filters. + +For architecture and pipeline details, see the [design document](https://github.com/velero-io/velero/blob/main/design/restore-filter-enhancement/fine-grained-restore-filters-design.md). + +--- + +## Introduction + +Velero's traditional restore filters apply the same namespace list, resource types, and label selector to every namespace being restored. Common scenarios need more control: + +- **Selective restore from a full backup** — restore only specific application components from a namespace, leaving out monitoring or logging resources that were also backed up. +- **Cross-environment migration** — restore StatefulSets and PVCs in a database namespace, but only Deployments and Services in a frontend namespace. +- **Filter by resource name** — restore `app-config` and `app-secret` without restoring `monitoring-config` from the same namespace. +- **Restore-time override** — apply different label selectors during restore than were used during backup to handle environment differences. + +Fine-grained filters add two optional sections to the ResourcePolicy ConfigMap: + +| Section | Scope | Behavior | +|---------|-------|----------| +| `namespacedFilterPolicies` | Namespaces you match (exact name or glob) | **Exclusive allowlist** — only resource kinds listed in `resourceFilters` (or covered by a catch-all) are restored for those namespaces, provided they pass global filters. | +| `clusterScopedFilterPolicy` | Cluster-scoped resources globally | **Refinement overlay** — listed kinds get per-kind label and name rules; unlisted cluster-scoped kinds still use global RestoreSpec filters. | + +**Backward compatible:** Fine-grained restore filters are optional. If a restore does not reference a ResourcePolicy, Velero relies solely on standard RestoreSpec filters (includedNamespaces, includedResources, labelSelector, etc.). + +--- + +## Prerequisites and wiring + +### What you need + +- A ResourcePolicy ConfigMap in the Velero namespace (`velero` by default). +- Permission to create Restores that reference the ConfigMap. + +### End-to-end pattern + +Every example below follows the same three steps: + +1. **Create or update** a ConfigMap with `data.policy` containing `version: v1` and your filter rules. +2. **Create a Restore** that includes the target namespaces and references the ConfigMap. +3. **Verify** with `velero restore describe` and inspect the restored resources. + +### Minimal skeleton + +Use this once; later examples show only the `policy:` body. + +**ResourcePolicy ConfigMap:** + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-restore-filter-policy + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + - namespaces: + - my-namespace + resourceFilters: + - kinds: [ConfigMap] + labelSelector: + matchLabels: + app: my-app +``` + +**Restore:** + +```yaml +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: my-restore + namespace: velero +spec: + backupName: my-backup + includedNamespaces: + - my-namespace + resourcePolicy: + kind: configmap + name: my-restore-filter-policy +``` + +**CLI equivalent:** + +```bash +velero restore create my-restore \ + --from-backup my-backup \ + --include-namespaces my-namespace \ + --resource-policies-configmap my-restore-filter-policy +``` + +**Verify:** + +```bash +velero restore describe my-restore +``` + +### Important: Interaction with Global Filters + +The restore pipeline evaluates **global resource filters first**: +- `RestoreSpec.IncludedResources` and `RestoreSpec.ExcludedResources` act as a global gate. +- A resource kind **must** pass the global gate before per-namespace filters are evaluated. +- **A namespace policy cannot re-include a globally excluded kind.** If you globally exclude `secrets`, listing `Secret` in a namespace policy will have no effect. + +--- + +## Examples + +Each example includes: **goal**, **policy YAML**, **restore notes**, and **expected outcome**. + +--- + +### Example 0 — Baseline (no new filters) + +**Goal:** Confirm that namespaces without a `namespacedFilterPolicies` entry still use global RestoreSpec filters. + +**Policy:** Omit `namespacedFilterPolicies` and `clusterScopedFilterPolicy` entirely. + +**Restore:** + +```yaml +spec: + includedNamespaces: + - ns-a + - ns-b + # No resourcePolicy — global filters only +``` + +**Expected outcome:** All resources in included namespaces follow `includedNamespaces`, `labelSelector`, `includedResources`, and related global fields — same as before this feature. + +--- + +### Example 1 — Per-namespace kinds and labels + +**Goal:** In `ns-a`, restore only ConfigMaps, Secrets, Deployments, and Pods with `app=my-app`. In `ns-b`, use global filters (no policy entry for that namespace). + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret, Deployment, Pod] + labelSelector: + matchLabels: + app: my-app +``` + +**Restore:** + +```yaml +spec: + includedNamespaces: + - ns-a + - ns-b + resourcePolicy: + kind: configmap + name: per-namespace-resource-filter-policy +``` + +**Expected outcome:** + +- **ns-a:** Only listed kinds with label `app=my-app` (e.g. `app-config`, `app-secret`, `app-deployment`). Resources like `monitoring-config` (different labels) are excluded. +- **ns-b:** Everything allowed by global filters (no namespace policy match). + +--- + +### Example 2 — Exact resource names + +**Goal:** Restore only two ConfigMaps by exact name, optionally requiring a label. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - target-namespace + resourceFilters: + - kinds: [ConfigMap] + names: [vm-1, vm-2] + labelSelector: + matchLabels: + resource-type: VirtualMachine +``` + +**Expected outcome:** Only `vm-1` and `vm-2` ConfigMaps with `resource-type=VirtualMachine` are restored. `vm-3` and other ConfigMaps are skipped. + +--- + +### Example 3 — Glob name patterns with exclusions + +**Goal:** Restore `app-*` ConfigMaps and Secrets in `production`, but exclude temporary and debug names. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [ConfigMap, Secret] + names: ["app-*"] + excludedNames: ["*-tmp-*", "*-debug-*", "*-tmp", "*-debug"] +``` + +**Expected outcome:** + +- **Included:** `app-config`, `app-cache-config`, `app-secret` +- **Excluded:** `app-config-tmp`, `app-tmp-config`, `app-debug-config`, `monitoring-tmp-secret` + +`excludedNames` takes precedence over `names` when both match. + +--- + +### Example 4 — Per-kind label selectors + +**Goal:** Apply different label rules to different resource types in the same namespace. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - target-namespace + resourceFilters: + - kinds: [ConfigMap] + orLabelSelectors: + - matchLabels: + app: production-workload-1 + component: vm-group + - matchLabels: + app: production-workload-2 + component: vm-service +``` + +**Expected outcome:** ConfigMaps matching either label combination are restored; other ConfigMaps in the namespace are not. + +**Note:** Prefer `matchExpressions` with `In` for value-OR on a single key (see next example). Use `orLabelSelectors` when you need OR across **independent multi-key groups**. `labelSelector` and `orLabelSelectors` cannot appear in the same `resourceFilters` entry. + +--- + +### Example 4b — Set-based label selectors (`matchExpressions`) + +**Goal:** Restore Deployments and Pods that are in `prod` or `staging`, belong to `app=my-app`, and do **not** carry a skip label. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [Deployment, Pod] + labelSelector: + matchLabels: + app: my-app + matchExpressions: + - key: environment + operator: In + values: [prod, staging] + - key: do-not-restore + operator: DoesNotExist +``` + +**Supported operators:** `In`, `NotIn`, `Exists`, `DoesNotExist` (same as Kubernetes / Velero global `--selector`). + +**Other useful patterns:** + +```yaml +# Exclude environments +matchExpressions: + - key: environment + operator: NotIn + values: [dev, test] + +# Require a label key to be present (any value) +matchExpressions: + - key: tier + operator: Exists +``` + +**Expected outcome:** Only Deployments/Pods with `app=my-app`, `environment` in `{prod, staging}`, and without `do-not-restore` are restored. + +--- + +### Example 5 — OR label selectors across kinds + +**Goal:** Restore ConfigMaps, Secrets, or Deployments that match any of several label conditions. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret] + orLabelSelectors: + - matchLabels: + app: my-app + - matchLabels: + app: monitoring + - kinds: [Deployment] + orLabelSelectors: + - matchLabels: + app: my-app + - matchLabels: + app: monitoring + - matchLabels: + component: backend +``` + +**Expected outcome:** Resources included if they match **any** selector in `orLabelSelectors` for their kind (AND within each selector, OR across the list). + +--- + +### Example 6 — Multiple criteria on one kind + +**Goal:** Combine exact names with OR label selectors for a single kind. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - target-namespace + resourceFilters: + - kinds: [ConfigMap] + names: [vm-1, vm-2] + orLabelSelectors: + - matchLabels: + resource-type: VirtualMachine + - matchLabels: + component: vm-group + - matchLabels: + component: vm-service +``` + +**Expected outcome:** Only `vm-1` and `vm-2` that also satisfy one of the label OR branches. + +--- + +### Example 7 — One policy entry, multiple namespaces + +**Goal:** Apply the same rules to `ns-a`, `ns-b`, and `production` in a single policy block. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + - ns-b + - production + resourceFilters: + - kinds: [ConfigMap] + - kinds: [Deployment] + labelSelector: + matchLabels: + tier: web +``` + +**Expected outcome:** + +- All ConfigMaps in those namespaces (no label filter on that entry). +- Deployments with `tier=web` only. + +--- + +### Example 8 — Namespace glob patterns and ordering + +**Goal:** Different restore breadth for `team-frontend-prod`, `team-frontend-dev`, and `team-backend-test` using glob patterns. + +**Note on Precedence:** Exact namespace matches always take precedence regardless of where they are listed. However, if multiple glob patterns could match a namespace, they are evaluated in the order they appear. Always list specific globs before broad globs. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + # Globs must be ordered specific-to-broad + - namespaces: + - "team-frontend-*" # specific pattern match + resourceFilters: + - kinds: [Deployment, Service, ConfigMap] + - namespaces: + - "team-*" # broad pattern + resourceFilters: + - kinds: [Deployment, Service] + + # Exact matches always win, even if placed at the bottom + - namespaces: + - team-frontend-prod # exact match + resourceFilters: + - kinds: [Deployment, Service, ConfigMap, Secret, PersistentVolumeClaim] +``` + +**Expected outcome:** + +| Namespace | Matched policy | Kinds restored | +|-----------|----------------|-----------------| +| `team-frontend-prod` | `team-frontend-prod` (Exact match priority) | 5 kinds | +| `team-frontend-dev` | `team-frontend-*` (First matching glob) | 3 kinds | +| `team-backend-test` | `team-*` (First matching glob) | 2 kinds | + +Velero uses **first-match** semantics: the first policy entry whose namespace pattern matches wins. + +--- + +### Example 9 — Catch-all by label + +**Goal:** Restore any resource kind that has a given label, without listing every kind. Kind-specific entries override the catch-all. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: ["*"] # catch-all + labelSelector: + matchLabels: + app: common-app + - kinds: [ConfigMap, Secret] # override for these kinds + labelSelector: + matchLabels: + app: specialized-app +``` + +**Rules:** + +- At most **one** catch-all per namespace policy entry. +- Catch-all entries **cannot** use `names` or `excludedNames`. +- Catch-all does **not** inherit `RestoreSpec.LabelSelector`. + +**Expected outcome:** ConfigMaps and Secrets use `app=specialized-app`; all other kinds listed only via catch-all use `app=common-app`. + +--- + +### Example 10 — Catch-all with per-kind name overrides + +**Goal:** Pin critical Deployments and Secrets by exact name; restore everything else with a label convention. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [Deployment] + names: [api-server, worker] + - kinds: [Secret] + names: [db-credentials, tls-cert] + - kinds: ["*"] + labelSelector: + matchLabels: + restore: "true" +``` + +**Expected outcome:** + +- Deployments: only `api-server` and `worker` +- Secrets: only `db-credentials` and `tls-cert` +- Other kinds (ConfigMap, Service, …): resources with `restore=true` only + +--- + +### Example 11 — Override-only catch-all (no label on catch-all) + +**Goal:** Apply a strict name filter to one kind while restoring all other kinds without listing them or adding labels. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [Secret] + names: [app-secret] + - kinds: ["*"] # no labelSelector — all other kinds included +``` + +**Expected outcome:** + +- Secrets: only `app-secret` +- Other kinds in `ns-a`: all instances restored (subject to global filters) + +--- + +### Example 12 — Cluster-scoped refinement + +**Goal:** Refine which cluster-scoped resources are restored by name and label, without replacing global cluster-scoped inclusion. + +**Policy:** + +```yaml +version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: [StorageClass] + names: ["my-app-*"] + - kinds: [ClusterRole, ClusterRoleBinding] + labelSelector: + matchLabels: + app: my-app +``` + +**Restore (required):** You must still include cluster-scoped kinds on the Restore: + +```yaml +spec: + includeClusterResources: true + resourcePolicy: + kind: configmap + name: cluster-scoped-filter-policy +``` + +**Expected outcome:** + +- StorageClasses matching `my-app-*` only +- ClusterRoles and ClusterRoleBindings with `app=my-app` only +- Other cluster-scoped resources: restored according to global filters. + +**Differences from namespace policies:** + +- **Not** an allowlist — unlisted cluster-scoped kinds fall back to global filters. +- **No catch-all** — `kinds: []` or `kinds: ["*"]` is invalid and fails validation. + +--- + +### Example 13 — Global `ExcludedResources` and namespace filters + +**Goal:** Understand that global **exclusions** cannot be overridden per namespace. + +**Restore:** +```yaml +spec: + excludedResources: + - secrets +``` + +**Policy:** +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret, Deployment] + labelSelector: + matchLabels: + app: my-app +``` + +**Result:** No Secrets are restored — the namespace policy cannot re-include a globally excluded kind. Velero logs a warning at restore start if you list an excluded kind in `namespacedFilterPolicies`. + +--- + +### Example 14 — Separate ConfigMaps for Backup and Restore + +**Goal:** Understand why you cannot use a single ConfigMap for both backup and restore operations if it contains backup-specific policies. + +**Policy:** + +```yaml +version: v1 +volumePolicies: + - conditions: + capacity: "0,10Gi" + action: + type: fs-backup +namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [ConfigMap, Secret] + names: ["app-*"] +``` + +**Expected outcome:** The restore operation will **fail validation**. The Velero restore pipeline strictly rejects any ResourcePolicy ConfigMap containing `volumePolicies` or `includeExcludePolicy`. To avoid this, the restore-side ConfigMap should contain only the restore-supported sections (`namespacedFilterPolicies` and/or `clusterScopedFilterPolicy`). + +--- + +### Example 15 — `velero.io/exclude-from-backup=true` always wins + +**Goal:** Ensure explicitly excluded resources never appear in the restore. + +If a resource was backed up (perhaps before the label was added, or manually modified in the archive) but has `velero.io/exclude-from-backup: "true"`, the restore pipeline honors it. Any item carrying this label is skipped regardless of whether it matches global or per-namespace restore filters. + +--- + +## Concepts reference + +### `resourceFilters` fields + +| Field | Description | +|-------|-------------| +| `kinds` | Resource type names (e.g. `ConfigMap`, `deployments`). Empty or `["*"]` = catch-all (namespace policies only). | +| `labelSelector` | Kubernetes-style selector with `matchLabels` and/or `matchExpressions` (`In`, `NotIn`, `Exists`, `DoesNotExist`). All requirements are AND-ed. | +| `orLabelSelectors` | List of selectors; match if **any** entry matches (AND within each, OR across the list). Use for OR of multi-key groups; prefer `In` for value-OR on one key. Mutually exclusive with `labelSelector`. | +| `names` | Exact names or glob patterns to include. | +| `excludedNames` | Patterns to exclude; wins over `names` when both match. | + +### Glob pattern syntax + +Name and namespace patterns use the same glob style as elsewhere in Velero (`gobwas/glob`): + +- Supported: `*`, `?`, `[abc]`, `[a-z]` +- Not supported: `**`, regex, `|`, `()`, `!`, `{}`, `,` + +Examples: `app-*`, `team-frontend-*`, `*-tmp`. + +### Precedence cheat sheet + +**Namespaces** + +1. `RestoreSpec.ExcludedNamespaces` — excluded namespaces are never restored. +2. `namespacedFilterPolicies` — first matching pattern (exact match checked before globs in pattern order). +3. No match — use global RestoreSpec filters. + +**Namespace-scoped resources (when a namespace policy matches)** + +1. Global `RestoreSpec.IncludedResources` / `ExcludedResources` apply first. +2. Only kinds in `resourceFilters` (or catch-all) are allowlisted for restoration. +3. Per-kind `labelSelector` / `orLabelSelectors` replace global selectors. +4. Per-kind `names` / `excludedNames` filter by resource name. +5. Label `velero.io/exclude-from-backup=true` always excludes. +6. **Plugin Additional Items** bypass fine-grained filters to ensure dependencies (like PVs) are restored. + +**Cluster-scoped resources** + +1. Must be allowed by global cluster settings (`includeClusterResources`). +2. If `clusterScopedFilterPolicy` lists the kind, apply its label and name rules. +3. If not listed in `clusterScopedFilterPolicy`, use global RestoreSpec filters. +4. `velero.io/exclude-from-backup=true` always excludes. + +### Catch-all summary + +| Rule | Detail | +|------|--------| +| Syntax | `kinds: ["*"]` or `kinds: []` | +| Count | At most one catch-all per `namespacedFilterPolicies` entry | +| Names | `names` / `excludedNames` not allowed on catch-all | +| Override | Kind-specific entries take precedence over catch-all | +| Label inheritance | Does not use `RestoreSpec.LabelSelector` | +| Cluster-scoped | Catch-all **not** supported in `clusterScopedFilterPolicy` | + +--- + +## Troubleshooting and validation + +### Verify a restore + +```bash +velero restore describe RESTORE_NAME +velero restore logs RESTORE_NAME +``` + +The output of `velero restore describe` will show the `Resource Policy` field if a ConfigMap was used. + +### Common misconfigurations + +| Symptom | Likely cause | Fix | +|---------|----------------|-----| +| Fewer resources than expected in `team-frontend-prod` | Broad namespace pattern listed before specific one | Reorder policies: most specific `namespaces` first | +| Namespace policy lists Secrets but none restored | `RestoreSpec.ExcludedResources` excludes `secrets` globally | Remove global exclusion or accept no Secrets | +| `ClusterRole` in namespace policy has no effect | Cluster-scoped kind in `namespacedFilterPolicies` | Move rule to `clusterScopedFilterPolicy`; check logs for warning | +| Catch-all does not use restore-wide label | By design | Set `labelSelector` on the catch-all entry | +| Cluster-scoped policy validation error on `kinds: ["*"]` | Catch-all not allowed for cluster policy | List each cluster-scoped kind explicitly | + +### Velero logs + +```bash +kubectl logs -n velero deployment/velero | grep -i "namespacedFilterPolicies\|clusterScopedFilterPolicy" +kubectl logs -n velero deployment/velero | grep "globally excluded by RestoreSpec.ExcludedResources" +``` + +### Validation errors (policy ConfigMap) + +Velero validates the ResourcePolicy when a restore starts. Common errors: + +| Error (summary) | Cause | +|-----------------|--------| +| `at least one namespace must be specified` | Empty `namespaces: []` | +| `at least one resourceFilter must be specified` | Empty `resourceFilters: []` | +| `names or excludedNames cannot be specified for catch-all filters` | Name patterns on catch-all entry | +| `only one catch-all resource filter is allowed` | Multiple catch-alls in one policy entry | +| `kind "X" appears in both resourceFilters[...]` | Same kind in two entries | +| `labelSelector and orLabelSelectors cannot co-exist` | Both set in one entry | +| `invalid label selector` | Bad operator, values, or label key/value syntax | +| `duplicate namespace pattern` | Same namespace string in two policy entries | +| `invalid glob pattern` | Bad characters in namespace or name pattern | +| `clusterScopedFilterPolicy... kinds must be specified (catch-all is not supported)` | Empty or `["*"]` kinds in cluster policy | + +### Silent edge cases (no error) + +- Namespace pattern matches no existing namespace in the backup — policy loaded but never applied. +- Kind listed but no instances in namespace — empty result, restore still succeeds. +- `excludedNames` narrows `names` — e.g. `names: ["app-*"]` + `excludedNames: ["app-config"]` excludes `app-config` only. + +--- + +## Related links + +- [Fine-grained restore filters design](https://github.com/velero-io/velero/blob/main/design/restore-filter-enhancement/fine-grained-restore-filters-design.md) diff --git a/site/content/docs/v1.18/resource-filtering.md b/site/content/docs/v1.18/resource-filtering.md index 69b2cb5e1c..304f1083ef 100644 --- a/site/content/docs/v1.18/resource-filtering.md +++ b/site/content/docs/v1.18/resource-filtering.md @@ -5,8 +5,8 @@ layout: docs *Filter objects by namespace, type, labels or resource policies.* -This page describes how to filter resource for backup and restore. -User could use the include and exclude flags with the `velero backup` and `velero restore` commands. And user could also use resource policies to handle backup. +This page describes how to filter resources for backup and restore. +Users can use include and exclude flags with the `velero backup` and `velero restore` commands. Users can also use resource policies for fine-grained resource filtering during backup and restore, as well as volume handling during backup. By default, Velero includes all objects in a backup or restore when no filtering options are used. ## Includes @@ -229,99 +229,139 @@ Kubernetes namespace resources to exclude from the backup, formatted as resource ``` ## Resource policies -Velero provides resource policies to filter resources to do backup, which may contain `includeExcludePolicy` and `volumePolicies`. -### Creating resource policies +Velero provides resource policies (defined in a ConfigMap and referenced via `--resource-policies-configmap` or `spec.resourcePolicy`) to define fine-grained resource filters and volume handling rules. -Below is the two-step of using resource policies in backup: -1. Creating resource policies configmap +Resource policies support both **Backup** and **Restore** operations, though certain policy sections are specific to backup workflows. - Users need to create one configmap in Velero install namespace from a YAML file that defined resource policies. The creating command would be like the below: +### Supported policy sections by operation + +| Policy Section | Description | Supported Operations | Learn More | +| --- | --- | --- | --- | +| `namespacedFilterPolicies` | Fine-grained per-namespace and per-kind filters with label selectors and resource name patterns. | **Backup** & **Restore** | [Fine-Grained Backup Filters](fine-grained-backup-filters.md) / [Fine-Grained Restore Filters](fine-grained-restore-filters.md) | +| `clusterScopedFilterPolicy` | Fine-grained cluster-scoped filter overlays with per-kind label selectors and resource name patterns. | **Backup** & **Restore** | [Fine-Grained Backup Filters](fine-grained-backup-filters.md) / [Fine-Grained Restore Filters](fine-grained-restore-filters.md) | +| `volumePolicies` | Rules to control volume data backup methods (`skip`, `snapshot`, `fs-backup`) based on conditions. | **Backup** only | See [VolumePolicy](#volumepolicy-backup-only) | +| `includeExcludePolicy` | Reusable scoped resource include/exclude filters. | **Backup** only | See [IncludeExcludePolicy](#includeexcludepolicy-backup-only) | + +### Creating and referencing resource policies + +Using resource policies is a two-step process: + +1. **Create the resource policies ConfigMap** + + Create a ConfigMap in the Velero installation namespace (typically `velero`) containing your YAML policy definition: ```bash kubectl create cm --from-file -n velero ``` -2. Creating a backup reference to the defined resource policies - Users create a backup with the flag `--resource-policies-configmap`, which will reference the current backup to the defined resource policies. The creating command would be like the below: - ```bash - velero backup create --resource-policies-configmap - ``` - This flag could also be combined with the other include and exclude filters above +2. **Reference the resource policies ConfigMap in a Backup or Restore** + + * **For Backup:** Reference the ConfigMap via CLI flag or in the Backup CR spec: + ```bash + velero backup create --resource-policies-configmap + ``` + Or in `Backup.spec`: + ```yaml + spec: + resourcePolicy: + kind: ConfigMap + name: + ``` + + * **For Restore:** Reference the ConfigMap via CLI flag or in the Restore CR spec: + ```bash + velero restore create --from-backup --resource-policies-configmap + ``` + Or in `Restore.spec`: + ```yaml + spec: + resourcePolicy: + kind: ConfigMap + name: + ``` + + These flags and fields can also be combined with standard include and exclude options. ### YAML template -The policies YAML config file would look like this: -- Yaml template: - ```yaml - # currently only supports v1 version - version: v1 - # The filters in includeExcludePolicy work the same as the scoped resources filters in the Spec of a Backup - # NOTE: similar to scoped filters in Backup Spec, the includeExcludePolicy does not work with --include-resources, --exclude-resources and --include-cluster-resources filters in Backup. - includeExcludePolicy: - includedClusterScopedResources: - - "crd" - - "pv" - excludedClusterScopedResources: [] - includedNamespaceScopedResources: - - "pod" - - "service" - - "deployment" - - "pvc" - excludedNamespaceScopedResources: - - "configmap" - - "secret" - volumePolicies: - # each policy consists of a list of conditions and an action - # we could have lots of policies, but if the resource matched the first policy, the latter will be ignored - # each key in the object is one condition, and one policy will apply to resources that meet ALL conditions - # NOTE: capacity or storageClass is suited for [Persistent Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes), and pod [Volume](https://kubernetes.io/docs/concepts/storage/volumes) not support it. - - conditions: - # capacity condition matches the volumes whose capacity falls into the range - capacity: "10,100Gi" - # pv matches specific csi driver - csi: - driver: ebs.csi.aws.com - # pv matches one of the storage class list - storageClass: - - gp2 - - standard - # pvc matches specific phase(s) - pvcPhase: - - Pending - action: - type: skip - - conditions: - capacity: "0,100Gi" - # nfs volume source with specific server and path (nfs could be empty or only config server or path) - nfs: - server: 192.168.200.90 - path: /mnt/data - action: - type: skip - - conditions: - nfs: - server: 192.168.200.90 - action: - type: fs-backup - - conditions: - # nfs could be empty which matches any nfs volume source - nfs: {} - action: - type: skip - - conditions: - # csi could be empty which matches any csi volume source - csi: {} - action: - type: snapshot - - conditions: - volumeTypes: - - emptyDir - - downwardAPI - - configmap - - cinder - action: - type: skip - ``` -### IncludeExcludePolicy + +The policies YAML config file showing all supported sections: + +```yaml +# Currently supports v1 version +version: v1 + +# Fine-grained namespace-scoped filters (Supported for both Backup and Restore) +namespacedFilterPolicies: + - namespace: "app-ns-*" + resourceFilters: + - kind: "deployment" + labelSelector: + matchLabels: + app: frontend + includedResourceNames: + - "web-*" + - kind: "secret" + excludedResourceNames: + - "sensitive-secret" + +# Fine-grained cluster-scoped filter overlay (Supported for both Backup and Restore) +clusterScopedFilterPolicy: + resourceFilters: + - kind: "storageclass" + labelSelector: + matchLabels: + tier: gold + +# Volume handling policies (Supported for Backup ONLY) +volumePolicies: + - conditions: + capacity: "10,100Gi" + csi: + driver: ebs.csi.aws.com + storageClass: + - gp2 + - standard + pvcPhase: + - Pending + pvcVolumeMode: Block + pvcAccessModes: + - ReadWriteOnce + action: + type: skip + - conditions: + nfs: {} + action: + type: fs-backup + +# Legacy scoped resource include/exclude filters (Supported for Backup ONLY) +# NOTE: Cannot be combined with --include-resources, --exclude-resources, or --include-cluster-resources in Backup. +includeExcludePolicy: + includedClusterScopedResources: + - "crd" + - "pv" + excludedClusterScopedResources: [] + includedNamespaceScopedResources: + - "pod" + - "service" + - "deployment" + - "pvc" + excludedNamespaceScopedResources: + - "configmap" + - "secret" +``` + +### Fine-grained backup and restore filters + +`namespacedFilterPolicies` and `clusterScopedFilterPolicy` allow defining per-namespace and per-kind rules with independent label selectors and resource name patterns. + +* **During Backup:** Controls which resources are backed up from matching namespaces or kinds. +* **During Restore:** Controls which resources are restored from a backup archive without modifying the backup itself. + +For comprehensive guides, syntax details, and detailed examples, see: +* [Fine-Grained Backup Filters](fine-grained-backup-filters.md) +* [Fine-Grained Restore Filters](fine-grained-restore-filters.md) + +### IncludeExcludePolicy (Backup only) The `includeExcludePolicy` is used to filter resources based on the namespace-scoped and cluster-scoped resources. User can use it to define a group of filters and reuse them across different backups. @@ -360,7 +400,7 @@ velero backup create --resource-policies-configmap my-policy --inc The backup will include all resources in namespace `my-workload-ns`, including `configmap` and `event`, and all CRDs and `apiservices` in the cluster. -### VolumePolicy +### VolumePolicy (Backup only) VolumePolicy is a data structure to control how velero handle the volumes matching certain conditions. #### Supported VolumePolicy actions diff --git a/site/data/docs/v1-18-toc.yml b/site/data/docs/v1-18-toc.yml index dacec06518..08ec47e81b 100644 --- a/site/data/docs/v1-18-toc.yml +++ b/site/data/docs/v1-18-toc.yml @@ -33,6 +33,10 @@ toc: url: /enable-api-group-versions-feature - page: Resource filtering url: /resource-filtering + - page: Fine-Grained Backup Filters + url: /fine-grained-backup-filters + - page: Fine-grained restore filters + url: /fine-grained-restore-filters - page: Namespace glob patterns url: /namespace-glob-patterns - page: Backup reference diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index 54388b70a2..06a7de23de 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -22,10 +22,10 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" . "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test/util/k8s" diff --git a/test/e2e/basic/api-group/enable_api_group_versions.go b/test/e2e/basic/api-group/enable_api_group_versions.go index 13ee3a39e1..264e6f2ba6 100644 --- a/test/e2e/basic/api-group/enable_api_group_versions.go +++ b/test/e2e/basic/api-group/enable_api_group_versions.go @@ -27,10 +27,10 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/test/e2e/basic/backup-volume-info/base.go b/test/e2e/basic/backup-volume-info/base.go index 2cd574b5c3..7a1ff39e12 100644 --- a/test/e2e/basic/backup-volume-info/base.go +++ b/test/e2e/basic/backup-volume-info/base.go @@ -22,9 +22,9 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" . "github.com/vmware-tanzu/velero/test" diff --git a/test/e2e/basic/resources-check/namespaces.go b/test/e2e/basic/resources-check/namespaces.go index 922e4ed07b..6b1a577f4e 100644 --- a/test/e2e/basic/resources-check/namespaces.go +++ b/test/e2e/basic/resources-check/namespaces.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/vmware-tanzu/velero/test/e2e/test" diff --git a/test/e2e/basic/resources-check/namespaces_annotation.go b/test/e2e/basic/resources-check/namespaces_annotation.go index e698d48185..fd8012127f 100644 --- a/test/e2e/basic/resources-check/namespaces_annotation.go +++ b/test/e2e/basic/resources-check/namespaces_annotation.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" . "github.com/vmware-tanzu/velero/test/e2e/test" . "github.com/vmware-tanzu/velero/test/util/k8s" diff --git a/test/e2e/basic/resources-check/rbac.go b/test/e2e/basic/resources-check/rbac.go index b79c3615f4..0d40f00e61 100644 --- a/test/e2e/basic/resources-check/rbac.go +++ b/test/e2e/basic/resources-check/rbac.go @@ -36,8 +36,8 @@ import ( "fmt" "strings" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" - "github.com/pkg/errors" . "github.com/vmware-tanzu/velero/test/e2e/test" . "github.com/vmware-tanzu/velero/test/util/k8s" diff --git a/test/e2e/nodeagentconfig/node-agent-config.go b/test/e2e/nodeagentconfig/node-agent-config.go index dc88d98bdb..ce50cf1e49 100644 --- a/test/e2e/nodeagentconfig/node-agent-config.go +++ b/test/e2e/nodeagentconfig/node-agent-config.go @@ -23,8 +23,8 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" . "github.com/onsi/gomega" - "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/test/e2e/pv-backup/pv-backup-filter.go b/test/e2e/pv-backup/pv-backup-filter.go index 5a6730551e..510c686db8 100644 --- a/test/e2e/pv-backup/pv-backup-filter.go +++ b/test/e2e/pv-backup/pv-backup-filter.go @@ -6,9 +6,9 @@ import ( "strings" "unicode" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" . "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test/e2e/test" diff --git a/test/e2e/repomaintenance/repo_maintenance_config.go b/test/e2e/repomaintenance/repo_maintenance_config.go index c0092c4bf4..7c53655b53 100644 --- a/test/e2e/repomaintenance/repo_maintenance_config.go +++ b/test/e2e/repomaintenance/repo_maintenance_config.go @@ -22,8 +22,8 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" . "github.com/onsi/gomega" - "github.com/pkg/errors" batchv1api "k8s.io/api/batch/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/test/e2e/resource-filtering/base.go b/test/e2e/resource-filtering/base.go index f4070d9e73..e36de71455 100644 --- a/test/e2e/resource-filtering/base.go +++ b/test/e2e/resource-filtering/base.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/vmware-tanzu/velero/test/e2e/test" diff --git a/test/e2e/resource-filtering/exclude_label.go b/test/e2e/resource-filtering/exclude_label.go index 695d7d8ed3..6cfd2d0305 100644 --- a/test/e2e/resource-filtering/exclude_label.go +++ b/test/e2e/resource-filtering/exclude_label.go @@ -19,9 +19,9 @@ package filtering import ( "fmt" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/test/e2e/resource-filtering/exclude_namespaces.go b/test/e2e/resource-filtering/exclude_namespaces.go index 1b8e5da550..b90caa6e21 100644 --- a/test/e2e/resource-filtering/exclude_namespaces.go +++ b/test/e2e/resource-filtering/exclude_namespaces.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" . "github.com/vmware-tanzu/velero/test/e2e/test" diff --git a/test/e2e/resource-filtering/exclude_resources.go b/test/e2e/resource-filtering/exclude_resources.go index b8a7d2e73f..a2346f0af7 100644 --- a/test/e2e/resource-filtering/exclude_resources.go +++ b/test/e2e/resource-filtering/exclude_resources.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/test/e2e/resource-filtering/include_namespaces.go b/test/e2e/resource-filtering/include_namespaces.go index d511de2123..538473db7f 100644 --- a/test/e2e/resource-filtering/include_namespaces.go +++ b/test/e2e/resource-filtering/include_namespaces.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" . "github.com/vmware-tanzu/velero/test/e2e/test" diff --git a/test/e2e/resource-filtering/include_resources.go b/test/e2e/resource-filtering/include_resources.go index 22d7e968b2..593efcc247 100644 --- a/test/e2e/resource-filtering/include_resources.go +++ b/test/e2e/resource-filtering/include_resources.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/test/e2e/resource-filtering/label_selector.go b/test/e2e/resource-filtering/label_selector.go index 9ecc66a0ce..75d013b59f 100644 --- a/test/e2e/resource-filtering/label_selector.go +++ b/test/e2e/resource-filtering/label_selector.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/test/e2e/resourcemodifiers/resource_modifiers.go b/test/e2e/resourcemodifiers/resource_modifiers.go index 06cbd91f15..e5efa8a06c 100644 --- a/test/e2e/resourcemodifiers/resource_modifiers.go +++ b/test/e2e/resourcemodifiers/resource_modifiers.go @@ -20,9 +20,9 @@ import ( "fmt" "strings" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" . "github.com/vmware-tanzu/velero/test/e2e/test" "github.com/vmware-tanzu/velero/test/util/common" diff --git a/test/e2e/resourcepolicies/resource_policies.go b/test/e2e/resourcepolicies/resource_policies.go index f3254eb041..306fa6c717 100644 --- a/test/e2e/resourcepolicies/resource_policies.go +++ b/test/e2e/resourcepolicies/resource_policies.go @@ -21,9 +21,9 @@ import ( "strings" "unicode" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" diff --git a/test/e2e/schedule/ordered_resources.go b/test/e2e/schedule/ordered_resources.go index df0d8b9725..8e5852f832 100644 --- a/test/e2e/schedule/ordered_resources.go +++ b/test/e2e/schedule/ordered_resources.go @@ -23,9 +23,9 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" "k8s.io/apimachinery/pkg/labels" waitutil "k8s.io/apimachinery/pkg/util/wait" kbclient "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index 1a91c115f4..57890d6791 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -23,9 +23,9 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" . "github.com/vmware-tanzu/velero/test" diff --git a/test/perf/basic/basic.go b/test/perf/basic/basic.go index 76bf605a68..6aa2bab0ef 100644 --- a/test/perf/basic/basic.go +++ b/test/perf/basic/basic.go @@ -21,7 +21,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" . "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test/perf/test" diff --git a/test/perf/e2e_suite_test.go b/test/perf/e2e_suite_test.go index 48a5ceec9e..e0c4751ca5 100644 --- a/test/perf/e2e_suite_test.go +++ b/test/perf/e2e_suite_test.go @@ -23,9 +23,9 @@ import ( "testing" "time" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" . "github.com/vmware-tanzu/velero/test" diff --git a/test/perf/metrics/minio.go b/test/perf/metrics/minio.go index 8ea7ae4c33..d4cf968b93 100644 --- a/test/perf/metrics/minio.go +++ b/test/perf/metrics/minio.go @@ -17,7 +17,7 @@ limitations under the License. package metrics import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/test/util/metrics" ) diff --git a/test/perf/metrics/nfs.go b/test/perf/metrics/nfs.go index 043a4f976d..a0cfb689d0 100644 --- a/test/perf/metrics/nfs.go +++ b/test/perf/metrics/nfs.go @@ -19,7 +19,7 @@ package metrics import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/test/util/metrics" ) diff --git a/test/perf/metrics/pod.go b/test/perf/metrics/pod.go index 56572f6728..78908d3467 100644 --- a/test/perf/metrics/pod.go +++ b/test/perf/metrics/pod.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" metricsclientset "k8s.io/metrics/pkg/client/clientset/versioned" diff --git a/test/perf/restore/restore.go b/test/perf/restore/restore.go index 6adbff5f4e..8fa6660444 100644 --- a/test/perf/restore/restore.go +++ b/test/perf/restore/restore.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" . "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test/perf/test" diff --git a/test/perf/test/test.go b/test/perf/test/test.go index 1bc8a3fc06..5716e5b118 100644 --- a/test/perf/test/test.go +++ b/test/perf/test/test.go @@ -22,9 +22,9 @@ import ( "math/rand" "time" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" . "github.com/vmware-tanzu/velero/test" diff --git a/test/pkg/client/client.go b/test/pkg/client/client.go index 331b786f26..142683970c 100644 --- a/test/pkg/client/client.go +++ b/test/pkg/client/client.go @@ -20,7 +20,7 @@ import ( "fmt" "runtime" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" diff --git a/test/pkg/client/config.go b/test/pkg/client/config.go index 687c303e7b..2a96e3467b 100644 --- a/test/pkg/client/config.go +++ b/test/pkg/client/config.go @@ -23,7 +23,7 @@ import ( "strconv" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" ) const ( diff --git a/test/pkg/client/factory.go b/test/pkg/client/factory.go index 340cba587f..9691c3492b 100644 --- a/test/pkg/client/factory.go +++ b/test/pkg/client/factory.go @@ -24,7 +24,7 @@ import ( k8scheme "k8s.io/client-go/kubernetes/scheme" kbclient "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" diff --git a/test/util/csi/common.go b/test/util/csi/common.go index 373bc1502d..b7c80732d7 100644 --- a/test/util/csi/common.go +++ b/test/util/csi/common.go @@ -21,9 +21,9 @@ import ( "fmt" "strings" + "github.com/cockroachdb/errors" volumeSnapshotV1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned" - "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" diff --git a/test/util/k8s/common.go b/test/util/k8s/common.go index 8869caab34..40c37c12ed 100644 --- a/test/util/k8s/common.go +++ b/test/util/k8s/common.go @@ -25,7 +25,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" diff --git a/test/util/k8s/configmap.go b/test/util/k8s/configmap.go index 39bcb0907e..665f7b2a11 100644 --- a/test/util/k8s/configmap.go +++ b/test/util/k8s/configmap.go @@ -22,7 +22,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/test/util/k8s/crd.go b/test/util/k8s/crd.go index fe17fb0ae2..a7a63f0f1e 100644 --- a/test/util/k8s/crd.go +++ b/test/util/k8s/crd.go @@ -24,7 +24,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) diff --git a/test/util/k8s/namespace.go b/test/util/k8s/namespace.go index b46075fee2..557782b996 100644 --- a/test/util/k8s/namespace.go +++ b/test/util/k8s/namespace.go @@ -24,7 +24,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/test/util/k8s/persistentvolumes.go b/test/util/k8s/persistentvolumes.go index aaa6b9ea28..7860e73a2a 100644 --- a/test/util/k8s/persistentvolumes.go +++ b/test/util/k8s/persistentvolumes.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/test/util/k8s/pod.go b/test/util/k8s/pod.go index 9906e08b53..718beab980 100644 --- a/test/util/k8s/pod.go +++ b/test/util/k8s/pod.go @@ -22,7 +22,7 @@ import ( "fmt" "path" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" diff --git a/test/util/k8s/rbac.go b/test/util/k8s/rbac.go index b660a58d7e..82b3ad200b 100644 --- a/test/util/k8s/rbac.go +++ b/test/util/k8s/rbac.go @@ -21,7 +21,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/test/util/k8s/sc.go b/test/util/k8s/sc.go index e6cd8e3b1c..0d8e777aca 100644 --- a/test/util/k8s/sc.go +++ b/test/util/k8s/sc.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) diff --git a/test/util/k8s/secret.go b/test/util/k8s/secret.go index ea02f51d0f..14b94feb87 100644 --- a/test/util/k8s/secret.go +++ b/test/util/k8s/secret.go @@ -22,7 +22,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/test/util/k8s/service.go b/test/util/k8s/service.go index e8cd098e17..a54df3eb67 100644 --- a/test/util/k8s/service.go +++ b/test/util/k8s/service.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/test/util/k8s/serviceaccount.go b/test/util/k8s/serviceaccount.go index 31773d8466..1658a8b5ae 100644 --- a/test/util/k8s/serviceaccount.go +++ b/test/util/k8s/serviceaccount.go @@ -22,7 +22,7 @@ import ( "os" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/test/util/k8s/statefulset.go b/test/util/k8s/statefulset.go index f0ac3a6513..027fbd9ae6 100644 --- a/test/util/k8s/statefulset.go +++ b/test/util/k8s/statefulset.go @@ -22,7 +22,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go index 5948a2c7b4..b2d5c8ed96 100644 --- a/test/util/kibishii/kibishii_utils.go +++ b/test/util/kibishii/kibishii_utils.go @@ -28,8 +28,8 @@ import ( "context" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" - "github.com/pkg/errors" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" diff --git a/test/util/metrics/minio.go b/test/util/metrics/minio.go index 163212713c..289efe1cc8 100644 --- a/test/util/metrics/minio.go +++ b/test/util/metrics/minio.go @@ -17,7 +17,7 @@ limitations under the License. package metrics import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/test/util/providers" ) diff --git a/test/util/metrics/nfs.go b/test/util/metrics/nfs.go index 2ea6b2f1e3..d2b0da87a8 100644 --- a/test/util/metrics/nfs.go +++ b/test/util/metrics/nfs.go @@ -21,7 +21,7 @@ import ( "os/exec" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" ) func GetNFSPathDiskUsage(ctx context.Context, nfsServerPath string) (string, error) { diff --git a/test/util/providers/aws_utils.go b/test/util/providers/aws_utils.go index 7b8916cefd..d12e3d71c2 100644 --- a/test/util/providers/aws_utils.go +++ b/test/util/providers/aws_utils.go @@ -36,7 +36,7 @@ import ( ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go-v2/service/s3" s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/test" diff --git a/test/util/providers/azure_utils.go b/test/util/providers/azure_utils.go index 468dbfe409..6c3d7cf1c8 100644 --- a/test/util/providers/azure_utils.go +++ b/test/util/providers/azure_utils.go @@ -36,10 +36,10 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" - "github.com/joho/godotenv" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" + "github.com/vmware-tanzu/velero/pkg/util/dotenv" . "github.com/vmware-tanzu/velero/test" ) @@ -127,7 +127,7 @@ func loadCredentialsIntoEnv(credentialsFile string) error { return nil } - if err := godotenv.Overload(credentialsFile); err != nil { + if err := dotenv.Overload(credentialsFile); err != nil { return errors.Wrapf(err, "error loading environment from credentials file (%s)", credentialsFile) } return nil diff --git a/test/util/providers/common.go b/test/util/providers/common.go index a7c68dc37e..2886e51ebf 100644 --- a/test/util/providers/common.go +++ b/test/util/providers/common.go @@ -25,7 +25,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/internal/volume" velerotest "github.com/vmware-tanzu/velero/test" diff --git a/test/util/providers/gcloud_utils.go b/test/util/providers/gcloud_utils.go index 022d92ff25..a07a5b39a2 100644 --- a/test/util/providers/gcloud_utils.go +++ b/test/util/providers/gcloud_utils.go @@ -26,7 +26,7 @@ import ( "context" "cloud.google.com/go/storage" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "golang.org/x/oauth2/google" "google.golang.org/api/compute/v1" "google.golang.org/api/iterator" diff --git a/test/util/report/report.go b/test/util/report/report.go index 6d5955392c..8661bf9496 100644 --- a/test/util/report/report.go +++ b/test/util/report/report.go @@ -19,8 +19,8 @@ package report import ( "os" - "github.com/pkg/errors" - "gopkg.in/yaml.v3" + "github.com/cockroachdb/errors" + "go.yaml.in/yaml/v3" "github.com/vmware-tanzu/velero/test" ) diff --git a/test/util/velero/install.go b/test/util/velero/install.go index 5ef7b0001d..b7c62fb609 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -28,7 +28,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index 5c9b39a849..32f12cdae7 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -36,7 +36,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "golang.org/x/mod/semver" schedulingv1api "k8s.io/api/scheduling/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"