From 41d8271b1ca73ecc89ac879e054fcbd03e8077a4 Mon Sep 17 00:00:00 2001 From: Piotr Galar Date: Wed, 9 Mar 2022 12:44:47 +0100 Subject: [PATCH] Initial commit --- .github/actions/git-config-user/action.yml | 10 + .github/actions/git-push/action.yml | 39 +++ .github/workflows/apply.yml | 95 ++++++ .github/workflows/plan.yml | 171 ++++++++++ .github/workflows/plan_post.yml | 26 ++ .github/workflows/plan_pre.yml | 13 + .github/workflows/sync.yml | 123 +++++++ .github/workflows/update.yml | 49 +++ .github/workflows/upgrade.yml | 12 + .github/workflows/upgrade_reusable.yml | 55 ++++ .gitignore | 30 ++ CHANGELOG.md | 7 + README.md | 298 +++++++++++++++++ .../$ORGANIZATION_NAME/branch_protection.json | 1 + github/$ORGANIZATION_NAME/repository.json | 1 + scripts/sync.sh | 131 ++++++++ terraform/data.tf | 29 ++ terraform/locals.tf | 8 + terraform/outputs.tf | 84 +++++ terraform/providers.tf | 5 + terraform/resources.tf | 302 ++++++++++++++++++ terraform/resources_override.tf | 24 ++ terraform/terraform.tf | 12 + terraform/terraform_override.tf | 9 + terraform/variables.tf | 5 + 25 files changed, 1539 insertions(+) create mode 100644 .github/actions/git-config-user/action.yml create mode 100644 .github/actions/git-push/action.yml create mode 100644 .github/workflows/apply.yml create mode 100644 .github/workflows/plan.yml create mode 100644 .github/workflows/plan_post.yml create mode 100644 .github/workflows/plan_pre.yml create mode 100644 .github/workflows/sync.yml create mode 100644 .github/workflows/update.yml create mode 100644 .github/workflows/upgrade.yml create mode 100644 .github/workflows/upgrade_reusable.yml create mode 100644 .gitignore create mode 100644 CHANGELOG.md create mode 100644 README.md create mode 100644 github/$ORGANIZATION_NAME/branch_protection.json create mode 100644 github/$ORGANIZATION_NAME/repository.json create mode 100755 scripts/sync.sh create mode 100644 terraform/data.tf create mode 100644 terraform/locals.tf create mode 100644 terraform/outputs.tf create mode 100644 terraform/providers.tf create mode 100644 terraform/resources.tf create mode 100644 terraform/resources_override.tf create mode 100644 terraform/terraform.tf create mode 100644 terraform/terraform_override.tf create mode 100644 terraform/variables.tf diff --git a/.github/actions/git-config-user/action.yml b/.github/actions/git-config-user/action.yml new file mode 100644 index 0000000..9bb2851 --- /dev/null +++ b/.github/actions/git-config-user/action.yml @@ -0,0 +1,10 @@ +name: Configure git user +description: Configure git user + +runs: + using: composite + steps: + - run: | + git config --global user.email '${{ github.actor }}@users.noreply.github.com>' + git config --global user.name '${{ github.actor }}' + shell: bash diff --git a/.github/actions/git-push/action.yml b/.github/actions/git-push/action.yml new file mode 100644 index 0000000..54ae873 --- /dev/null +++ b/.github/actions/git-push/action.yml @@ -0,0 +1,39 @@ +name: Push to a git branch +description: Push to a git branch + +inputs: + suffix: + description: Branch name suffix + required: true + working-directory: + description: Working directory + required: false + default: ${{ github.workspace }} + +runs: + using: composite + steps: + - run: | + protected="$(gh api 'repos/{owner}/{repo}/branches/${{ github.ref_name }}' --jq '.protected')" + + if [[ "$protected" == 'true' ]]; then + git_branch='${{ github.ref_name }}-${{ inputs.suffix }}' + else + git_branch='${{ github.ref_name }}' + fi + + git checkout -B "$git_branch" + + if [[ "$protected" == 'true' ]]; then + git push origin "$git_branch" --force + if [[ ! -z "$(git diff --name-only 'origin/${{ github.ref_name }}')" ]]; then + state="$(gh pr view "$git_branch" --json state --jq .state 2> /dev/null || echo '')" + if [[ "$state" != 'OPEN' ]]; then + gh pr create --body 'The changes in this PR were made by a bot. Please review carefully.' --head "$git_branch" --base '${{ github.ref_name }}' --fill + fi + fi + else + git push origin "$git_branch" + fi + shell: bash + working-directory: ${{ inputs.working-directory }} diff --git a/.github/workflows/apply.yml b/.github/workflows/apply.yml new file mode 100644 index 0000000..b9739cc --- /dev/null +++ b/.github/workflows/apply.yml @@ -0,0 +1,95 @@ +name: Apply + +on: + push: + workflow_dispatch: + +jobs: + prepare: + if: github.ref_name == github.event.repository.default_branch && + github.event.repository.is_template == false + permissions: + contents: read + issues: read + pull-requests: read + name: Prepare + runs-on: ubuntu-latest + outputs: + workspaces: ${{ steps.workspaces.outputs.this }} + sha: ${{ steps.sha.outputs.this }} + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Discover workspaces + id: workspaces + run: echo "::set-output name=this::$(ls github | jq --raw-input '[.]' | jq -sc add)" + - name: Find pull request number + id: pull_request + if: github.event_name == 'push' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: protocol/github-api-action-library/find-content-by-query@v1 + with: + query: repository:${{ github.repository }} ${{ github.sha }} + - name: Find sha for plan + id: sha + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [[ '${{ github.event_name }}' == 'push' ]]; then + number="$(jq -r '.[0].number // ""' <<< '${{ steps.pull_request.outputs.issues-or-pull-requests }}')" + if [[ ! -z "$number" ]]; then + sha="$(gh pr view "$number" --json commits --jq '.commits[-1].oid')" + fi + else + sha='${{ github.sha }}' + fi + echo "::set-output name=this::$sha" + apply: + needs: [prepare] + if: needs.prepare.outputs.sha != '' && needs.prepare.outputs.workspaces != '' + permissions: + actions: read + contents: read + strategy: + fail-fast: false + matrix: + workspace: ${{ fromJson(needs.prepare.outputs.workspaces) }} + name: Apply + runs-on: ubuntu-latest + env: + TF_IN_AUTOMATION: 1 + TF_INPUT: 0 + TF_WORKSPACE: ${{ matrix.workspace }} + AWS_ACCESS_KEY_ID: ${{ secrets.RW_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.RW_AWS_SECRET_ACCESS_KEY }} + GITHUB_APP_ID: ${{ secrets.RW_GITHUB_APP_ID }} + GITHUB_APP_INSTALLATION_ID: ${{ secrets[format('RW_GITHUB_APP_INSTALLATION_ID_{0}', matrix.workspace)] }} + GITHUB_APP_PEM_FILE: ${{ secrets.RW_GITHUB_APP_PEM_FILE }} + TF_VAR_write_delay_ms: 300 + defaults: + run: + shell: bash + working-directory: terraform + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup terraform + uses: hashicorp/setup-terraform@3d8debd658c92063839bc97da5c2427100420dec # v1.3.2 + with: + terraform_version: 1.1.4 + - name: Initialize terraform + run: terraform init + - name: Retrieve targets from config + id: target + run: echo "::set-output name=this::$(jq -r 'split(" ")[:-1] | map("-target=github_\(sub(".json$"; "")).this") | join(" ")' <<< '"'"$(ls | tr '\n' ' ')"'"')" + working-directory: github/${{ env.TF_WORKSPACE }} + - name: Terraform Plan Download + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh run download -n ${{ env.TF_WORKSPACE }}_${{ needs.prepare.outputs.sha }}.tfplan --repo ${{ github.repository }} + - name: Terraform Apply + run: terraform apply -auto-approve -lock-timeout=0s -no-color ${{ env.TF_WORKSPACE }}.tfplan diff --git a/.github/workflows/plan.yml b/.github/workflows/plan.yml new file mode 100644 index 0000000..0aab872 --- /dev/null +++ b/.github/workflows/plan.yml @@ -0,0 +1,171 @@ +name: Plan + +on: + workflow_run: + workflows: + - "Plan (pre)" + types: + - completed + workflow_dispatch: + +jobs: + prepare: + if: (github.event_name == 'workflow_dispatch' && + github.ref_name == github.event.repository.default_branch) || + (github.event_name == 'workflow_run' && + github.event.workflow_run.conclusion == 'success') + permissions: + actions: read + contents: read + statuses: write + name: Prepare + runs-on: ubuntu-latest + outputs: + workspaces: ${{ steps.workspaces.outputs.this }} + repository: ${{ steps.github.outputs.repository }} + sha: ${{ steps.github.outputs.sha }} + number: ${{ steps.github.outputs.number }} + defaults: + run: + shell: bash + steps: + - name: Find repository and sha to checkout + id: github + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [[ '${{ github.event_name }}' == 'workflow_dispatch' ]]; then + repository='${{ github.repository }}' + sha='${{ github.sha }}' + elif [[ '${{ github.event_name }}' == 'workflow_run' ]]; then + repository='${{ github.event.workflow_run.head_repository.full_name }}' + sha='${{ github.event.workflow_run.head_commit.id }}' + number="$(gh api '/repos/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}' --jq '.pull_requests[0].number')" + fi + echo "::set-output name=repository::$repository" + echo "::set-output name=sha::$sha" + echo "::set-output name=number::$number" + - run: sha=${{ steps.github.outputs.sha }} + - env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh api 'repos/${{ github.repository }}/statuses/${{ steps.github.outputs.sha }}' -f context='Plan' -f state='pending' -f target_url='${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}' + - name: Checkout + uses: actions/checkout@v2 + with: + repository: ${{ steps.github.outputs.repository }} + ref: ${{ steps.github.outputs.sha }} + - name: Discover workspaces + id: workspaces + run: echo "::set-output name=this::$(ls github | jq --raw-input '[.]' | jq -sc add)" + plan: + needs: [prepare] + if: needs.prepare.outputs.workspaces != '' + permissions: + contents: read + strategy: + fail-fast: false + matrix: + workspace: ${{ fromJson(needs.prepare.outputs.workspaces) }} + name: Plan + runs-on: ubuntu-latest + env: + TF_IN_AUTOMATION: 1 + TF_INPUT: 0 + TF_WORKSPACE: ${{ matrix.workspace }} + AWS_ACCESS_KEY_ID: ${{ secrets.RO_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.RO_AWS_SECRET_ACCESS_KEY }} + GITHUB_APP_ID: ${{ secrets.RO_GITHUB_APP_ID }} + GITHUB_APP_INSTALLATION_ID: ${{ secrets[format('RO_GITHUB_APP_INSTALLATION_ID_{0}', matrix.workspace)] }} + GITHUB_APP_PEM_FILE: ${{ secrets.RO_GITHUB_APP_PEM_FILE }} + TF_VAR_write_delay_ms: 300 + defaults: + run: + shell: bash + working-directory: terraform + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + repository: ${{ needs.prepare.outputs.repository }} + ref: ${{ needs.prepare.outputs.sha }} + - name: Setup terraform + uses: hashicorp/setup-terraform@3d8debd658c92063839bc97da5c2427100420dec # v1.3.2 + with: + terraform_version: 1.1.4 + - name: Initialize terraform + run: terraform init + - name: Check terraform lock + if: github.event_name == 'workflow_run' + run: git diff --exit-code .terraform.lock.hcl + - name: Format terraform + run: terraform fmt -check + - name: Validate terraform + run: terraform validate -no-color + - name: Retrieve targets from config + id: target + run: echo "::set-output name=this::$(jq -r 'split(" ")[:-1] | map("-target=github_\(sub(".json$"; "")).this") | join(" ")' <<< '"'"$(ls | tr '\n' ' ')"'"')" + working-directory: github/${{ env.TF_WORKSPACE }} + - name: Plan terraform + id: plan + run: terraform plan ${{ steps.target.outputs.this }} -refresh=false -lock=false -out=${{ env.TF_WORKSPACE }}.tfplan -no-color + - name: Upload terraform plan + uses: actions/upload-artifact@v2 + with: + name: ${{ env.TF_WORKSPACE }}_${{ needs.prepare.outputs.sha }}.tfplan + path: terraform/${{ env.TF_WORKSPACE }}.tfplan + if-no-files-found: error + retention-days: 90 + comment: + needs: [prepare, plan] + if: github.event_name == 'workflow_run' + permissions: + contents: read + pull-requests: write + name: Comment + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.RO_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.RO_AWS_SECRET_ACCESS_KEY }} + defaults: + run: + shell: bash + working-directory: terraform + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + repository: ${{ needs.prepare.outputs.repository }} + ref: ${{ needs.prepare.outputs.sha }} + - name: Setup terraform + uses: hashicorp/setup-terraform@3d8debd658c92063839bc97da5c2427100420dec # v1.3.2 + with: + terraform_version: 1.1.4 + terraform_wrapper: false + - name: Initialize terraform + run: terraform init + - name: Download terraform plans + uses: actions/download-artifact@v2 + with: + path: terraform + - name: Show terraform plans + run: | + echo 'COMMENT<> $GITHUB_ENV + for plan in $(find . -type f -name '*.tfplan'); do + echo "
$(basename "$plan" '.tfplan')" >> $GITHUB_ENV + echo '' >> $GITHUB_ENV + echo '```' >> $GITHUB_ENV + echo "$(terraform show -no-color "$plan" 2>&1)" >> $GITHUB_ENV + echo '```' >> $GITHUB_ENV + echo '' >> $GITHUB_ENV + echo '
' >> $GITHUB_ENV + done + echo 'EOF' >> $GITHUB_ENV + - name: Comment on pull request + uses: marocchino/sticky-pull-request-comment@39c5b5dc7717447d0cba270cd115037d32d28443 # v2.2.0 + with: + number: ${{ needs.prepare.outputs.number }} + message: | + Before merge, verify that all the following plans are correct. They will be applied as-is after the merge. + + #### Terraform plans + ${{ env.COMMENT }} diff --git a/.github/workflows/plan_post.yml b/.github/workflows/plan_post.yml new file mode 100644 index 0000000..232d1ff --- /dev/null +++ b/.github/workflows/plan_post.yml @@ -0,0 +1,26 @@ +name: "Plan (post)" + +on: + workflow_run: + workflows: + - Plan + types: + - completed + +jobs: + notify: + permissions: + actions: read + statuses: write + name: "Notify" + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + sha="$(gh api '/repos/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}/jobs' --jq '.jobs[] | .steps[] | .name | select(startswith("Run sha="))')" + sha="${sha#Run sha=}" + gh api "repos/${{ github.repository }}/statuses/$sha" -f context='Plan' -f state='${{ github.event.workflow_run.conclusion }}' -f target_url='${{ github.event.workflow_run.html_url }}' diff --git a/.github/workflows/plan_pre.yml b/.github/workflows/plan_pre.yml new file mode 100644 index 0000000..a714e78 --- /dev/null +++ b/.github/workflows/plan_pre.yml @@ -0,0 +1,13 @@ +name: "Plan (pre)" + +on: + pull_request: + +jobs: + trigger: + if: github.event.pull_request.base.ref == github.event.repository.default_branch && + github.event.repository.is_template == false + name: "Trigger" + runs-on: ubuntu-latest + steps: + - run: "true" diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml new file mode 100644 index 0000000..d3db744 --- /dev/null +++ b/.github/workflows/sync.yml @@ -0,0 +1,123 @@ +name: Sync + +on: + workflow_dispatch: + inputs: + workspaces: + description: Space separated list of workspaces to sync (leave blank to sync all) + required: false + lock: + description: Whether to acquire terraform state lock during sync + required: false + default: "true" + debug: + description: Print out all the commands being executed + required: false + default: "false" + +jobs: + prepare: + name: Prepare + runs-on: ubuntu-latest + outputs: + workspaces: ${{ steps.workspaces.outputs.this }} + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Discover workspaces + id: workspaces + run: | + if [[ -z '${{ github.event.inputs.workspaces }}' ]]; then + workspaces="$(ls github | jq --raw-input '[.]' | jq -sc add)" + else + workspaces="$(echo '${{ github.event.inputs.workspaces }}' | jq --raw-input 'split(" ")')" + fi + echo "::set-output name=this::$workspaces" + sync: + needs: [prepare] + if: needs.prepare.outputs.workspaces != '' + permissions: + contents: write + strategy: + fail-fast: false + matrix: + workspace: ${{ fromJson(needs.prepare.outputs.workspaces) }} + name: Sync + runs-on: ubuntu-latest + env: + TF_IN_AUTOMATION: 1 + TF_INPUT: 0 + TF_LOCK: ${{ github.event.inputs.lock }} + TF_DEBUG: ${{ github.event.inputs.debug }} + AWS_ACCESS_KEY_ID: ${{ secrets.RW_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.RW_AWS_SECRET_ACCESS_KEY }} + GITHUB_APP_ID: ${{ secrets.RW_GITHUB_APP_ID }} + GITHUB_APP_INSTALLATION_ID: ${{ secrets[format('RW_GITHUB_APP_INSTALLATION_ID_{0}', matrix.workspace)] }} + GITHUB_APP_PEM_FILE: ${{ secrets.RW_GITHUB_APP_PEM_FILE }} + TF_VAR_write_delay_ms: 300 + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup terraform + uses: hashicorp/setup-terraform@v1 + with: + terraform_version: 1.1.4 + terraform_wrapper: false + - name: Initialize terraform + run: terraform init + working-directory: terraform + - name: Select terraform workspace + run: | + terraform workspace select '${{ matrix.workspace }}' || terraform workspace new '${{ matrix.workspace }}' + echo 'TF_WORKSPACE=${{ matrix.workspace }}' >> $GITHUB_ENV + working-directory: terraform + - name: Sync + run: ./scripts/sync.sh + - uses: ./.github/actions/git-config-user + - env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git_branch='${{ github.ref_name }}-sync-${{ env.TF_WORKSPACE }}' + git checkout -B "$git_branch" + git add --all + git diff-index --quiet HEAD || git commit --message='sync ${{ env.TF_WORKSPACE }} (#${{ github.run_number }})' + git push origin "$git_branch" --force + pull_request: + needs: [prepare, sync] + if: needs.prepare.outputs.workspaces != '' + name: Pull request + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Generate app token + id: token + uses: tibdex/github-app-token@7ce9ffdcdeb2ba82b01b51d6584a6a85872336d4 # v1.5.1 + with: + app_id: ${{ secrets.RW_GITHUB_APP_ID }} + installation_id: ${{ secrets[format('RW_GITHUB_APP_INSTALLATION_ID_{0}', github.repository_owner)] }} + private_key: ${{ secrets.RW_GITHUB_APP_PEM_FILE }} + - name: Checkout + uses: actions/checkout@v2 + with: + token: ${{ steps.token.outputs.token }} + - uses: ./.github/actions/git-config-user + - run: | + while read workspace; do + workspace_branch="${{ github.ref_name }}-sync-$workspace" + git fetch origin "$workspace_branch" + git merge --strategy-option=theirs "origin/$workspace_branch" + git push origin --delete "$workspace_branch" + done <<< "$(jq -r '.[]' <<< '${{ needs.prepare.outputs.workspaces }}')" + - uses: ./.github/actions/git-push + env: + GITHUB_TOKEN: ${{ steps.token.outputs.token }} + with: + suffix: sync diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml new file mode 100644 index 0000000..cd00701 --- /dev/null +++ b/.github/workflows/update.yml @@ -0,0 +1,49 @@ +name: Update + +on: + workflow_run: + workflows: + - "Apply" + types: + - completed + workflow_dispatch: + +jobs: + update: + if: (github.event_name == 'workflow_dispatch' && + github.ref_name == github.event.repository.default_branch) || + (github.event_name == 'workflow_run' && + github.event.workflow_run.conclusion == 'success') + name: Update + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Generate app token + id: token + uses: tibdex/github-app-token@7ce9ffdcdeb2ba82b01b51d6584a6a85872336d4 # v1.5.1 + with: + app_id: ${{ secrets.RW_GITHUB_APP_ID }} + installation_id: ${{ secrets[format('RW_GITHUB_APP_INSTALLATION_ID_{0}', github.repository_owner)] }} + private_key: ${{ secrets.RW_GITHUB_APP_PEM_FILE }} + - name: Update PRs + env: + GITHUB_TOKEN: ${{ steps.token.outputs.token }} + run: | + while read pr; do + if [[ ! -z "$pr" ]]; then + number="$(jq -r '.number' <<< "$pr")" + base_label="$(jq -r '.base.label' <<< "$pr")" + head_label="$(jq -r '.head.label' <<< "$pr")" + behind_by="$(gh api "/repos/${{ github.repository }}/compare/$base_label...$head_label" --jq '.behind_by')" + if [[ "$behind_by" == '0' ]]; then + echo '{"message":"Not updating pull request branch.","url":"https://api.github.com/repos/${{ github.repository }}/pulls/'"$number"'"}' + elif ! gh api -X PUT "/repos/${{ github.repository }}/pulls/$number/update-branch"; then + echo '' + echo "::warning title=update-branch failure::${{github.repository}}#$number" + else + echo '' + fi + fi + done <<< "$(gh api '/repos/${{ github.repository }}/pulls' --jq '.[]')" diff --git a/.github/workflows/upgrade.yml b/.github/workflows/upgrade.yml new file mode 100644 index 0000000..09e4ad5 --- /dev/null +++ b/.github/workflows/upgrade.yml @@ -0,0 +1,12 @@ +name: Upgrade + +on: + workflow_dispatch: + +jobs: + upgrade: + uses: protocol/github-mgmt-template/.github/workflows/upgrade_reusable.yml@master + secrets: + GITHUB_APP_ID: ${{ secrets.RW_GITHUB_APP_ID }} + GITHUB_APP_INSTALLATION_ID: ${{ secrets[format('RW_GITHUB_APP_INSTALLATION_ID_{0}', github.repository_owner)] }} + GITHUB_APP_PEM_FILE: ${{ secrets.RW_GITHUB_APP_PEM_FILE }} diff --git a/.github/workflows/upgrade_reusable.yml b/.github/workflows/upgrade_reusable.yml new file mode 100644 index 0000000..b4342d0 --- /dev/null +++ b/.github/workflows/upgrade_reusable.yml @@ -0,0 +1,55 @@ +name: Upgrade (reusable) + +on: + workflow_call: + secrets: + GITHUB_APP_ID: + required: true + GITHUB_APP_INSTALLATION_ID: + required: true + GITHUB_APP_PEM_FILE: + required: true + +jobs: + upgrade: + name: Upgrade + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Generate app token + id: token + uses: tibdex/github-app-token@7ce9ffdcdeb2ba82b01b51d6584a6a85872336d4 # v1.5.1 + with: + app_id: ${{ secrets.GITHUB_APP_ID }} + installation_id: ${{ secrets.GITHUB_APP_INSTALLATION_ID }} + private_key: ${{ secrets.GITHUB_APP_PEM_FILE }} + - name: Checkout GitHub Management template + uses: actions/checkout@v2 + with: + repository: protocol/github-mgmt-template + path: github-mgmt-template + - name: Checkout GitHub Management + uses: actions/checkout@v2 + with: + path: github-mgmt + token: ${{ steps.token.outputs.token }} + - name: Copy files from the template + run: | + for file in $(git ls-files .github scripts terraform .gitignore ':!:terraform/*_override.tf' ':!:.github/workflows/*_reusable.yml'); do + mkdir -p "../github-mgmt/$(dirname "$file")" + cp -f "$file" "../github-mgmt/$file" + done + working-directory: github-mgmt-template + - uses: ./github-mgmt-template/.github/actions/git-config-user + - run: | + git add --all + git diff-index --quiet HEAD || git commit --message='upgrade (#${{ github.run_number }})' + working-directory: github-mgmt + - uses: ./github-mgmt-template/.github/actions/git-push + env: + GITHUB_TOKEN: ${{ steps.token.outputs.token }} + with: + suffix: upgrade + working-directory: github-mgmt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..34adc7e --- /dev/null +++ b/.gitignore @@ -0,0 +1,30 @@ +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log + +# Ignore any .tfvars files that are generated automatically for each Terraform run. Most +# .tfvars files are managed as part of configuration and so should be included in +# version control. +# +# example.tfvars + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +# override.tf +# override.tf.json +# *_override.tf +# *_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* +*.tfplan diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..d94e3d6 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] diff --git a/README.md b/README.md new file mode 100644 index 0000000..e32b355 --- /dev/null +++ b/README.md @@ -0,0 +1,298 @@ +# GitHub Management via Terraform: Template + +This repository is meant to serve as a template for creating new repositories responsible for managing GitHub configuration as code with Terraform. It provides an opinionated way to manage GitHub configuration without following the Terraform usage guidelines to the letter. It was designed with managing multiple, medium to large sized GitHub organisations in mind and that is exactly what it is/is going to be optimised for. + +## Key features + +- 2-way sync between GitHub Management and the actual GitHub configuration (including bootstrapping) +- PR-based configuration change review process which guarantees the reviewed plan is the one being applied +- control over what resources and what properties are managed by GitHub Management +- auto-upgrades from the template repository + +## How does it work? + +GitHub Management allows management of GitHub configuration as code. It uses Terraform and GitHub Actions to achieve this. + +The JSON configuration files for a specific organisation are stored in [github/$ORGANIZATION_NAME](github/$ORGANIZATION_NAME) directory. GitHub Management lets you manage multiple organizations from a single repository. It uses separate terraform workspaces per each organisation. The local workspaces are called like the organisations themselves. Each workspace has its state hosted in the remote [S3 backend](https://www.terraform.io/language/settings/backends/s3). + +The configuration files are named after the [GitHub Provider resources](https://registry.terraform.io/providers/integrations/github/latest/docs) they configure but are stripped of the `github_` prefix. + +Each configuration file contains a single, top-level *JSON* object. The keys in that object are resource identifiers - the required argument values of that resource type. The values - objects describing other arguments, attributes and the id of that resource type. + +For example, [github_repository](https://registry.terraform.io/providers/integrations/github/latest/docs/resources/repository#argument-reference) resource has one required argument - `name` - so the keys inside the object in [repository.json](github/$ORGANIZATION_NAME/repository.json) would be the names of the repositories owned by the organisation. The values in that object would be objects describing remaining [arguments](https://registry.terraform.io/providers/integrations/github/latest/docs/resources/repository#argument-reference) and [attributes](https://registry.terraform.io/providers/integrations/github/latest/docs/resources/repository#attributes-reference). + +Another example would be [github_branch_protection](https://registry.terraform.io/providers/integrations/github/latest/docs/resources/branch_protection#argument-reference) which has two required arguments - `repository_id` and `pattern`. In such case, the keys would be nested under each other. The keys in the top-level object in [branch_protection.json](github/$ORGANIZATION_NAME/branch_protection.json) would be the IDs of the repositories owned by the organisation and their values would be objects with patterns of branch protection rules for that repository as keys. Where possible, the IDs in key values are replaced with more human friendly values. That's why, the actual top-level key values in [branch_protection.json](github/$ORGANIZATION_NAME/branch_protection.json) would be repository names, not repository IDs. + +Whether resources of a specific type are managed via GitHub Management or not is controlled by the existence of the corresponding configuration files. If such a file exists, GitHub Management manages all the arguments and attributes of that resource type except for the ones specified in the `ignore_changes` lists in [terraform/resources_override.tf](terraform/resources_override.tf) or [terraform/resources.tf](terraform/resources.tf). + +GitHub Management is capable of both applying the changes made to the JSON configuration files to the actual GitHub configuraiton state and of translating the current GitHub configuration state into the JSON configuration files. + +The workflow for introducing changes to GitHub via JSON configuration files is as follows: +1. Modify the JSON configuration file. +1. Create a PR and wait for the GitHub Action workflow triggered on PRs to comment on it with a terraform plan. +1. Review the plan. +1. Merge the PR and wait for the GitHub Action workflow triggered on pushes to the default branch to apply it. + +Neither creating the terraform plan nor applying it refreshes the underlying terraform state i.e. going through this workflow does **NOT** ask GitHub if the actual GitHub configuration state has changed. This makes the workflow fast and rate limit friendly because the number of requests to GitHub is minimised. This can result in the plan failing to be applied, e.g. if the underlying resource has been deleted. This assumes that JSON configuration should be the main source of truth for GitHub configuration state. The plans that are created during the PR GitHub Action workflow are applied exactly as-is after the merge. + +The workflow for synchronising the current GitHub configuration state with JSON configuration files is as follows: +1. Run the `Sync` GitHub Action workflow and wait for the PR to be created. +1. If a PR was created, wait for the GitHub Action workflow triggered on PRs to comment on it with a terraform plan. +1. Ensure that the plan introduces no changes. +1. Merge the PR. + +Running the `Sync` GitHub Action workflows refreshes the underlying terraform state. It also automatically imports all the resources that were created outside GitHub Management into the state and removes any that were deleted. After the `Sync` flow, all the other open PRs should have their GitHub Action workflows rerun because merging them without it would result in the application of their plans to fail due to the plans being created against a different state. + +## Supported resources + +| Resource | JSON | Key(s) | Dependencies | Description | +| --- | --- | --- | --- | --- | +| [github_membership](https://registry.terraform.io/providers/integrations/github/latest/docs/resources/membership) | `membership.json` | `username` | n/a | add/remove users from your organization | +| [github_repository](https://registry.terraform.io/providers/integrations/github/latest/docs/resources/repository) | `repository.json` | `repository.name` | n/a | create and manage repositories within your GitHub organization | +| [github_repository_collaborator](https://registry.terraform.io/providers/integrations/github/latest/docs/resources/repository_collaborator) | `repository_collaborator.json` | `repository.name`: `username` | `github_repository` | add/remove collaborators from repositories in your organization | +| [github_branch_protection](https://registry.terraform.io/providers/integrations/github/latest/docs/resources/branch_protection) | `branch_protection.json` | `repository.name`: `pattern` | `github_repository` | configure branch protection for repositories in your organization | +| [github_team](https://registry.terraform.io/providers/integrations/github/latest/docs/resources/team) | `team.json` | `team.name` | n/a | add/remove teams from your organization | +| [github_team_repository](https://registry.terraform.io/providers/integrations/github/latest/docs/resources/team_repository) | `team_repository.json` | `team.name`: `repository.name` | `github_team` | manage relationships between teams and repositories in your GitHub organization | +| [github_team_membership](https://registry.terraform.io/providers/integrations/github/latest/docs/resources/team_membership) | `team_membership.json` | `team.name`: `username` | `github_team` | add/remove users from teams in your organization | + +### Limitations + +Branch protection rules managed via GitHub Management cannot contain wildcards. They also have to match exactly one existing branch. This limitation comes from the fact that there is no GitHub API endpoint which returns a list of branch protection rule patterns for a repository. + +## How to... + +### ...get started? + +*NOTE*: The following TODO list is complete - it contains all the steps you should complete to get GitHub Management up. You might be able to skip some of them if you completed them before. + +- [ ] [Create a repository](https://docs.github.com/en/repositories/creating-and-managing-repositories/creating-a-repository-from-a-template) from the template - *this is the place for GitHub Management to live in* + +#### AWS + +- [ ] [Create a S3 bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/creating-bucket.html) - *this is where Terraform states for the organisations will be stored* +- [ ] [Create a DynamoDB table](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/getting-started-step-1.html) using `LockID` of type `String` as the partition key - *this is where Terraform state locks will be stored* +- [ ] [Create 2 IAM policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create.html) - *they are going to be attached to the users that GitHub Management is going to use to interact with AWS* +
Read-only + + ``` + { + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "s3:ListBucket" + ], + "Effect": "Allow", + "Resource": "arn:aws:s3:::$S3_BUCKET_NAME" + }, + { + "Action": [ + "s3:GetObject" + ], + "Effect": "Allow", + "Resource": "arn:aws:s3:::$S3_BUCKET_NAME/*" + }, + { + "Action": [ + "dynamodb:GetItem" + ], + "Effect": "Allow", + "Resource": "arn:aws:dynamodb:*:*:table/$DYNAMO_DB_TABLE_NAME" + } + ] + } + ``` +
+
Read & Write + + ``` + { + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "s3:ListBucket" + ], + "Effect": "Allow", + "Resource": "arn:aws:s3:::$S3_BUCKET_NAME" + }, + { + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject" + ], + "Effect": "Allow", + "Resource": "arn:aws:s3:::$S3_BUCKET_NAME/*" + }, + { + "Action": [ + "dynamodb:GetItem", + "dynamodb:PutItem", + "dynamodb:DeleteItem" + ], + "Effect": "Allow", + "Resource": "arn:aws:dynamodb:*:*:table/$DYNAMO_DB_TABLE_NAME" + } + ] + } + ``` +
+- [ ] [Create 2 IAM Users](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html) and save their `AWS_ACCESS_KEY_ID`s and `AWS_SECRET_ACCESS_KEY`s - *they are going to be used by GitHub Management to interact with AWS* + - [ ] one with read-only policy attached + - [ ] one with read & write policy attached +- [ ] Modify [terraform/terraform_override.tf](terraform/terraform_override.tf) to reflect your AWS setup + +#### GitHub App + +*NOTE*: If you already have a GitHub App with required permissions you can skip the app creation step. + +- [ ] [Create 2 GitHub Apps](https://docs.github.com/en/developers/apps/building-github-apps/creating-a-github-app) in the GitHub organisation with the following permissions - *they are going to be used by terraform and GitHub Actions to authenticate with GitHub*: +
read-only + + - `Repository permissions` + - `Administration`: `Read-only` + - `Contents`: `Read-only` + - `Metadata`: `Read-only` + - `Organization permissions` + - `Members`: `Read-only` +
+
read & write + + - `Repository permissions` + - `Administration`: `Read & Write` + - `Contents`: `Read & Write` + - `Metadata`: `Read-only` + - `Pull requests`: `Read & Write` + - `Workflows`: `Read & Write` + - `Organization permissions` + - `Members`: `Read & Write` +
+- [ ] [Install the GitHub Apps](https://docs.github.com/en/developers/apps/managing-github-apps/installing-github-apps) in the GitHub organisation for `All repositories` + +#### GitHub Repository Secrets + +- [ ] [Create encrypted secrets](https://docs.github.com/en/actions/security-guides/encrypted-secrets#creating-encrypted-secrets-for-an-organization) for the GitHub organisation and allow the repository to access them (\*replace `$GITHUB_ORGANIZATION_NAME` with the GitHub organisation name) - *these secrets are read by the GitHub Action workflows* + - [ ] Go to `https://github.com/organizations/$GITHUB_ORGANIZATION_NAME/settings/apps/$GITHUB_APP_NAME` and copy the `App ID` + - [ ] `RO_GITHUB_APP_ID` + - [ ] `RW_GITHUB_APP_ID` + - [ ] Go to `https://github.com/organizations/$GITHUB_ORGANIZATION_NAME/settings/installations`, click `Configure` next to the `$GITHUB_APP_NAME` and copy the numeric suffix from the URL + - [ ] `RO_GITHUB_APP_INSTALLATION_ID_$GITHUB_ORGANIZATION_NAME` + - [ ] `RW_GITHUB_APP_INSTALLATION_ID_$GITHUB_ORGANIZATION_NAME` + - [ ] Go to `https://github.com/organizations/$GITHUB_ORGANIZATION_NAME/settings/apps/$GITHUB_APP_NAME`, click `Generate a private key` and copy the contents of the downloaded PEM file + - [ ] `RO_GITHUB_APP_PEM_FILE` + - [ ] `RW_GITHUB_APP_PEM_FILE` + - [ ] Use the values generated during [AWS](#aws) setup + - [ ] `RO_AWS_ACCESS_KEY_ID` + - [ ] `RW_AWS_ACCESS_KEY_ID` + - [ ] `RO_AWS_SECRET_ACCESS_KEY` + - [ ] `RW_AWS_SECRET_ACCESS_KEY` + +#### GitHub Management Repository Setup + +*NOTE*: Advanced users might want to modify the resource types and their arguments/attributes managed by GitHub Management at this stage. + +- [ ] [Clone the repository](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository) +- [ ] Replace placeholder strings in the clone - *the repository needs to be customised for the specific organisation it is supposed to manage* + - [ ] Rename the `$GITHUB_ORGANIZATION_NAME` directory in `github` to the name of the GitHub organisation +- [ ] Push the changes to `$GITHUB_MGMT_REPOSITORY_DEFAULT_BRANCH` + +#### GitHub Management Sync Flow + +- [ ] Follow [How to synchronize GitHub Management with GitHub?](#synchronize-github-management-with-github) to commit the terraform lock and initialize terraform state + +#### GitHub Management Repository Protections + +*NOTE*: Advanced users might have to skip/adjust this step if they are not managing some of the arguments/attributes mentioned here with GitHub Management. + +*NOTE*: If you want to require PRs to be created but don't care about reviews, then change `required_approving_review_count` value to `0`. It seems for some reason the provider's default is `1` instead of `0`. The next `Sync` will remove this value from the configuration file and will leave an empty object inside `required_pull_request_reviews` which is the desired state. + +*NOTE*: Branch protection rules are not available for private repositories on Free plan. + +- [ ] Manually set values that are impossible to control this value via terraform currently + - [ ] `Settings` > `Actions` > `General` > `Fork pull request workflows from outside collaborators` > `Require approval for all outside collaborators` + - [ ] `Settings` > `Actions` > `General` > `Workflow permissions` > `Read repository contents permission` +- [ ] Pull remote changes to the default branch +- [ ] Enable required PRs, peer reviews, status checks and branch up-to-date check on the repository by making sure [github/$ORGANIZATION_NAME/branch_protection.json](github/$ORGANIZATION_NAME/branch_protection.json) contains the following entry: + ``` + "$GITHUB_MGMT_REPOSITORY_NAME": { + "$GITHUB_MGMT_REPOSITORY_DEFAULT_BRANCH": { + "required_pull_request_reviews": [ + { + "required_approving_review_count": 1 + } + ], + "required_status_checks": [ + { + "contexts": [ + "Plan" + ], + "strict": true + } + ] + } + } + ``` +- [ ] Push the changes to a branch other than the default branch + +#### GitHub Management PR Flow + +*NOTE*: Advanced users might have to skip this step if they skipped setting up [GitHub Management Repository Protections](#github-management-repository-protections) via GitHub Management. + +- [ ] Follow [How to apply GitHub Management changes to GitHub?](#apply-github-management-changes-to-github) to apply protections to the repository + +### ...add an organisation to be managed by GitHub Management? + +- [ ] Follow [How to get started with GitHub App?](#github-app) to create a GitHub App for the organisation +- [ ] Follow [How to get started with GitHub Organization Secrets?](#github-organisation-secrets) to set up secrets that GitHub Management is going to use +- [ ] Create a new directory called like the organisation under [github](github) directory which is going to store the configuration files +- [ ] Follow [How to add a resource type to be managed by GitHub Management?](#add-a-resource-type-to-be-managed-by-github-management) to add some resources to be managed by GitHub Management +- [ ] Follow [How to synchronize GitHub Management with GitHub?](#synchronize-github-management-with-github) while using the `branch` with your changes as a target to import all the resources you want to manage for the organisation + +### ...add a resource type to be managed by GitHub Management? + +- [ ] Create a new JSON file with `{}` as content for one of the [supported resources](#supported-resources) under `github/$ORGANIZATION_NAME` directory +- [ ] Follow [How to synchronize GitHub Management with GitHub?](#synchronize-github-management-with-github) while using the `branch` with your changes as a target to import all the resources you want to manage for the organisation + +### ...add a resource argument/attribute to be managed by GitHub Management? + +*NOTE*: You cannot set the values of attributes via GitHub Management but sometimes it is useful to have them available in the configuration files. For example, it might be a good idea to have `github_team.id` unignored if you want to manage `github_team.parent_team_id` via GitHub Management so that the users can quickly check each team's id without leaving the JSON configuration file. + +- [ ] Comment out the argument/attribute you want to start managing using GitHub Management in [terraform/resources.tf](terraform/resources.tf) +- [ ] Follow [How to synchronize GitHub Management with GitHub?](#synchronize-github-management-with-github) while using the `branch` with your changes as a target to import all the resources you want to manage for the organisation + +### ...add a resource? + +*NOTE*: You do not have to specify all the arguments/attributes when creating a new resource. If you don't, defaults as defined by the [GitHub Provider](https://registry.terraform.io/providers/integrations/github/latest/docs) will be used. The next `Sync` will fill out the remaining arguments/attributes in the JSON configuration file. + +*NOTE*: When creating a new resource, you can specify all the arguments that the resource supports even if changes to them are ignored. If you do specify arguments to which changes are ignored, their values are going to be applied during creation but a future `Sync` will remove them from configuration JSON. + +- [ ] Add a new JSON object `{}` under unique key in the JSON configuration file for one of the [supported resource](#supported-resources) +- [ ] Follow [How to apply GitHub Management changes to GitHub?](#apply-github-management-changes-to-github) to create your newly added resource + +### ...modify a resource? + +- [ ] Change the value of an argument/attribute in the JSON configuration file for one of the [supported resource](#supported-resources) +- [ ] Follow [How to apply GitHub Management changes to GitHub?](#apply-github-management-changes-to-github) to create your newly added resource + +### ...apply GitHub Management changes to GitHub? + +- [ ] [Create a pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request) from the branch to the default branch +- [ ] Merge the pull request once the `Plan` check passes and you verify the plan posted as a comment +- [ ] Confirm that the `Apply` GitHub Action workflow run applied the plan by inspecting the output + +### ...synchronize GitHub Management with GitHub? + +*NOTE*: Remember that the `Sync` operation modifes terraform state. Even if you run it from a branch, it modifies the global state that is shared with other branches. There is only one terraform state per organisation. + +*NOTE*: If you run the `Sync` from an unprotected branch, then the workflow will commit changes to it directly. + +*Note*: `Sync` is also going to sort the keys in all the objects lexicographically. + +- [ ] Run `Sync` GitHub Action workflow from your desired `branch` - *this will import all the resources from the actual GitHub configuration state into GitHub Management* +- [ ] Merge the pull request that the workflow created once the `Plan` check passes and you verify the plan posted as a comment - *the plan should not contain any changes* + +### ...upgrade GitHub Management? + +- [ ] Run `Upgrade` GitHub Action workflow +- [ ] Merge the pull request that the workflow created once the `Plan` check passes and you verify the plan posted as a comment - *the plan should not contain any changes* diff --git a/github/$ORGANIZATION_NAME/branch_protection.json b/github/$ORGANIZATION_NAME/branch_protection.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/github/$ORGANIZATION_NAME/branch_protection.json @@ -0,0 +1 @@ +{} diff --git a/github/$ORGANIZATION_NAME/repository.json b/github/$ORGANIZATION_NAME/repository.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/github/$ORGANIZATION_NAME/repository.json @@ -0,0 +1 @@ +{} diff --git a/scripts/sync.sh b/scripts/sync.sh new file mode 100755 index 0000000..52f5be7 --- /dev/null +++ b/scripts/sync.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash + +set -e +set -u +set -o pipefail + +debug="${TF_DEBUG:-false}" + +if [[ "$debug" == 'true' ]]; then + set -x +fi + +if [[ " $@ " == ' -h ' || " $@ " == ' --help ' || " $@ " =~ ' -help ' ]]; then + echo "Usage: $0 [options] [path]" + echo '' + echo 'This script synchronizes organization files with the remote GitHub state.' + echo 'If no path is specified, it will use the current directory.' + echo 'It will execute terraform in the current directory either way.' + echo '' + echo 'WARNING: It performs writes to the terraform state!' + echo 'WARNING: It performs writes to files tracked by git!' + echo '' + echo 'Options:' + echo ' -h | -help | --help: Show this help output.' + echo '' + echo 'Examples:' + echo " $0" + echo " $0 ." + echo " $0 -help" + exit 0 +fi + +at_address () { + jq 'try(.values.root_module.resources) // [] + try(.values.root_module.child_modules | map(.resources) | add) // [] | map(select(.address | startswith($address)))' --arg address "$1" <<< "$2" +} + +root="$(realpath "$(dirname "$0")/..")" + +pushd "$root/terraform" + +lock="${TF_LOCK:-true}" + +organization="$(terraform workspace show)" + +separator="$(cat "$root/terraform/locals.tf" | tr -d '\n' | grep -oP 'separator\s*=\s*"\K[^"]*')" +resources="$(jq 'split(" ")[:-1] | map(sub(".json$"; ""))' <<< '"'"$(ls "$root/github/$organization" | tr '\n' ' ')"'"')" +resource_targets="$(jq -r 'map("-target=github_\(.).this") | join(" ")' <<< "$resources")" + +data_targets="[]" +while read resource; do + data="$(cat "$root/terraform/data.tf" | tr -d '[:space:]' | grep -oP '#@resources.'"$resource"'.data[^"]+"\K.*?"' | tr -d '[:space:]' | tr '"' ' ')" + data="$(jq 'split(" ")' <<< '"'"${data:0:-1}"'"')" + data_targets="$(jq '$data + .' --argjson data "$data" <<< "$data_targets")" +done <<< "$(jq -r '.[]' <<< "$resources")" +data_targets="$(jq -r 'unique | map("-target=data.\(.).this") | join(" ")' <<< "$data_targets")" + +echo "Refreshing resources" +terraform refresh $resource_targets -lock=$lock +echo "Applying data changes" +terraform apply $data_targets -auto-approve -lock=$lock + +echo "Retrieving state" +state="$(terraform show -json)" +echo "Retrieving output" +output="$(terraform output -json)" + +while read resource; do + echo "Importing $resource" + + echo "Retrieving existing indices from state" + existing_indices="$(at_address "github_$resource.this" "$state" | jq 'map(.index)')" + echo "Retrieving all indices from outputs" + remote_data="$(jq '.[$resource].value' --arg resource "$resource" <<< "$output")" + remote_indices="$(jq 'map(.index)' <<< "$remote_data")" + + while read index; do + if [[ ! -z "$index" ]]; then + id="$(jq -r 'map(select(.index == $index)) | .[0].id' --argjson index "$index" <<< "$remote_data")" + + echo "Importing $index ($id)" + terraform import -lock=$lock "github_$resource.this[$index]" "$id" + fi + done <<< "$(jq '. - $existing_indices | .[]' --argjson existing_indices "$existing_indices" <<< "$remote_indices")" + + while read index; do + if [[ ! -z "$index" ]]; then + echo "Removing $index" + terraform state rm -lock=$lock "github_$resource.this[$index]" + fi + done <<< "$(jq '. - $remote_indices | .[]' --argjson remote_indices "$remote_indices" <<< "$existing_indices")" +done <<< "$(jq -r '.[]' <<< "$resources")" + +echo "Retrieving state" +state="$(terraform show -json)" + +while read resource; do + echo "Finding required arguments for $resource" + required="$(cat "$root/terraform/resources.tf" | tr -d '[:space:]' | grep -oP '#@resources.'"$resource"'.required\K.*?=' | tr -d '[:space:]')" + required="$(jq 'split("=")' <<< '"'"${required:0:-1}"'"')" + + echo "Finding ignored arguments/attributes for $resource" + ignore_changes="$(cat "$root/terraform/resources_override.tf" | tr -d '[:space:]' | { grep -oP '#@resources.'"$resource"'.ignore_changesignore_changes=\K.*?[^0]\]' || true; } | tr -d '[:space:]')" + if [[ -z "$ignore_changes" ]]; then + ignore_changes="$(cat "$root/terraform/resources.tf" | tr -d '[:space:]' | grep -oP '#@resources.'"$resource"'.ignore_changesignore_changes=\K.*?[^0]\]' | tr -d '[:space:]')" + fi + ignore_changes="$(jq 'split(",") | map(select(startswith("#") | not))' <<< '"'"${ignore_changes:1:-1}"'"')" + ignore="$(echo "$required" "$ignore_changes" | jq -s 'add')" + ignore_string="$(jq -r 'map(".\(.)") | join(", ")' <<< "$ignore")" + + echo "Retrieving resources from state" + resource_config="$(at_address "github_$resource.this" "$state" | jq 'map({"key": .index, "value": .values}) | from_entries')" + + echo "Ignoring ignored and required arguments" + resource_config="$(jq "map_values(del($ignore_string))" <<< "$resource_config")" + + if (( $(jq 'length' <<< "$required") > 1 )); then + echo "Breaking up top level keys" + resource_config="$(jq 'to_entries | + map(.key |= split($separator)) | + map({"key": .key[0], "value": {"key": .key[1], "value": .value}}) | + group_by(.key) | + map({"key": .[0].key, "value": map(.value) | from_entries}) | + from_entries' --arg separator "$separator" <<< "$resource_config")" + fi + + echo "Saving new resource configuration" + jq '.' <<< "$resource_config" > "$root/github/$organization/$resource.json" +done <<< "$(jq -r '.[]' <<< "$resources")" + +echo "Done" +popd diff --git a/terraform/data.tf b/terraform/data.tf new file mode 100644 index 0000000..d2ada8b --- /dev/null +++ b/terraform/data.tf @@ -0,0 +1,29 @@ +# @resources.membership.data +data "github_organization" "this" { + name = local.organization +} + +# @resources.repository.data +data "github_repositories" "this" { + query = "org:${local.organization}" +} + +# @resources.repository_collaborator.data +data "github_collaborators" "this" { + for_each = toset(data.github_repositories.this.names) + + owner = local.organization + repository = each.value + affiliation = "direct" +} + +# @resources.branch_protection.data +data "github_repository" "this" { + for_each = toset(data.github_repositories.this.names) + name = each.value +} + +# @resources.team.data +# @resources.team_repository.data +# @resources.team_membership.data +data "github_organization_teams" "this" {} diff --git a/terraform/locals.tf b/terraform/locals.tf new file mode 100644 index 0000000..e2a195d --- /dev/null +++ b/terraform/locals.tf @@ -0,0 +1,8 @@ +locals { + separator = " → " + organization = terraform.workspace + github = { + for file in fileset("${path.module}/../github/${local.organization}", "*.json") : + trimsuffix(file, ".json") => jsondecode(file("${path.module}/../github/${local.organization}/${file}")) + } +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf new file mode 100644 index 0000000..05a4cd8 --- /dev/null +++ b/terraform/outputs.tf @@ -0,0 +1,84 @@ +output "team" { + value = [ + for team in data.github_organization_teams.this.teams : + { + id = team.id + index = team.name + } + ] +} + +output "repository" { + value = [ + for name in data.github_repositories.this.names : + { + id = name + index = name + } + ] +} + +output "team_repository" { + value = flatten([ + for team in data.github_organization_teams.this.teams : + [ + for repository in team.repositories : + { + id = "${team.id}:${repository}" + index = "${team.name}${local.separator}${repository}" + } + ] + ]) +} + +output "team_membership" { + value = flatten([ + for team in data.github_organization_teams.this.teams : + [ + for member in team.members : + { + id = "${team.id}:${member}" + index = "${team.name}${local.separator}${member}" + } + ] + ]) +} + +output "membership" { + value = [ + for member in data.github_organization.this.members : + { + id = "${local.organization}:${member}" + index = member + } + ] +} + +output "repository_collaborator" { + value = flatten([ + for repository, collaborators in data.github_collaborators.this : + [ + for collaborator in collaborators.collaborator : + { + id = "${repository}:${collaborator.login}" + index = "${repository}${local.separator}${collaborator.login}" + } + ] + ]) +} + +output "branch_protection" { + value = flatten([ + for repository, config in data.github_repository.this : + [ + # unfortunately, we have to assume the branch protection rule is the same as the pattern + # once the provider migrates to GraphQL API, we'll be able to retrieve true patterns + # if it is needed before that, we can write our own custom GraphQL query to retrieve patterns + for branch in config.branches : + { + id = "${repository}:${branch.name}" + index = "${repository}${local.separator}${branch.name}" + } if branch.protected + ] + ]) +} diff --git a/terraform/providers.tf b/terraform/providers.tf new file mode 100644 index 0000000..2a306bb --- /dev/null +++ b/terraform/providers.tf @@ -0,0 +1,5 @@ +provider "github" { + owner = local.organization + write_delay_ms = var.write_delay_ms + app_auth {} +} diff --git a/terraform/resources.tf b/terraform/resources.tf new file mode 100644 index 0000000..fcaf864 --- /dev/null +++ b/terraform/resources.tf @@ -0,0 +1,302 @@ +resource "github_membership" "this" { + for_each = lookup(local.github, "membership", {}) + + # @resources.membership.required + username = each.key + + role = try(each.value.role, null) + + lifecycle { + # @resources.membership.ignore_changes + ignore_changes = [ + etag, + id, + role + ] + } +} + +resource "github_repository" "this" { + for_each = lookup(local.github, "repository", {}) + + # @resources.repository.required + name = each.key + + allow_auto_merge = try(each.value.allow_auto_merge, null) + allow_merge_commit = try(each.value.allow_merge_commit, null) + allow_rebase_merge = try(each.value.allow_rebase_merge, null) + allow_squash_merge = try(each.value.allow_squash_merge, null) + archive_on_destroy = try(each.value.archive_on_destroy, null) + archived = try(each.value.archived, null) + auto_init = try(each.value.auto_init, null) + # default_branch = try(each.value.default_branch, null) + delete_branch_on_merge = try(each.value.delete_branch_on_merge, null) + description = try(each.value.description, null) + gitignore_template = try(each.value.gitignore_template, null) + has_downloads = try(each.value.has_downloads, null) + has_issues = try(each.value.has_issues, null) + has_projects = try(each.value.has_projects, null) + has_wiki = try(each.value.has_wiki, null) + homepage_url = try(each.value.homepage_url, null) + is_template = try(each.value.is_template, null) + license_template = try(each.value.license_template, null) + # private = try(each.value.private, null) + topics = try(each.value.topics, null) + visibility = try(each.value.visibility, null) + vulnerability_alerts = try(each.value.vulnerability_alerts, null) + + dynamic "pages" { + for_each = try(each.value.pages, []) + content { + cname = try(pages.value["cname"], null) + dynamic "source" { + for_each = pages.value["source"] + content { + branch = source.value["branch"] + path = try(source.value["path"], null) + } + } + } + } + dynamic "template" { + for_each = try(each.value.template, []) + content { + owner = template.value["owner"] + repository = template.value["repository"] + } + } + + lifecycle { + # @resources.repository.ignore_changes + ignore_changes = [ + allow_auto_merge, + allow_merge_commit, + allow_rebase_merge, + allow_squash_merge, + archive_on_destroy, + archived, + auto_init, + branches, + default_branch, + delete_branch_on_merge, + description, + etag, + full_name, + git_clone_url, + gitignore_template, + has_downloads, + has_issues, + has_projects, + has_wiki, + homepage_url, + html_url, + http_clone_url, + id, + is_template, + license_template, + node_id, + pages, + pages[0].cname, + pages[0].source[0].path, + private, + repo_id, + ssh_clone_url, + svn_url, + template, + topics, + visibility, + vulnerability_alerts + ] + } +} + +resource "github_repository_collaborator" "this" { + for_each = merge([ + for repository, collaborators in lookup(local.github, "repository_collaborator", {}) : + { + for username, collaborator in collaborators : + "${repository}${local.separator}${username}" => merge({ + repository = repository + username = username + }, collaborator) + } + ]...) + + depends_on = [github_repository.this] + + # @resources.repository_collaborator.required + repository = each.value.repository + # @resources.repository_collaborator.required + username = each.value.username + + permission = try(each.value.permission, null) + permission_diff_suppression = try(each.value.permission_diff_suppression, null) + + lifecycle { + # @resources.repository_collaborator.ignore_changes + ignore_changes = [ + id, + invitation_id, + permission, + permission_diff_suppression + ] + } +} + +resource "github_branch_protection" "this" { + for_each = merge([ + for repository, branch_protection_rules in lookup(local.github, "branch_protection", {}) : + { + for pattern, branch_protection_rule in branch_protection_rules : + "${repository}${local.separator}${pattern}" => merge({ + pattern = pattern + repository_id = github_repository.this[repository].node_id + }, branch_protection_rule) + } + ]...) + + # @resources.branch_protection.required + pattern = each.value.pattern + # @resources.branch_protection.required + repository_id = each.value.repository_id + + allows_deletions = try(each.value.allows_deletions, null) + allows_force_pushes = try(each.value.allows_force_pushes, null) + enforce_admins = try(each.value.enforce_admins, null) + push_restrictions = try(each.value.push_restrictions, null) + require_conversation_resolution = try(each.value.require_conversation_resolution, null) + require_signed_commits = try(each.value.require_signed_commits, null) + required_linear_history = try(each.value.required_linear_history, null) + + dynamic "required_pull_request_reviews" { + for_each = try(each.value.required_pull_request_reviews, []) + content { + dismiss_stale_reviews = try(required_pull_request_reviews.value["dismiss_stale_reviews"], null) + dismissal_restrictions = try(required_pull_request_reviews.value["dismissal_restrictions"], null) + require_code_owner_reviews = try(required_pull_request_reviews.value["require_code_owner_reviews"], null) + required_approving_review_count = try(required_pull_request_reviews.value["required_approving_review_count"], null) + restrict_dismissals = try(required_pull_request_reviews.value["restrict_dismissals"], null) + } + } + dynamic "required_status_checks" { + for_each = try(each.value.required_status_checks, []) + content { + contexts = try(required_status_checks.value["contexts"], null) + strict = try(required_status_checks.value["strict"], null) + } + } + + lifecycle { + # @resources.branch_protection.ignore_changes + ignore_changes = [ + allows_deletions, + allows_force_pushes, + enforce_admins, + id, + push_restrictions, + require_conversation_resolution, + require_signed_commits, + required_linear_history, + required_pull_request_reviews, + required_pull_request_reviews[0].dismiss_stale_reviews, + required_pull_request_reviews[0].dismissal_restrictions, + required_pull_request_reviews[0].require_code_owner_reviews, + required_pull_request_reviews[0].required_approving_review_count, + required_pull_request_reviews[0].restrict_dismissals, + required_status_checks, + required_status_checks[0].contexts, + required_status_checks[0].strict + ] + } +} + +resource "github_team" "this" { + for_each = lookup(local.github, "team", {}) + + # @resources.team.required + name = each.key + + create_default_maintainer = try(each.value.create_default_maintainer, null) + description = try(each.value.description, null) + ldap_dn = try(each.value.ldap_dn, null) + parent_team_id = try(each.value.parent_team_id, null) + privacy = try(each.value.privacy, null) + + lifecycle { + # @resources.team.ignore_changes + ignore_changes = [ + id, + create_default_maintainer, + description, + etag, + ldap_dn, + members_count, + node_id, + parent_team_id, + privacy, + slug + ] + } +} + +resource "github_team_repository" "this" { + for_each = merge([ + for team, repositories in lookup(local.github, "team_repository", {}) : + { + for repository, config in repositories : + "${team}${local.separator}${repository}" => merge({ + repository = repository + team_id = github_team.this[team].id + }, config) + } + ]...) + + depends_on = [ + github_repository.this + ] + + # @resources.team_repository.required + repository = each.value.repository + # @resources.team_repository.required + team_id = each.value.team_id + + permission = try(each.value.permission, null) + + lifecycle { + # @resources.team_repository.ignore_changes + ignore_changes = [ + etag, + id, + permission + ] + } +} + +resource "github_team_membership" "this" { + for_each = merge([ + for team, members in lookup(local.github, "team_membership", {}) : + { + for member, config in members : + "${team}${local.separator}${member}" => merge({ + team_id = github_team.this[team].id + username = member + }, config) + } + ]...) + + # @resources.team_membership.required + team_id = each.value.team_id + # @resources.team_membership.required + username = each.value.username + + role = try(each.value.role, null) + + lifecycle { + # @resources.team_membership.ignore_changes + ignore_changes = [ + etag, + id, + role + ] + } +} diff --git a/terraform/resources_override.tf b/terraform/resources_override.tf new file mode 100644 index 0000000..ed5aaff --- /dev/null +++ b/terraform/resources_override.tf @@ -0,0 +1,24 @@ +resource "github_branch_protection" "this" { + lifecycle { + # @resources.branch_protection.ignore_changes + ignore_changes = [ + allows_deletions, + allows_force_pushes, + enforce_admins, + id, + push_restrictions, + require_conversation_resolution, + require_signed_commits, + required_linear_history, + # required_pull_request_reviews, + required_pull_request_reviews[0].dismiss_stale_reviews, + required_pull_request_reviews[0].dismissal_restrictions, + required_pull_request_reviews[0].require_code_owner_reviews, + required_pull_request_reviews[0].required_approving_review_count, + required_pull_request_reviews[0].restrict_dismissals, + # required_status_checks, + # required_status_checks[0].contexts, + # required_status_checks[0].strict + ] + } +} diff --git a/terraform/terraform.tf b/terraform/terraform.tf new file mode 100644 index 0000000..00b91e7 --- /dev/null +++ b/terraform/terraform.tf @@ -0,0 +1,12 @@ +terraform { + backend "s3" {} + + required_providers { + github = { + source = "integrations/github" + version = "4.19.2" + } + } + + required_version = "~> 1.1.4" +} diff --git a/terraform/terraform_override.tf b/terraform/terraform_override.tf new file mode 100644 index 0000000..a929d7a --- /dev/null +++ b/terraform/terraform_override.tf @@ -0,0 +1,9 @@ +terraform { + backend "s3" { + region = "us-west-2" + bucket = "github-mgmt" + key = "terraform.tfstate" + workspace_key_prefix = "org" + dynamodb_table = "github-mgmt" + } +} diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000..ee53007 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,5 @@ +variable "write_delay_ms" { + description = "Amount of time in milliseconds to sleep in between writes to GitHub API." + type = number + default = 1000 +}