diff --git a/.github/composite/build-image/action.yml b/.github/composite/build-image/action.yml index 292eee1bd..b911dd740 100644 --- a/.github/composite/build-image/action.yml +++ b/.github/composite/build-image/action.yml @@ -2,58 +2,23 @@ name: "Build & Upload Docker Image" description: "Build & (optionally) upload Docker Image to Docker Registry" inputs: - GPG_PRIVATE_KEY: - description: "GPG Private Key" - required: true - GPG_PASSPHRASE: - description: "GPG Passphrase" - required: true DOCKER_UPLOAD: description: "Boolean indicating whether the image should be uploaded to Docker registry or not." required: false - default: true - TAG_PREFIX: - description: "Docker tags prefix" - required: false - SERVER_PROFILES: - description: "Profile(s) to apply to Codebloom instance." - required: false - default: prod + default: "true" + ENVIRONMENT: + description: "'staging' or 'production'" + required: true + +outputs: + tag: + description: "Built Docker image tag (git SHA, with optional prefix)" + value: ${{ steps.build-image.outputs.tag }} runs: using: "composite" steps: - - name: Setup CI - uses: ./.github/composite/setup-ci - with: - GPG_PRIVATE_KEY: ${{ inputs.GPG_PRIVATE_KEY }} - GPG_PASSPHRASE: ${{ inputs.GPG_PASSPHRASE }} - - - name: Set up pnpm - uses: pnpm/action-setup@master - with: - version: 10.24.0 - cache: true - cache_dependency_path: js/pnpm-lock.yaml - package_json_file: js/package.json - - - name: Set up OpenJDK 25 - uses: actions/setup-java@v4 - with: - distribution: "temurin" - java-version: "25" - - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: | - ~/.m2 - ~/repository - key: ${{ github.job }}-${{ hashFiles('**/pom.xml') }} - - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v3 - - name: Run script + id: build-image shell: bash - run: bun run .github/scripts/build-image --tag-prefix=${{ inputs.TAG_PREFIX }} --docker-upload=${{ inputs.DOCKER_UPLOAD }} --server-profiles=${{ inputs.SERVER_PROFILES }} + run: bun run .github/scripts/build-image --environment "${{ inputs.ENVIRONMENT }}" --docker-upload=${{ inputs.DOCKER_UPLOAD }} --getGhaOutput=true diff --git a/.github/composite/build-image/internal/standup-bot/action.yml b/.github/composite/build-image/internal/standup-bot/action.yml index 45ef58317..41da4bbb7 100644 --- a/.github/composite/build-image/internal/standup-bot/action.yml +++ b/.github/composite/build-image/internal/standup-bot/action.yml @@ -13,6 +13,11 @@ inputs: required: false default: true +outputs: + tag: + description: "Built Docker image tag (git SHA)" + value: ${{ steps.build-image.outputs.tag }} + runs: using: "composite" steps: @@ -32,5 +37,6 @@ runs: uses: crazy-max/ghaction-github-runtime@v3 - name: Run script + id: build-image shell: bash - run: bun run .github/scripts/build-image/internal/standup-bot --docker-upload=${{ inputs.DOCKER_UPLOAD }} + run: bun run .github/scripts/build-image/internal/standup-bot --docker-upload=${{ inputs.DOCKER_UPLOAD }} --getGhaOutput=true diff --git a/.github/composite/deploy-standup-bot/action.yaml b/.github/composite/deploy-standup-bot/action.yaml new file mode 100644 index 000000000..ea18f44a2 --- /dev/null +++ b/.github/composite/deploy-standup-bot/action.yaml @@ -0,0 +1,27 @@ +name: Deploy Standup Bot Service +description: Builds standup-bot image and deploys to k8s manifests + +inputs: + ENVIRONMENT: + description: "'staging' or 'production'" + required: true + GPG_PRIVATE_KEY: + description: "GPG Private Key" + required: true + GPG_PASSPHRASE: + description: "GPG Passphrase" + required: true + +runs: + using: composite + steps: + - name: Build and push standup-bot image + id: build-image + uses: ./.github/composite/build-image/internal/standup-bot + with: + GPG_PRIVATE_KEY: ${{ inputs.GPG_PRIVATE_KEY }} + GPG_PASSPHRASE: ${{ inputs.GPG_PASSPHRASE }} + + - name: Deploy standup-bot image tag + shell: bash + run: bun run .github/scripts/deploy --environment "${{ inputs.ENVIRONMENT }}" --type standup-bot --newTagVersion "${{ steps.build-image.outputs.tag }}" diff --git a/.github/composite/deploy-web/action.yaml b/.github/composite/deploy-web/action.yaml new file mode 100644 index 000000000..d50423b83 --- /dev/null +++ b/.github/composite/deploy-web/action.yaml @@ -0,0 +1,19 @@ +name: Deploy Web Service +description: Builds web image and deploys to k8s manifests + +inputs: + ENVIRONMENT: + description: "'staging' or 'production'" + required: true + +runs: + using: composite + steps: + - name: Build and push web image + id: build-image + shell: bash + run: bun run .github/scripts/build-image --environment "${{ inputs.ENVIRONMENT }}" --docker-upload=true --getGhaOutput=true + + - name: Deploy web image tag + shell: bash + run: bun run .github/scripts/deploy --environment "${{ inputs.ENVIRONMENT }}" --type web --newTagVersion "${{ steps.build-image.outputs.tag }}" diff --git a/.github/scripts/build-image/index.ts b/.github/scripts/build-image/index.ts index 964b18f27..521d8ac15 100644 --- a/.github/scripts/build-image/index.ts +++ b/.github/scripts/build-image/index.ts @@ -1,3 +1,5 @@ +import type { Environment } from "types"; + import { $ } from "bun"; import { getEnvVariables } from "load-secrets/env/load"; import { backend } from "utils/run-backend-instance"; @@ -7,25 +9,34 @@ import { hideBin } from "yargs/helpers"; process.env.TZ = "America/New_York"; -const { tagPrefix, dockerUpload, serverProfiles } = await yargs( - hideBin(process.argv), -) - .option("tagPrefix", { - type: "string", - demandOption: true, - }) - .option("dockerUpload", { - type: "boolean", - default: false, - demandOption: true, - }) - .option("serverProfiles", { - type: "string", - default: "prod", - demandOption: true, - }) - .strict() - .parse(); +const { environment, dockerUpload, getGhaOutput, githubOutputFile } = + await yargs(hideBin(process.argv)) + .option("environment", { + choices: ["staging", "production"] satisfies Environment[], + describe: "Deployment environment (staging or production)", + demandOption: true, + }) + .option("dockerUpload", { + type: "boolean", + default: false, + demandOption: true, + }) + .option("getGhaOutput", { + type: "boolean", + describe: + "Enable GitHub Actions output to receive latest built tag version", + default: false, + }) + .option("githubOutputFile", { + type: "string", + describe: "Path to GITHUB_OUTPUT (passed in automatically in CI)", + default: process.env.GITHUB_OUTPUT, + }) + .strict() + .parse(); + +const tagPrefix = environment === "staging" ? "staging-" : ""; +const serverProfiles = environment === "staging" ? "stg" : "prod"; async function main() { try { @@ -103,6 +114,14 @@ async function main() { .`; console.log("Image pushed successfully."); + + if (getGhaOutput && githubOutputFile) { + console.log("Outputting image tag..."); + const w = Bun.file(githubOutputFile).writer(); + await w.write(`tag<) { diff --git a/.github/scripts/bun.lock b/.github/scripts/bun.lock index faa94ddd9..b2f8e3114 100644 --- a/.github/scripts/bun.lock +++ b/.github/scripts/bun.lock @@ -14,7 +14,9 @@ "coolify": "https://github.com/tahminator/Coolify-TypeScript-SDK#8e19e18635ea2b095a9c39c574d22eefd938441f", "discord.js": "^14.25.1", "octokit": "^5.0.5", + "yaml": "^2.8.2", "yargs": "^18.0.0", + "zod": "^4.3.6", }, "devDependencies": { "@eslint/js": "^9.39.2", @@ -350,7 +352,7 @@ "concat-map": ["concat-map@0.0.1", "", {}, "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg=="], - "coolify": ["coolify@github:tahminator/Coolify-TypeScript-SDK#8e19e18", { "peerDependencies": { "zod": ">= 3" } }, "tahminator-Coolify-TypeScript-SDK-8e19e18"], + "coolify": ["coolify@github:tahminator/Coolify-TypeScript-SDK#8e19e18", { "peerDependencies": { "zod": ">= 3" } }, "tahminator-Coolify-TypeScript-SDK-8e19e18", "sha512-UkMpLpeVnJkqOys/wBiTdxIHnvtJxMkMQqK5/7iBZVzOZoQWVLNeW0s0ycDMdyCMDdeBmfAbdBssQsQFW4j0aQ=="], "core-util-is": ["core-util-is@1.0.3", "", {}, "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="], @@ -692,6 +694,8 @@ "y18n": ["y18n@5.0.8", "", {}, "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="], + "yaml": ["yaml@2.8.2", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A=="], + "yargs": ["yargs@18.0.0", "", { "dependencies": { "cliui": "^9.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "string-width": "^7.2.0", "y18n": "^5.0.5", "yargs-parser": "^22.0.0" } }, "sha512-4UEqdc2RYGHZc7Doyqkrqiln3p9X2DZVxaGbwhn2pi7MrRagKaOcIKe8L3OxYcbhXLgLFUS3zAYuQjKBQgmuNg=="], "yargs-parser": ["yargs-parser@22.0.0", "", {}, "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw=="], diff --git a/.github/scripts/deploy/index.ts b/.github/scripts/deploy/index.ts new file mode 100644 index 000000000..0efeb70f8 --- /dev/null +++ b/.github/scripts/deploy/index.ts @@ -0,0 +1,70 @@ +import type { Environment, Type } from "types"; + +import { getEnvVariables } from "load-secrets/env/load"; +import { updateK8sTagWithPR } from "utils/create-k8s-pr"; +import yargs from "yargs"; +import { hideBin } from "yargs/helpers"; + +const { environment, newTagVersion, type } = await yargs(hideBin(process.argv)) + .option("newTagVersion", { + type: "string", + demandOption: true, + }) + .option("environment", { + choices: ["staging", "production"] satisfies Environment[], + describe: "Deployment environment (staging or production)", + demandOption: true, + }) + .option("type", { + choices: ["web", "standup-bot"] satisfies Type[], + describe: "Service type to deploy", + demandOption: true, + }) + .strict() + .parse(); + +async function main() { + const ciEnv = await getEnvVariables(["ci"]); + const { githubPat } = parseCiEnv(ciEnv); + + if (type === "web") { + await updateK8sTagWithPR({ + githubPat, + kustomizationFilePath: `apps/${environment}/codebloom/kustomization.yaml`, + imageName: "docker.io/tahminator/codebloom", + newTag: newTagVersion, + environment, + }); + } + + if (type === "standup-bot") { + await updateK8sTagWithPR({ + githubPat, + kustomizationFilePath: `apps/${environment}/codebloom-standup-bot/kustomization.yaml`, + imageName: "docker.io/tahminator/codebloom-standup-bot", + newTag: newTagVersion, + environment, + }); + } +} + +function parseCiEnv(ciEnv: Record) { + const githubPat = (() => { + const v = ciEnv["GITHUB_PAT"]; + if (!v) { + throw new Error("Missing GITHUB_PAT from .env.ci"); + } + return v; + })(); + + return { githubPat }; +} + +main() + .then(() => { + process.exit(0); + }) + .catch((e) => { + console.error(e); + process.exit(1); + }); diff --git a/.github/scripts/package.json b/.github/scripts/package.json index 166da1f08..c6aeeb125 100644 --- a/.github/scripts/package.json +++ b/.github/scripts/package.json @@ -26,7 +26,9 @@ "coolify": "https://github.com/tahminator/Coolify-TypeScript-SDK#8e19e18635ea2b095a9c39c574d22eefd938441f", "discord.js": "^14.25.1", "octokit": "^5.0.5", - "yargs": "^18.0.0" + "yaml": "^2.8.2", + "yargs": "^18.0.0", + "zod": "^4.3.6" }, "devDependencies": { "typescript": "^5.9.0", diff --git a/.github/scripts/types.ts b/.github/scripts/types.ts index 19d0f4f8b..5981d3489 100644 --- a/.github/scripts/types.ts +++ b/.github/scripts/types.ts @@ -1,3 +1,5 @@ export type Environment = "staging" | "production"; export type Location = "backend" | "frontend"; + +export type Type = "web" | "standup-bot"; diff --git a/.github/scripts/utils/create-k8s-pr.ts b/.github/scripts/utils/create-k8s-pr.ts new file mode 100644 index 000000000..276408a23 --- /dev/null +++ b/.github/scripts/utils/create-k8s-pr.ts @@ -0,0 +1,140 @@ +import type { Environment } from "types"; + +import { Octokit } from "octokit"; +import { generateShortId } from "utils/short-id"; +import yaml from "yaml"; +import { z } from "zod"; + +const MANIFEST_REPO = "k8s-personal"; +const MANIFEST_REPO_OWNER = "tahminator"; + +// this should point to this repo name +const ORIGIN = "tahminator/codebloom"; + +const kustomizeSchema = z.object({ + kind: z.literal("Kustomization"), + images: z + .array( + z.object({ + name: z.string(), + newTag: z.string(), + }), + ) + .optional(), +}); + +/** + * + * Update k8s manifest repo with new tag version. + * + * @note `kustomizationFile` must look like this: + * + * ```yaml + * apiVersion: kustomize.config.k8s.io/v1beta1 + * kind: Kustomization + * resources: + * - deployment.yaml + * - secrets.yaml + * - service.yaml + * - monitor.yaml + * commonLabels: + * app: codebloom + * environment: production + * # This part specifically + * images: + * - name: docker.io/tahminator/codebloom + * newTag: a70ee0e + * ``` + */ +export async function updateK8sTagWithPR({ + githubPat, + newTag, + imageName, + kustomizationFilePath, + environment, +}: { + githubPat: string; + newTag: string; + imageName: string; + kustomizationFilePath: string; + environment: Environment; +}) { + const client = new Octokit({ + auth: githubPat, + }); + + const newBranchName = `${imageName}-${newTag}-${generateShortId()}`; + + const { data: repo } = await client.rest.repos.get({ + owner: MANIFEST_REPO_OWNER, + repo: MANIFEST_REPO, + }); + + const baseBranch = repo.default_branch; + + const { data: ref } = await client.rest.git.getRef({ + owner: MANIFEST_REPO_OWNER, + repo: MANIFEST_REPO, + ref: `heads/${baseBranch}`, + }); + + await client.rest.git.createRef({ + owner: MANIFEST_REPO_OWNER, + repo: MANIFEST_REPO, + ref: `refs/heads/${newBranchName}`, + sha: ref.object.sha, + }); + + const { data: file } = await client.rest.repos.getContent({ + owner: MANIFEST_REPO_OWNER, + repo: MANIFEST_REPO, + path: kustomizationFilePath, + ref: newBranchName, + }); + + if (Array.isArray(file)) throw new Error("Unexpected file shape found"); + if (!file) throw new Error("Kustomization file not found"); + if (file.type !== "file") throw new Error("Unexpected file type found"); + if (!file.content) throw new Error("Kustomization file is empty"); + + const currentYaml = Buffer.from(file.content ?? "", "base64").toString(); + const doc = yaml.parseDocument(currentYaml); + const yamlObj = kustomizeSchema.parse(doc.toJS()); + + const targetImage = yamlObj.images?.find((img) => img.name === imageName); + if (!targetImage) { + console.debug(yamlObj); + throw new Error("Target image could not be found."); + } + + targetImage.newTag = newTag; + doc.set("images", yamlObj.images); + + const updatedYaml = doc.toString(); + + await client.rest.repos.createOrUpdateFileContents({ + owner: MANIFEST_REPO_OWNER, + repo: MANIFEST_REPO, + path: kustomizationFilePath, + message: `deploy: update ${imageName} to ${newTag}`, + content: Buffer.from(updatedYaml).toString("base64"), + sha: file.sha, + branch: newBranchName, + }); + + const { data: pr } = await client.rest.pulls.create({ + owner: MANIFEST_REPO_OWNER, + repo: MANIFEST_REPO, + title: `Deploying ${newTag} for ${imageName} in ${environment}`, + head: newBranchName, + base: baseBranch, + body: `Automated image tag change to ${newTag} for ${imageName} in ${environment} triggered by [${ORIGIN}](https://github.com/${ORIGIN}).`, + }); + + await client.rest.pulls.merge({ + owner: MANIFEST_REPO_OWNER, + repo: MANIFEST_REPO, + pull_number: pr.number, + merge_method: "squash", + }); +} diff --git a/.github/scripts/utils/short-id.ts b/.github/scripts/utils/short-id.ts new file mode 100644 index 000000000..c603f6d68 --- /dev/null +++ b/.github/scripts/utils/short-id.ts @@ -0,0 +1,10 @@ +/** + * Generate a short ID (default len = 7). + * + * @note THIS DOES NOT GUARANTEE EXTREMELY LOW COLLISIONS. + */ +export function generateShortId(len = 7) { + return Math.random() + .toString(36) + .substring(2, 2 + len); +} diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index ac570ae1f..519e18c63 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -90,13 +90,42 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Run workflow - uses: ./.github/composite/build-image + - name: Setup CI + uses: ./.github/composite/setup-ci with: GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - TAG_PREFIX: test- - DOCKER_UPLOAD: false + + - name: Set up pnpm + uses: pnpm/action-setup@master + with: + version: 10.24.0 + cache: true + cache_dependency_path: js/pnpm-lock.yaml + package_json_file: js/package.json + + - name: Set up OpenJDK 25 + uses: actions/setup-java@v4 + with: + distribution: "temurin" + java-version: "25" + + - name: Cache Maven packages + uses: actions/cache@v5 + with: + path: | + ~/.m2 + ~/repository + key: ${{ github.job }}-${{ hashFiles('**/pom.xml') }} + + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v3 + + - name: Run workflow + uses: ./.github/composite/build-image + with: + ENVIRONMENT: staging + DOCKER_UPLOAD: "false" testBuildStandupBotImage: name: Build Test Docker Image for codebloom-standup-bot @@ -130,10 +159,11 @@ jobs: GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - buildImage: - name: Build Docker Image & Upload to Registry + deployWeb: + name: Deploy Web to Production runs-on: ubuntu-latest needs: [validateDBSchema, backendTests, frontendTests] + environment: production if: github.ref_name == 'main' @@ -141,60 +171,57 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Run workflow - uses: ./.github/composite/build-image + - name: Setup CI + uses: ./.github/composite/setup-ci with: GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - buildStandupBotImage: - name: Build codebloom-standup-bot Docker Image & Upload to Registry - runs-on: ubuntu-latest - - if: github.ref_name == 'main' - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Run workflow - uses: ./.github/composite/build-image/internal/standup-bot + - name: Set up pnpm + uses: pnpm/action-setup@master with: - GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} - GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + version: 10.24.0 + cache: true + cache_dependency_path: js/pnpm-lock.yaml + package_json_file: js/package.json - redeploy: - name: Redeploy on DigitalOcean - runs-on: ubuntu-latest - needs: buildImage - environment: production + - name: Set up OpenJDK 25 + uses: actions/setup-java@v4 + with: + distribution: "temurin" + java-version: "25" - if: github.ref_name == 'main' + - name: Cache Maven packages + uses: actions/cache@v5 + with: + path: | + ~/.m2 + ~/repository + key: ${{ github.job }}-${{ hashFiles('**/pom.xml') }} - steps: - - name: Checkout Repository - uses: actions/checkout@v4 + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v3 - - name: Run workflow - uses: ./.github/composite/redeploy + - name: Deploy Web Service + uses: ./.github/composite/deploy-web with: - GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} - GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + ENVIRONMENT: production - redeployStandupBot: - name: Redeploy codebloom-standup-bot on Coolify + deployStandupBot: + name: Deploy Standup Bot to Production runs-on: ubuntu-latest - needs: buildStandupBotImage + needs: [validateDBSchema, backendTests, frontendTests] environment: production if: github.ref_name == 'main' steps: - - name: Checkout Repository + - name: Checkout repository uses: actions/checkout@v4 - - name: Run workflow - uses: ./.github/composite/redeploy/internal/standup-bot + - name: Deploy Standup Bot Service + uses: ./.github/composite/deploy-standup-bot with: + ENVIRONMENT: production GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} diff --git a/.github/workflows/deploy-command.yml b/.github/workflows/deploy-command.yml index 2d4ba363c..65688ade2 100644 --- a/.github/workflows/deploy-command.yml +++ b/.github/workflows/deploy-command.yml @@ -209,13 +209,41 @@ jobs: with: ref: ${{ needs.getPRHead.outputs.sha }} - - name: Run workflow - uses: ./.github/composite/build-image + - name: Setup CI + uses: ./.github/composite/setup-ci with: GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - TAG_PREFIX: staging- - SERVER_PROFILES: stg + + - name: Set up pnpm + uses: pnpm/action-setup@master + with: + version: 10.24.0 + cache: true + cache_dependency_path: js/pnpm-lock.yaml + package_json_file: js/package.json + + - name: Set up OpenJDK 25 + uses: actions/setup-java@v4 + with: + distribution: "temurin" + java-version: "25" + + - name: Cache Maven packages + uses: actions/cache@v5 + with: + path: | + ~/.m2 + ~/repository + key: ${{ github.job }}-${{ hashFiles('**/pom.xml') }} + + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v3 + + - name: Run workflow + uses: ./.github/composite/build-image + with: + ENVIRONMENT: staging redeploy: name: Redeploy on DigitalOcean