From b208d5619f2444e2a7276c431651610908c9c546 Mon Sep 17 00:00:00 2001 From: Puneet Punamiya Date: Fri, 24 Jul 2026 20:02:43 +0530 Subject: [PATCH 1/2] Adds release script to do the minor or patch release --- .github/workflows/release.yml | 77 +++++++ hack/release.sh | 405 ++++++++++++++++++++++++++++++++++ 2 files changed, 482 insertions(+) create mode 100644 .github/workflows/release.yml create mode 100755 hack/release.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..4b812d4a6 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,77 @@ +name: Release + +on: + push: + tags: + - "v*" + +permissions: + contents: write + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + MCP_IMAGE_NAME: ${{ github.repository }}-mcp-server + +jobs: + release: + name: Create Draft Release + runs-on: ubuntu-latest + steps: + - name: Validate tag format + run: | + TAG="${{ github.ref_name }}" + if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::Invalid tag format: ${TAG}. Expected vMAJOR.MINOR.PATCH (e.g., v0.12.0)" + exit 1 + fi + + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Determine release type + id: info + run: | + TAG="${{ github.ref_name }}" + PATCH_NUM="${TAG##*.}" + MINOR_VERSION="${TAG%.*}" + if [[ "$PATCH_NUM" == "0" ]]; then + echo "type=minor" >> "$GITHUB_OUTPUT" + else + echo "type=patch" >> "$GITHUB_OUTPUT" + fi + echo "minor_version=${MINOR_VERSION}" >> "$GITHUB_OUTPUT" + + - name: Run release script (minor) + if: steps.info.outputs.type == 'minor' + env: + GH_TOKEN: ${{ github.token }} + run: | + chmod +x hack/release.sh + hack/release.sh minor --version "${{ github.ref_name }}" --force + + - name: Run release script (patch) + if: steps.info.outputs.type == 'patch' + env: + GH_TOKEN: ${{ github.token }} + run: | + chmod +x hack/release.sh + hack/release.sh patch "${{ steps.info.outputs.minor_version }}" --version "${{ github.ref_name }}" --force + + - name: Summary + run: | + TAG="${{ github.ref_name }}" + { + echo "## Draft Release ${TAG}" + echo "" + echo "- **Release**: https://github.com/${{ github.repository }}/releases/tag/${TAG}" + echo "- **Controller image**: \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG}\`" + echo "- **MCP server image**: \`${{ env.REGISTRY }}/${{ env.MCP_IMAGE_NAME }}:${TAG}\`" + if [[ "${{ steps.info.outputs.type }}" == "minor" ]]; then + echo "- **Release branch**: \`release-${{ steps.info.outputs.minor_version }}.x\`" + fi + echo "" + echo "> **Note:** This is a draft release. Review the release notes and publish when ready." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/hack/release.sh b/hack/release.sh new file mode 100755 index 000000000..af6f4931e --- /dev/null +++ b/hack/release.sh @@ -0,0 +1,405 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO="" +IMAGE_BASE="" +UPSTREAM_REMOTE="origin" +DRY_RUN=false +FORCE=false +VERSION_OVERRIDE="" + +# --- Colors --- +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +BOLD='\033[1m' +NC='\033[0m' + +info() { echo -e "${BLUE}ℹ${NC} $*"; } +success() { echo -e "${GREEN}✔${NC} $*"; } +warn() { echo -e "${YELLOW}⚠${NC} $*"; } +error() { echo -e "${RED}✖${NC} $*" >&2; } +header() { echo -e "\n${BOLD}${CYAN}$*${NC}"; } +dry_run() { echo -e " ${YELLOW}[DRY RUN]${NC} $*"; } + +run_cmd() { + if $DRY_RUN; then + dry_run "$*" + else + info "Running: $*" + eval "$@" + fi +} + +usage() { + cat </dev/null; then + error "gh CLI not found. Install it: https://cli.github.com" + exit 1 + fi + if ! gh auth status &>/dev/null 2>&1; then + error "gh CLI not authenticated. Run: gh auth login" + exit 1 + fi + success "gh CLI authenticated" + + if ! git remote get-url "$UPSTREAM_REMOTE" &>/dev/null; then + error "Remote '${UPSTREAM_REMOTE}' not found." + echo " Add it with: git remote add ${UPSTREAM_REMOTE} git@github.com:/.git" + exit 1 + fi + + local remote_url + remote_url=$(git remote get-url "$UPSTREAM_REMOTE") + REPO=$(echo "$remote_url" | sed -E 's#.*github\.com[:/]##' | sed 's/\.git$//') + IMAGE_BASE="ghcr.io/${REPO}" + success "Remote '${UPSTREAM_REMOTE}': ${remote_url}" + success "Repo: ${REPO}" + + info "Fetching ${UPSTREAM_REMOTE}..." + git fetch "$UPSTREAM_REMOTE" --tags --force --quiet + success "Fetched latest refs and tags" +} + +# --- Version helpers --- +get_all_tags() { + git tag -l 'v*' | sort -V +} + +get_latest_minor() { + get_all_tags | { grep -oP 'v\K[0-9]+\.[0-9]+' || true; } | sort -t. -k1,1n -k2,2n | tail -1 +} + +get_latest_patch_for_minor() { + local minor="$1" + get_all_tags | { grep -P "^v${minor}\.[0-9]+$" || true; } | sort -V | tail -1 +} + +get_max_patch_number() { + local minor="$1" + get_all_tags | { grep -P "^v${minor}\.[0-9]+$" || true; } | { grep -oP '\.[0-9]+$' || true; } | tr -d '.' | sort -n | tail -1 +} + +get_latest_tag() { + get_all_tags | tail -1 +} + +# --- Minor release --- +do_minor() { + preflight + + header "Version detection" + + local latest_minor + latest_minor=$(get_latest_minor) + if [[ -z "$latest_minor" ]]; then + error "No existing tags found. Cannot determine next version." + exit 1 + fi + + local next_version next_minor_num release_branch prev_release_branch prev_tag + + if [[ -n "$VERSION_OVERRIDE" ]]; then + next_version="${VERSION_OVERRIDE}" + next_minor_num=$(echo "$next_version" | grep -oP 'v0\.\K[0-9]+') + release_branch="release-v0.${next_minor_num}.x" + + local prev_minor_num=$((next_minor_num - 1)) + prev_release_branch="release-v0.${prev_minor_num}.x" + prev_tag=$(get_latest_patch_for_minor "0.${prev_minor_num}") + else + local latest_minor_num + latest_minor_num=$(echo "$latest_minor" | cut -d. -f2) + next_minor_num=$((latest_minor_num + 1)) + next_version="v0.${next_minor_num}.0" + release_branch="release-v0.${next_minor_num}.x" + + prev_release_branch="release-v0.${latest_minor_num}.x" + prev_tag=$(get_latest_patch_for_minor "0.${latest_minor_num}") + fi + + info "Latest minor: v${latest_minor}" + info "Previous release branch: ${prev_release_branch}" + info "Latest tag on branch: ${prev_tag:-none}" + info "Next version: ${BOLD}${next_version}${NC}" + + local branch_exists=false + if git rev-parse "${UPSTREAM_REMOTE}/${release_branch}" &>/dev/null 2>&1; then + branch_exists=true + fi + + if ! git rev-parse "$next_version" &>/dev/null 2>&1; then + error "Tag ${next_version} does not exist. Create and push the tag first." + exit 1 + fi + + + local commits_since + if git rev-parse "${UPSTREAM_REMOTE}/${prev_release_branch}" &>/dev/null 2>&1; then + commits_since="${UPSTREAM_REMOTE}/${prev_release_branch}" + elif [[ -n "$prev_tag" ]]; then + commits_since="${prev_tag}" + fi + + if [[ -n "${commits_since:-}" ]]; then + local total_commits + total_commits=$(git rev-list --count "${commits_since}..${UPSTREAM_REMOTE}/main") + + if (( total_commits == 0 )); then + echo "" + warn "No new commits on ${UPSTREAM_REMOTE}/main since ${commits_since}" + if ! $FORCE; then + error "Nothing to release. Use --force to proceed anyway." + exit 1 + fi + else + header "Commits included (${commits_since}..${UPSTREAM_REMOTE}/main) — ${total_commits} commits" + git log --oneline "${commits_since}..${UPSTREAM_REMOTE}/main" | head -30 + if (( total_commits > 30 )); then + echo " ... and $((total_commits - 30)) more commits" + fi + echo "" + fi + fi + + if $DRY_RUN; then + header "Release plan (DRY RUN — no changes will be made)" + echo "" + dry_run "git branch ${release_branch} ${next_version} # create release branch" + dry_run "git push ${UPSTREAM_REMOTE} ${release_branch} # push release branch" + dry_run "gh release create ${next_version} --repo ${REPO} --generate-notes --draft --notes-start-tag ${prev_tag:-v0.0.0}" + echo "" + info "Images that will be built by CI:" + echo " ${IMAGE_BASE}:${next_version}" + echo " ${IMAGE_BASE}-mcp-server:${next_version}" + echo "" + info "To execute, run: $(basename "$0") minor" + return + fi + + header "Creating minor release ${next_version}" + + if $branch_exists; then + success "Branch ${release_branch} already exists — skipping" + else + run_cmd git branch "$release_branch" "$next_version" + run_cmd git push "$UPSTREAM_REMOTE" "$release_branch" + success "Created and pushed branch ${release_branch}" + fi + + if gh release view "$next_version" --repo "$REPO" &>/dev/null 2>&1; then + success "GitHub release ${next_version} already exists — skipping" + else + header "Creating draft GitHub release" + local notes_flag="" + if [[ -n "$prev_tag" ]]; then + notes_flag="--notes-start-tag ${prev_tag}" + fi + run_cmd gh release create "$next_version" \ + --repo "$REPO" \ + --generate-notes \ + --draft \ + $notes_flag + fi + + echo "" + success "Draft release ${next_version} created!" + info "Images:" + echo " ${IMAGE_BASE}:${next_version}" + echo " ${IMAGE_BASE}-mcp-server:${next_version}" + info "Release: https://github.com/${REPO}/releases/tag/${next_version}" +} + +# --- Patch release --- +do_patch() { + local minor_input="$1" + local minor + minor=$(echo "$minor_input" | sed 's/^v//') + + if [[ ! "$minor" =~ ^[0-9]+\.[0-9]+$ ]]; then + error "Invalid minor version format: ${minor_input}" + echo " Expected format: v0.X or 0.X (e.g., v0.10 or 0.10)" + exit 1 + fi + + preflight + + header "Version detection" + + local release_branch="release-v${minor}.x" + if ! git rev-parse "${UPSTREAM_REMOTE}/${release_branch}" &>/dev/null 2>&1; then + error "Release branch ${UPSTREAM_REMOTE}/${release_branch} not found." + echo " Available release branches:" + git branch -r --list "${UPSTREAM_REMOTE}/release-*" | sed 's/^/ /' + exit 1 + fi + success "Release branch: ${UPSTREAM_REMOTE}/${release_branch}" + + local latest_patch_tag + latest_patch_tag=$(get_latest_patch_for_minor "$minor") + local max_patch + max_patch=$(get_max_patch_number "$minor") + + if [[ -z "$max_patch" ]]; then + error "No existing tags found for v${minor}.x" + exit 1 + fi + + local next_version + if [[ -n "$VERSION_OVERRIDE" ]]; then + next_version="${VERSION_OVERRIDE}" + local override_patch="${next_version##*.}" + if (( override_patch > 0 )); then + latest_patch_tag="v${minor}.$((override_patch - 1))" + fi + else + local next_patch=$((max_patch + 1)) + next_version="v${minor}.${next_patch}" + fi + + info "Existing v${minor}.x tags: $(get_all_tags | grep -P "^v${minor}\.[0-9]+$" | tr '\n' ' ')" + info "Latest patch: ${latest_patch_tag}" + info "Next version: ${BOLD}${next_version}${NC}" + + if ! git rev-parse "$next_version" &>/dev/null 2>&1; then + error "Tag ${next_version} does not exist. Create and push the tag first." + exit 1 + fi + + local new_commits + new_commits=$(git rev-list --count "${latest_patch_tag}..${UPSTREAM_REMOTE}/${release_branch}") + + echo "" + if (( new_commits == 0 )); then + warn "No new commits on ${release_branch} since ${latest_patch_tag}" + if ! $FORCE; then + error "Nothing to release. Use --force to proceed anyway." + exit 1 + fi + else + header "Cherry-picks on ${release_branch} since ${latest_patch_tag} (${new_commits} commits)" + git log --oneline "${latest_patch_tag}..${UPSTREAM_REMOTE}/${release_branch}" + echo "" + fi + + if $DRY_RUN; then + header "Release plan (DRY RUN — no changes will be made)" + echo "" + dry_run "gh release create ${next_version} --repo ${REPO} --generate-notes --draft --notes-start-tag ${latest_patch_tag}" + echo "" + info "Images that will be built by CI:" + echo " ${IMAGE_BASE}:${next_version}" + echo " ${IMAGE_BASE}-mcp-server:${next_version}" + echo "" + info "To execute, run: $(basename "$0") patch ${minor_input}" + return + fi + + header "Creating patch release ${next_version}" + + if gh release view "$next_version" --repo "$REPO" &>/dev/null 2>&1; then + success "GitHub release ${next_version} already exists — skipping" + else + header "Creating draft GitHub release" + run_cmd gh release create "$next_version" \ + --repo "$REPO" \ + --generate-notes \ + --draft \ + --notes-start-tag "$latest_patch_tag" + fi + + echo "" + success "Draft release ${next_version} created!" + info "Images:" + echo " ${IMAGE_BASE}:${next_version}" + echo " ${IMAGE_BASE}-mcp-server:${next_version}" + info "Release: https://github.com/${REPO}/releases/tag/${next_version}" +} + +# --- Argument parsing --- +COMMAND="" +PATCH_MINOR="" + +while [[ $# -gt 0 ]]; do + case "$1" in + minor) + COMMAND="minor" + shift + ;; + patch) + COMMAND="patch" + if [[ -z "${2:-}" ]]; then + error "patch requires a minor version argument (e.g., v0.10)" + echo " Usage: $(basename "$0") patch v0.X [OPTIONS]" + exit 1 + fi + PATCH_MINOR="$2" + shift 2 + ;; + --dry-run) + DRY_RUN=true + shift + ;; + --upstream-remote) + UPSTREAM_REMOTE="${2:-}" + if [[ -z "$UPSTREAM_REMOTE" ]]; then + error "--upstream-remote requires a value" + exit 1 + fi + shift 2 + ;; + --version) + VERSION_OVERRIDE="${2:-}" + if [[ -z "$VERSION_OVERRIDE" ]]; then + error "--version requires a value (e.g., v0.11.0)" + exit 1 + fi + shift 2 + ;; + --force) + FORCE=true + shift + ;; + -h|--help) + usage + ;; + *) + error "Unknown argument: $1" + usage + ;; + esac +done + +if [[ -z "$COMMAND" ]]; then + usage +fi + +case "$COMMAND" in + minor) do_minor ;; + patch) do_patch "$PATCH_MINOR" ;; +esac From 04b0bf5adacba985f7401f1ff4bf9b283dcbb0e6 Mon Sep 17 00:00:00 2001 From: Puneet Punamiya Date: Wed, 8 Apr 2026 14:04:39 +0530 Subject: [PATCH 2/2] Add local development setup automation and documentation Add DEVELOPMENT.md guide covering the complete local development workflow including LocalStack setup, CRD installation, controller configuration, and e2e testing with required environment variables Introduce make targets (local-dev-setup, local-dev-cleanup) with corresponding shell scripts to automate local environment creation, including S3 buckets, SQS queues, and Docling service setup. Fix test-e2e target to pass KIND_CLUSTER and SKIP_CLUSTER_SETUP environment variables, preventing duplicate cluster creation and aligning local test execution with CI workflow. Signed-off-by: Puneet Punamiya --- DEVELOPMENT.md | 119 +++++++++++++++++++++++ Makefile | 10 +- scripts/local-dev-cleanup.sh | 49 ++++++++++ scripts/local-dev-setup.sh | 176 +++++++++++++++++++++++++++++++++++ scripts/localstack.sh | 161 ++++++++++++++++++++++++++++++++ 5 files changed, 514 insertions(+), 1 deletion(-) create mode 100644 DEVELOPMENT.md create mode 100755 scripts/local-dev-cleanup.sh create mode 100755 scripts/local-dev-setup.sh create mode 100755 scripts/localstack.sh diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 000000000..c9155911b --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,119 @@ +# Local Development Setup + +This guide walks you through setting up the Unstructured Data Controller for local development. + +## Prerequisites + +- Docker +- [Kind](https://kind.sigs.k8s.io/) +- kubectl +- Go +- [yq](https://github.com/mikefarah/yq) +- [AWS CLI](https://aws.amazon.com/cli/) with [awslocal](https://github.com/localstack/awscli-local) +- [Docling](https://github.com/docling-project/docling-serve) (`pip install "docling-serve[ui]"`) +- [Ollama](https://ollama.com/) (for embedding models) + +## Quick start (recommended) + +Edit these files with your credentials and config before running setup: + +- `config/samples/unstructured-secret.yaml` — S3 credentials, embedding endpoint, API keys +- `config/samples/operator_v1alpha1_controllerconfig.yaml` — Docling URL, storage bucket, concurrency + +Then run: + +```bash +make local-dev-setup +``` + +This single command: + +1. Creates a Kind cluster (`unstructured-data-controller-local`) +2. Creates namespace `unstructured-controller-namespace` +3. Creates the local cache directory +4. Starts Docling (if `docling-serve` is installed) +5. Starts Ollama and pulls the embedding model +6. Starts LocalStack in Docker on `localhost:4566` (container `localstack-dev`) +7. Creates S3 buckets (ingestion, storage, output) +8. Installs CRDs +9. Applies secrets (`operator-secret` and `pipeline-secret`), ControllerConfig, and UnstructuredDataPipeline +10. Runs the controller (`make run`) + +Press Ctrl+C to stop the controller. LocalStack and the Kind cluster keep running. + +### Subsequent runs + +If setup was already done and you only need to run the controller: + +```bash +make run +``` + +Ensure LocalStack is still running: + +```bash +curl -sf http://127.0.0.1:4566/_localstack/health +``` + +If not healthy, start it again: + +```bash +./scripts/localstack.sh start +``` + +### Cleanup + +Remove the full local environment (Docling, Ollama, LocalStack container, cache, Kind cluster): + +```bash +make local-dev-cleanup +``` + +## LocalStack only + +To start or manage LocalStack without the full setup: + +```bash +./scripts/localstack.sh start # start Docker container +./scripts/localstack.sh setup # create S3 buckets +./scripts/localstack.sh status # health check and resource list +./scripts/localstack.sh stop # stop and remove container +``` + +View LocalStack API requests: + +```bash +docker logs -f localstack-dev +``` + +See [Setting up LocalStack](docs/setup-localstack.md) for manual setup steps. + +## Manual setup + +If you prefer step-by-step setup instead of `make local-dev-setup`: + +1. [Setting up LocalStack](docs/setup-localstack.md) +2. [Unstructured Data Controller with LocalStack](docs/setup-unstructured-controller.md) + +## Verification + +```bash +kubectl get controllerconfig controllerconfig -n unstructured-controller-namespace -o yaml +kubectl get unstructureddatapipeline -n unstructured-controller-namespace -o wide +kubectl get sourcecrawler,documentprocessor,chunksgenerator,vectorembeddingsgenerator,destinationsyncer -n unstructured-controller-namespace +``` + +The ControllerConfig should show `ConfigReady` with `status: "True"`. Pipeline stages should appear after the pipeline is applied. + +## Next steps + +Follow the [Creating Sample File Guide](docs/creating-sample-file.md) to upload a test file and process it through the pipeline. + +## Running e2e tests + +E2e tests use in-cluster LocalStack (not the host Docker container). See the test workflow in `.github/workflows/test-e2e.yml`. + +```bash +export IMG=/: +make docker-build docker-push test-e2e +``` diff --git a/Makefile b/Makefile index 18ed3ee6f..9003e47bd 100644 --- a/Makefile +++ b/Makefile @@ -142,13 +142,21 @@ setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist .PHONY: test-e2e test-e2e: setup-test-e2e ## Run the e2e tests. Expected an isolated environment using Kind. - go test -count=1 -tags e2e ./test/e2e/ -v -timeout=60m + KIND_CLUSTER=$(KIND_CLUSTER) SKIP_CLUSTER_SETUP=true go test -count=1 -tags e2e ./test/e2e/ -v -timeout=60m $(MAKE) cleanup-test-e2e .PHONY: cleanup-test-e2e cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests @$(KIND) delete cluster --name $(KIND_CLUSTER) +.PHONY: local-dev-setup +local-dev-setup: ## Full local dev setup (Kind, LocalStack Docker, CRDs, samples, run) + @chmod +x scripts/local-dev-setup.sh && ./scripts/local-dev-setup.sh + +.PHONY: local-dev-cleanup +local-dev-cleanup: ## Clean up local development environment + @chmod +x scripts/local-dev-cleanup.sh && ./scripts/local-dev-cleanup.sh + .PHONY: lint lint: golangci-lint ## Run golangci-lint and yamllint linters $(GOLANGCI_LINT) run diff --git a/scripts/local-dev-cleanup.sh b/scripts/local-dev-cleanup.sh new file mode 100755 index 000000000..cc402d6db --- /dev/null +++ b/scripts/local-dev-cleanup.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Cleanup for local-dev-setup.sh +# Usage: make local-dev-cleanup +set -e + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOCAL_KIND_CLUSTER="${LOCAL_KIND_CLUSTER:-unstructured-data-controller-local}" + +echo "Cleaning up local development environment..." + +if [ -f /tmp/docling.pid ]; then + DOCLING_PID=$(cat /tmp/docling.pid) + if ps -p "${DOCLING_PID}" > /dev/null 2>&1; then + kill "${DOCLING_PID}" 2>/dev/null || true + fi + rm -f /tmp/docling.pid /tmp/docling.log + echo "✓ Stopped Docling" +fi + +if [ -f /tmp/ollama.pid ]; then + OLLAMA_PID=$(cat /tmp/ollama.pid) + if ps -p "${OLLAMA_PID}" > /dev/null 2>&1; then + kill "${OLLAMA_PID}" 2>/dev/null || true + fi + rm -f /tmp/ollama.pid /tmp/ollama.log + echo "✓ Stopped Ollama" +fi + +LOCALSTACK_CONTAINER="${LOCALSTACK_CONTAINER:-localstack-dev}" + +if docker ps -a --format '{{.Names}}' 2>/dev/null | grep -qx "${LOCALSTACK_CONTAINER}"; then + docker rm -f "${LOCALSTACK_CONTAINER}" >/dev/null + echo "✓ Stopped and removed LocalStack container '${LOCALSTACK_CONTAINER}'" +else + echo "✓ No LocalStack container '${LOCALSTACK_CONTAINER}' found" +fi + +CACHE_DIR="${REPO_ROOT}/$( + yq -r '.spec.cacheDirectory' \ + "${REPO_ROOT}/config/samples/operator_v1alpha1_controllerconfig.yaml" 2>/dev/null \ + | sed 's|/$||' || echo "tmp/cache" +)" +if [ -d "${CACHE_DIR}" ]; then + rm -rf "${CACHE_DIR}" + echo "✓ Removed cache directory ${CACHE_DIR}" +fi + +kind delete cluster --name "${LOCAL_KIND_CLUSTER}" 2>/dev/null || true +echo "✓ Local development environment removed" diff --git a/scripts/local-dev-setup.sh b/scripts/local-dev-setup.sh new file mode 100755 index 000000000..2c1a4ed08 --- /dev/null +++ b/scripts/local-dev-setup.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +# Usage: make local-dev-setup +# +# Before running, edit: +# config/samples/unstructured-secret.yaml — credentials +# config/samples/operator_v1alpha1_controllerconfig.yaml — operator config +set -e + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +LOCAL_KIND_CLUSTER="${LOCAL_KIND_CLUSTER:-unstructured-data-controller-local}" +LOCAL_NAMESPACE="${LOCAL_NAMESPACE:-unstructured-controller-namespace}" + +CONTROLLER_CONFIG_YAML="${REPO_ROOT}/config/samples/operator_v1alpha1_controllerconfig.yaml" +SOURCE_CRAWLER_YAML="${REPO_ROOT}/config/samples/operator_v1alpha1_sourcecrawler.yaml" +DEST_SYNCER_YAML="${REPO_ROOT}/config/samples/operator_v1alpha1_destinationsyncer.yaml" +PIPELINE_YAML="${REPO_ROOT}/config/samples/operator_v1alpha1_unstructureddatapipeline.yaml" + +echo "Setting up local development environment..." + +# 1. Create Kind cluster +echo "" +echo "1. Creating Kind cluster '${LOCAL_KIND_CLUSTER}'..." +if kind get clusters 2>/dev/null | grep -q "^${LOCAL_KIND_CLUSTER}$"; then + echo "✓ Kind cluster '${LOCAL_KIND_CLUSTER}' already exists" +else + kind create cluster --name "${LOCAL_KIND_CLUSTER}" + echo "✓ Kind cluster '${LOCAL_KIND_CLUSTER}' created" +fi + +# 2. Create namespace +echo "" +echo "2. Creating namespace '${LOCAL_NAMESPACE}'..." +if kubectl get namespace "${LOCAL_NAMESPACE}" &>/dev/null; then + echo "✓ Namespace '${LOCAL_NAMESPACE}' already exists" +else + kubectl create namespace "${LOCAL_NAMESPACE}" + echo "✓ Namespace '${LOCAL_NAMESPACE}' created" +fi + +# 3. Create cache directory (from ControllerConfig sample) +echo "" +echo "3. Creating local cache directory..." +mkdir -p "${REPO_ROOT}/$( + yq -r '.spec.cacheDirectory' "${CONTROLLER_CONFIG_YAML}" \ + | sed 's|/$||' +)" +echo "✓ Cache directory created" + +# 4. Check/Start Docling +echo "" +echo "4. Checking Docling service..." +if command -v docling-serve &>/dev/null; then + if curl -s http://localhost:5001/health &>/dev/null; then + echo "✓ Docling is already running" + else + echo "Starting Docling in the background..." + nohup docling-serve run --enable-ui > /tmp/docling.log 2>&1 & + echo $! > /tmp/docling.pid + sleep 2 + if curl -s http://localhost:5001/health &>/dev/null; then + echo "✓ Docling started (log: /tmp/docling.log)" + else + echo "Warning: Docling may still be starting. Check /tmp/docling.log" + fi + fi +else + echo "⚠ Docling is not installed. Start it manually or install docling-serve." +fi + +# 5. Check/Start Ollama (for embedding models) +echo "" +echo "5. Checking Ollama service..." +if command -v ollama &>/dev/null; then + if curl -s http://localhost:11434/api/tags &>/dev/null; then + echo "✓ Ollama is already running" + else + echo "Starting Ollama in the background..." + nohup ollama serve > /tmp/ollama.log 2>&1 & + echo $! > /tmp/ollama.pid + sleep 2 + if curl -s http://localhost:11434/api/tags &>/dev/null; then + echo "✓ Ollama started (log: /tmp/ollama.log)" + else + echo "Warning: Ollama may still be starting. Check /tmp/ollama.log" + fi + fi + echo "Pulling embedding model..." + ollama pull nomic-embed-text:latest 2>/dev/null || true + ollama cp nomic-embed-text:latest nomic-ai/nomic-embed-text-v1.5 2>/dev/null || true + echo "✓ Embedding model ready" +else + echo "⚠ Ollama is not installed. Install from https://ollama.com/ for embeddings." +fi + +# 6. Start LocalStack in Docker on localhost:4566 +echo "" +echo "6. Starting LocalStack..." +export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}" +LOCALSTACK_CONTAINER="${LOCALSTACK_CONTAINER:-localstack-dev}" +LOCALSTACK_IMAGE="${LOCALSTACK_IMAGE:-localstack/localstack:4.14.0}" +LOCALSTACK_ENDPOINT="http://127.0.0.1:4566" + +if curl -sf "${LOCALSTACK_ENDPOINT}/_localstack/health" &>/dev/null; then + echo "✓ LocalStack already running at ${LOCALSTACK_ENDPOINT}" +elif docker ps -a --format '{{.Names}}' | grep -qx "${LOCALSTACK_CONTAINER}"; then + echo "Starting existing container '${LOCALSTACK_CONTAINER}'..." + docker start "${LOCALSTACK_CONTAINER}" >/dev/null +else + echo "Creating container '${LOCALSTACK_CONTAINER}' from ${LOCALSTACK_IMAGE}..." + docker run -d \ + --name "${LOCALSTACK_CONTAINER}" \ + -p 127.0.0.1:4566:4566 \ + "${LOCALSTACK_IMAGE}" >/dev/null +fi + +echo "Waiting for LocalStack at ${LOCALSTACK_ENDPOINT}..." +for _ in $(seq 1 30); do + if curl -sf "${LOCALSTACK_ENDPOINT}/_localstack/health" &>/dev/null; then + echo "✓ LocalStack is healthy" + break + fi + sleep 1 +done +if ! curl -sf "${LOCALSTACK_ENDPOINT}/_localstack/health" &>/dev/null; then + echo "Error: LocalStack did not become healthy in time" >&2 + docker logs "${LOCALSTACK_CONTAINER}" --tail=40 >&2 || true + exit 1 +fi + +# 7. Create S3 buckets +echo "" +echo "7. Creating S3 buckets..." +INGESTION_BUCKET="$(yq -r '.spec.sourceCrawlerConfig.s3Config.bucket' "${SOURCE_CRAWLER_YAML}")" +STORAGE_BUCKET="$(yq -r '.spec.dataStorageBucket' "${CONTROLLER_CONFIG_YAML}")" +OUTPUT_BUCKET="$(yq -r '.spec.destinationSyncerConfig.s3DestinationConfig.bucket' "${DEST_SYNCER_YAML}")" + +for bucket in "${INGESTION_BUCKET}" "${STORAGE_BUCKET}" "${OUTPUT_BUCKET}"; do + if awslocal s3 ls "s3://${bucket}" &>/dev/null; then + echo "✓ Bucket '${bucket}' already exists" + else + awslocal s3 mb "s3://${bucket}" + echo "✓ Bucket '${bucket}' created" + fi +done + +# 8. Install CRDs +echo "" +echo "8. Installing CRDs..." +cd "${REPO_ROOT}" && make install +echo "✓ CRDs installed" + +# 9. Apply secrets (operator-secret + pipeline-secret) +echo "" +echo "9. Creating secrets..." +kubectl apply -f "${REPO_ROOT}/config/samples/unstructured-secret.yaml" \ + -n "${LOCAL_NAMESPACE}" +echo "✓ Secrets applied" + +# 10. Apply ControllerConfig +echo "" +echo "10. Creating ControllerConfig..." +kubectl apply -f "${CONTROLLER_CONFIG_YAML}" -n "${LOCAL_NAMESPACE}" +echo "✓ ControllerConfig applied" + +# 11. Apply UnstructuredDataPipeline (creates all stage CRs automatically) +echo "" +echo "11. Creating UnstructuredDataPipeline..." +kubectl apply -f "${PIPELINE_YAML}" -n "${LOCAL_NAMESPACE}" +echo "✓ UnstructuredDataPipeline applied" + +echo "" +echo "✓ Local development environment setup complete" +echo "" +echo "Starting controller (Ctrl+C to stop)..." +cd "${REPO_ROOT}" && make run diff --git a/scripts/localstack.sh b/scripts/localstack.sh new file mode 100755 index 000000000..cb31ecf3e --- /dev/null +++ b/scripts/localstack.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# LocalStack helper for local development. +# Usage: +# ./scripts/localstack.sh start # start container (no-op if healthy) +# ./scripts/localstack.sh stop # stop and remove container +# ./scripts/localstack.sh setup # start + create S3 buckets +# ./scripts/localstack.sh status # show health and resource summary +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CONTROLLER_CONFIG_YAML="${REPO_ROOT}/config/samples/operator_v1alpha1_controllerconfig.yaml" +SOURCE_CRAWLER_YAML="${REPO_ROOT}/config/samples/operator_v1alpha1_sourcecrawler.yaml" +DEST_SYNCER_YAML="${REPO_ROOT}/config/samples/operator_v1alpha1_destinationsyncer.yaml" + +CONTAINER="${LOCALSTACK_CONTAINER:-localstack-dev}" +IMAGE="${LOCALSTACK_IMAGE:-localstack/localstack:4.14.0}" +ENDPOINT="${LOCALSTACK_ENDPOINT:-http://127.0.0.1:4566}" +AWS_REGION="${AWS_DEFAULT_REGION:-us-east-1}" + +INGESTION_BUCKET="$(yq -r '.spec.sourceCrawlerConfig.s3Config.bucket' "${SOURCE_CRAWLER_YAML}")" +DATA_STORAGE_BUCKET="$(yq -r '.spec.dataStorageBucket' "${CONTROLLER_CONFIG_YAML}")" +OUTPUT_BUCKET="$(yq -r '.spec.destinationSyncerConfig.s3DestinationConfig.bucket' "${DEST_SYNCER_YAML}")" + +usage() { + cat < + +Commands: + start Start LocalStack in Docker (skips if already healthy) + stop Stop and remove the LocalStack container + setup Start LocalStack and create S3 buckets + status Print LocalStack health and configured resources + +Environment overrides: + LOCALSTACK_CONTAINER Docker container name (default: localstack-dev) + LOCALSTACK_IMAGE Docker image (default: localstack/localstack:4.14.0) + LOCALSTACK_ENDPOINT LocalStack URL (default: http://127.0.0.1:4566) +EOF +} + +require_command() { + if ! command -v "$1" &>/dev/null; then + echo "Error: '$1' is required but not installed." >&2 + exit 1 + fi +} + +aws_local() { + AWS_ACCESS_KEY_ID=test \ + AWS_SECRET_ACCESS_KEY=test \ + AWS_DEFAULT_REGION="${AWS_REGION}" \ + aws --endpoint-url="${ENDPOINT}" "$@" +} + +localstack_healthy() { + curl -sf "${ENDPOINT}/_localstack/health" &>/dev/null +} + +wait_for_localstack() { + echo "Waiting for LocalStack at ${ENDPOINT}..." + for _ in $(seq 1 30); do + if localstack_healthy; then + echo "✓ LocalStack is healthy" + return 0 + fi + sleep 1 + done + echo "Error: LocalStack did not become healthy in time" >&2 + docker logs "${CONTAINER}" --tail=40 >&2 || true + return 1 +} + +cmd_start() { + require_command docker + require_command curl + + if localstack_healthy; then + echo "✓ LocalStack already running at ${ENDPOINT}" + return 0 + fi + + if docker ps -a --format '{{.Names}}' | grep -qx "${CONTAINER}"; then + echo "Starting existing container '${CONTAINER}'..." + docker start "${CONTAINER}" >/dev/null + else + echo "Creating container '${CONTAINER}' from ${IMAGE}..." + docker run -d \ + --name "${CONTAINER}" \ + -p 127.0.0.1:4566:4566 \ + "${IMAGE}" >/dev/null + fi + + wait_for_localstack +} + +cmd_stop() { + require_command docker + + if docker ps -a --format '{{.Names}}' | grep -qx "${CONTAINER}"; then + docker rm -f "${CONTAINER}" >/dev/null + echo "✓ Stopped and removed '${CONTAINER}'" + else + echo "✓ No '${CONTAINER}' container found" + fi +} + +cmd_setup() { + require_command aws + require_command yq + + cmd_start + + echo "" + echo "Creating S3 buckets..." + for bucket in "${INGESTION_BUCKET}" "${DATA_STORAGE_BUCKET}" "${OUTPUT_BUCKET}"; do + if aws_local s3 ls "s3://${bucket}" &>/dev/null; then + echo "✓ Bucket '${bucket}' already exists" + else + aws_local s3 mb "s3://${bucket}" >/dev/null + echo "✓ Bucket '${bucket}' created" + fi + done + + echo "" + echo "✓ LocalStack setup complete" + echo " Endpoint: ${ENDPOINT}" +} + +cmd_status() { + require_command aws + + if localstack_healthy; then + echo "✓ LocalStack healthy at ${ENDPOINT}" + curl -s "${ENDPOINT}/_localstack/health" | sed 's/^/ /' + else + echo "✗ LocalStack not reachable at ${ENDPOINT}" + return 1 + fi + + echo "" + echo "S3 buckets:" + aws_local s3 ls 2>/dev/null | sed 's/^/ /' || echo " (none)" +} + +main() { + local command="${1:-}" + case "${command}" in + start) cmd_start ;; + stop) cmd_stop ;; + setup) cmd_setup ;; + status) cmd_status ;; + -h|--help|help|"") usage ;; + *) + echo "Error: unknown command '${command}'" >&2 + usage >&2 + exit 1 + ;; + esac +} + +main "$@"