feat: initial skeleton

alecthomas · alecthomas · commit c9fd86b91f48 · 2025-12-27T15:50:46.000+11:00
- Client and server packages, with helper scripts for local development.
- API's for cache and strategies
- Basic disk-backed LRU cache implementation
- Linter config
- CI config, includig release flow
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,54 @@
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - "v*.*.*"
+  pull_request:
+name: CI
+jobs:
+  formatted:
+    name: Formatted
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: cashapp/activate-hermit@v1
+      - run: |
+          just fmt
+          git diff
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: cashapp/activate-hermit@v1
+      - name: Setup xattr-supported filesystem
+        run: |
+          mkdir -p /tmp/xattr-fs
+          sudo mount -t tmpfs -o size=1G tmpfs /tmp/xattr-fs
+      - name: Run tests
+        run: just test
+        env:
+          TMPDIR: /tmp/xattr-fs
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    env:
+      GOPROXY: direct
+    steps:
+      - uses: actions/checkout@v4
+      - uses: cashapp/activate-hermit@v1
+      - run: just lint
+  release:
+    if: startsWith(github.ref, 'refs/tags/v')
+    needs: ["test", "lint"]
+    name: Release
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: cashapp/activate-hermit@v1
+      - run: goreleaser release --fail-fast
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,2 @@
+# Added by goreleaser init:
+dist/
diff --git a/.golangci.yml b/.golangci.yml
diff --git a/.goreleaser.yaml b/.goreleaser.yaml
@@ -0,0 +1,46 @@
+builds:
+  - main: ./cmd/sfptc
+    binary: sfptc
+    id: sfptc
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - darwin
+    goarch:
+      - amd64
+      - arm64
+  - main: ./cmd/sfptcd
+    id: sfptcd
+    binary: sfptcd
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - darwin
+    goarch:
+      - amd64
+      - arm64
+
+archives:
+  - format: tar.gz
+    # this name template makes the OS and Arch compatible with the results of uname.
+    name_template: >-
+      {{ .ProjectName }}-
+      {{- .Os }}-
+      {{- .Arch }}
+    format_overrides:
+      - goos: windows
+        format: zip
+checksum:
+  name_template: "checksums.txt"
+snapshot:
+  name_template: "{{ incpatch .Version }}-next"
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - "^docs:"
+      - "^test:"
+# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
+# vim: set ts=2 sw=2 tw=0 fo=cnqoj
diff --git a/Justfile b/Justfile
@@ -0,0 +1,16 @@
+_help:
+    @just -l
+
+# Run tests
+test:
+    go test ./...
+
+# Lint code
+lint:
+    golangci-lint run
+
+# Format code
+fmt:
+    just --unstable --fmt
+    golangci-lint fmt
+    go mod tidy
diff --git a/README.md b/README.md
@@ -1,37 +1,32 @@
-# sfptc README
+# Super-fast Pass-through Cache (SFPTC)
 
-Congrats, project leads! You got a new project to grow!
+SFPTC is a server and tooling for incredibly efficient, protocol-aware caching. It is
+designed to be used at scale, with minimal impact on upstream systems. By "protocol-aware", we mean that the proxy isn't
+just a naive HTTP proxy, it is aware of the higher level protocol being proxied (Git, Docker, etc.) and can make more efficient decisions.
 
-This stub is meant to help you form a strong community around your work. It's yours to adapt, and may 
-diverge from this initial structure. Just keep the files seeded in this repo, and the rest is yours to evolve! 
+## Git
 
-## Introduction
+Git causes a number of problems for us, but the most obvious are:
 
-Orient users to the project here. This is a good place to start with an assumption
-that the user knows very little - so start with the Big Picture and show how this
-project fits into it.
+1. Rate limiting by service providers.
+2. `git clone` is very slow, even discounting network overhead
 
-Then maybe a dive into what this project does.
+To solve this we apply two different strategies on the server:
 
-Diagrams and other visuals are helpful here. Perhaps code snippets showing usage.
+1. Periodic full `.tar.zst` snapshots of the repository. These snapshots restore 4-5x faster than `git clone`.
+2. Passthrough caching of the packs returned by `POST /repo.git/git-upload-pack` to support incremental pulls.
 
-Project leads should complete, alongside this `README`:
-* [CODEOWNERS](./CODEOWNERS) - set project lead(s)
-* [CONTRIBUTING.md](./CONTRIBUTING.md) - Fill out how to: install prereqs, build, test, run, access CI, chat, discuss, file issues
-* [Bug-report.md](.github/ISSUE_TEMPLATE/bug-report.md) - Fill out `Assignees` add codeowners @names
-* [config.yml](.github/ISSUE_TEMPLATE/config.yml) - remove "(/add your discord channel..)" and replace the url with your Discord channel if applicable
+On the client we redirect git to the proxy:
 
-The other files in this template repo may be used as-is:
-* [CODE_OF_CONDUCT.md](./CODE_OF_CONDUCT.md)
-* [GOVERNANCE.md](./GOVERNANCE.md)
-* [LICENSE](./LICENSE)
+```ini
+[url "https://sfptc.local/github/"]
+  insteadOf = https://github.com/
+```
 
-## Project Resources
+As Git itself isn't aware of the snapshots, Git-specific code in the SFPTC CLI can be used to reconstruct a repository.
 
-| Resource                                   | Description                                                                    |
-| ------------------------------------------ | ------------------------------------------------------------------------------ |
-| [CODEOWNERS](./CODEOWNERS)                 | Outlines the project lead(s)                                                   |
-| [CODE_OF_CONDUCT.md](./CODE_OF_CONDUCT.md) | Expected behavior for project contributors, promoting a welcoming environment |
-| [CONTRIBUTING.md](./CONTRIBUTING.md)       | Developer guide to build, test, run, access CI, chat, discuss, file issues     |
-| [GOVERNANCE.md](./GOVERNANCE.md)           | Project governance                                                             |
-| [LICENSE](./LICENSE)                       | Apache License, Version 2.0                                                    |
+## Docker
+
+## Hermit
+
+Hermit
diff --git a/bin/.go-1.25.5.pkg b/bin/.go-1.25.5.pkg
@@ -0,0 +1 @@
+hermit
diff --git a/bin/.golangci-lint-2.7.2.pkg b/bin/.golangci-lint-2.7.2.pkg
@@ -0,0 +1 @@
+hermit
diff --git a/bin/.goreleaser-2.13.2.pkg b/bin/.goreleaser-2.13.2.pkg
@@ -0,0 +1 @@
+hermit
diff --git a/bin/.just-1.45.0.pkg b/bin/.just-1.45.0.pkg
@@ -0,0 +1 @@
+hermit
diff --git a/bin/.lefthook-2.0.13.pkg b/bin/.lefthook-2.0.13.pkg
@@ -0,0 +1 @@
+hermit
diff --git a/bin/.svu-3.3.0.pkg b/bin/.svu-3.3.0.pkg
@@ -0,0 +1 @@
+hermit
diff --git a/bin/README.hermit.md b/bin/README.hermit.md
@@ -0,0 +1,7 @@
+# Hermit environment
+
+This is a [Hermit](https://github.com/cashapp/hermit) bin directory.
+
+The symlinks in this directory are managed by Hermit and will automatically
+download and install Hermit itself as well as packages. These packages are
+local to this environment.
diff --git a/bin/activate-hermit b/bin/activate-hermit
@@ -0,0 +1,21 @@
+#!/bin/bash
+# This file must be used with "source bin/activate-hermit" from bash or zsh.
+# You cannot run it directly
+#
+# THIS FILE IS GENERATED; DO NOT MODIFY
+
+if [ "${BASH_SOURCE-}" = "$0" ]; then
+  echo "You must source this script: \$ source $0" >&2
+  exit 33
+fi
+
+BIN_DIR="$(dirname "${BASH_SOURCE[0]:-${(%):-%x}}")"
+if "${BIN_DIR}/hermit" noop > /dev/null; then
+  eval "$("${BIN_DIR}/hermit" activate "${BIN_DIR}/..")"
+
+  if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ]; then
+      hash -r 2>/dev/null
+    fi
+
+    echo "Hermit environment $("${HERMIT_ENV}"/bin/hermit env HERMIT_ENV) activated"
+fi
diff --git a/bin/activate-hermit.fish b/bin/activate-hermit.fish
@@ -0,0 +1,24 @@
+#!/usr/bin/env fish
+
+# This file must be sourced with "source bin/activate-hermit.fish" from Fish shell.
+# You cannot run it directly.
+#
+# THIS FILE IS GENERATED; DO NOT MODIFY
+
+if status is-interactive
+    set BIN_DIR (dirname (status --current-filename))
+
+    if "$BIN_DIR/hermit" noop > /dev/null
+        # Source the activation script generated by Hermit
+        "$BIN_DIR/hermit" activate "$BIN_DIR/.." | source
+
+        # Clear the command cache if applicable
+        functions -c > /dev/null 2>&1
+
+        # Display activation message
+        echo "Hermit environment $($HERMIT_ENV/bin/hermit env HERMIT_ENV) activated"
+    end
+else
+    echo "You must source this script: source $argv[0]" >&2
+    exit 33
+end
diff --git a/bin/go b/bin/go
@@ -0,0 +1 @@
+.go-1.25.5.pkg
diff --git a/bin/gofmt b/bin/gofmt
@@ -0,0 +1 @@
+.go-1.25.5.pkg
diff --git a/bin/golangci-lint b/bin/golangci-lint
@@ -0,0 +1 @@
+.golangci-lint-2.7.2.pkg
diff --git a/bin/goreleaser b/bin/goreleaser
@@ -0,0 +1 @@
+.goreleaser-2.13.2.pkg
diff --git a/bin/hermit b/bin/hermit
@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+# THIS FILE IS GENERATED; DO NOT MODIFY
+
+set -eo pipefail
+
+export HERMIT_USER_HOME=~
+
+if [ -z "${HERMIT_STATE_DIR}" ]; then
+  case "$(uname -s)" in
+  Darwin)
+    export HERMIT_STATE_DIR="${HERMIT_USER_HOME}/Library/Caches/hermit"
+    ;;
+  Linux)
+    export HERMIT_STATE_DIR="${XDG_CACHE_HOME:-${HERMIT_USER_HOME}/.cache}/hermit"
+    ;;
+  esac
+fi
+
+export HERMIT_DIST_URL="${HERMIT_DIST_URL:-https://github.com/cashapp/hermit/releases/download/stable}"
+HERMIT_CHANNEL="$(basename "${HERMIT_DIST_URL}")"
+export HERMIT_CHANNEL
+export HERMIT_EXE=${HERMIT_EXE:-${HERMIT_STATE_DIR}/pkg/hermit@${HERMIT_CHANNEL}/hermit}
+
+if [ ! -x "${HERMIT_EXE}" ]; then
+  echo "Bootstrapping ${HERMIT_EXE} from ${HERMIT_DIST_URL}" 1>&2
+  INSTALL_SCRIPT="$(mktemp)"
+  # This value must match that of the install script
+  INSTALL_SCRIPT_SHA256="09ed936378857886fd4a7a4878c0f0c7e3d839883f39ca8b4f2f242e3126e1c6"
+  if [ "${INSTALL_SCRIPT_SHA256}" = "BYPASS" ]; then
+    curl -fsSL "${HERMIT_DIST_URL}/install.sh" -o "${INSTALL_SCRIPT}"
+  else
+    # Install script is versioned by its sha256sum value
+    curl -fsSL "${HERMIT_DIST_URL}/install-${INSTALL_SCRIPT_SHA256}.sh" -o "${INSTALL_SCRIPT}"
+    # Verify install script's sha256sum
+    openssl dgst -sha256 "${INSTALL_SCRIPT}" | \
+      awk -v EXPECTED="$INSTALL_SCRIPT_SHA256" \
+      '$2!=EXPECTED {print "Install script sha256 " $2 " does not match " EXPECTED; exit 1}'
+  fi
+  /bin/bash "${INSTALL_SCRIPT}" 1>&2
+fi
+
+exec "${HERMIT_EXE}" --level=fatal exec "$0" -- "$@"
diff --git a/bin/hermit.hcl b/bin/hermit.hcl
@@ -0,0 +1,6 @@
+env = {
+  "PATH": "${HERMIT_ENV}/scripts:${PATH}",
+}
+
+github-token-auth {
+}
diff --git a/bin/just b/bin/just
@@ -0,0 +1 @@
+.just-1.45.0.pkg
diff --git a/bin/lefthook b/bin/lefthook
@@ -0,0 +1 @@
+.lefthook-2.0.13.pkg
diff --git a/bin/svu b/bin/svu
@@ -0,0 +1 @@
+.svu-3.3.0.pkg
diff --git a/cmd/sfptc/main.go b/cmd/sfptc/main.go
@@ -0,0 +1,15 @@
+package main
+
+import (
+	"github.com/alecthomas/kong"
+
+	"github.com/block/sfptc/internal/logging"
+)
+
+var cli struct {
+	logging.Config
+}
+
+func main() {
+	kong.Parse(&cli)
+}
diff --git a/cmd/sfptcd/main.go b/cmd/sfptcd/main.go
@@ -0,0 +1,22 @@
+package main
+
+import (
+	"context"
+
+	"github.com/alecthomas/kong"
+
+	"github.com/block/sfptc/internal/logging"
+)
+
+var cli struct {
+	logging.Config `prefix:"log-"`
+}
+
+func main() {
+	kong.Parse(&cli)
+
+	ctx := context.Background()
+	logger, ctx := logging.Configure(ctx, cli.Config)
+
+	logger.InfoContext(ctx, "Starting sfptcd")
+}
diff --git a/go.mod b/go.mod
@@ -0,0 +1,22 @@
+module github.com/block/sfptc
+
+go 1.25.5
+
+require (
+	github.com/alecthomas/hcl v0.5.5
+	github.com/alecthomas/kong v1.13.0
+	github.com/lmittmann/tint v1.1.2
+)
+
+require (
+	github.com/hexops/gotextdiff v1.0.3 // indirect
+	golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f // indirect
+)
+
+require (
+	github.com/alecthomas/assert/v2 v2.11.0
+	github.com/alecthomas/errors v0.8.3
+	github.com/alecthomas/participle/v2 v2.0.0-beta.5 // indirect
+	github.com/alecthomas/repr v0.5.2 // indirect
+	github.com/pkg/xattr v0.4.12
+)
diff --git a/go.sum b/go.sum
diff --git a/internal/cache/api.go b/internal/cache/api.go
diff --git a/internal/cache/disk.go b/internal/cache/disk.go
diff --git a/internal/cache/disk_test.go b/internal/cache/disk_test.go
diff --git a/internal/logging/logging.go b/internal/logging/logging.go
diff --git a/internal/strategy/api.go b/internal/strategy/api.go
diff --git a/internal/strategy/default.go b/internal/strategy/default.go
diff --git a/lefthook.yml b/lefthook.yml
diff --git a/scripts/sfptc b/scripts/sfptc
diff --git a/scripts/sfptcd b/scripts/sfptcd

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+# Added by goreleaser init:`
	`2`	`+dist/`