Support s3 on docker compose

config/config.yaml

@@ -15,6 +15,27 @@ database:
 
 apiServer:
   port: 3001
+  # S3 storage configuration (optional)
+  storage:
+    # storage mode: local (local filesystem), s3 (S3-compatible object storage)
+    mode: local
+    s3:
+      # S3 endpoint URL (e.g., http://localhost:9000)
+      endpoint: ""
+      # S3 bucket name
+      bucket: ""
+      # S3 region
+      region: ""
+      # S3 access key
+      accessKey: ""
+      # S3 secret key
+      secretKey: ""
+      # use path-style addressing (required for MinIO)
+      usePathStyle: true
+      # S3 signing endpoint (optional, for presigned URLs)
+      signEndpoint: ""
+      # Store git repositories in S3 using s3fs (requires privileged mode)
+      repositories: false
 
 jobServer:
   enabled: true

deploy/config/config.s3.yaml

@@ -0,0 +1,49 @@
+# config for local dev env
+debug: true
+migrationPath: /etc/matrixhub/migrations
+log:
+  level: debug
+database:
+  # dsn: "matrixhub:changeme@tcp(127.0.0.1:30007)/matrixhub?charset=utf8mb4&multiStatements=true&parseTime=true"
+  driver: mysql
+  accessType: readwrite
+  maxOpenConns: 100
+  maxIdleConns: 10
+  connMaxLifetimeSeconds: 3600
+  connMaxIdleSeconds: 1800
+  migrate: true
+
+apiServer:
+  port: 3001
+  # S3 storage configuration (optional)
+  storage:
+    mode: s3
+    s3:
+      endpoint: http://s3:9000
+      bucket: matrixhub
+      region: us-east-1
+      accessKey: adminadmin
+      secretKey: adminadmin
+      usePathStyle: true
+      signEndpoint: http://127.0.0.1:9000
+      repositories: true
+
+jobServer:
+  enabled: true
+  shutdownGrace: 30s
+  syncPolicy:
+    pollInterval: 30s
+    maxConcurrent: 5
+    taskMaxDuration: 2h
+  syncTask:
+    pollInterval: 30s
+    maxConcurrent: 5
+    taskMaxDuration: 2h
+  syncJob:
+    pollInterval: 30s
+    maxConcurrent: 5
+    taskMaxDuration: 2h
+  logDir: ""
+
+ui:
+  staticDir: /app

deploy/docker-compose.s3.yml

@@ -0,0 +1,64 @@
+# S3 storage deployment. Run with: docker compose -f docker-compose.yml -f docker-compose.s3.yml up -d
+services:
+  mysql:
+    image: docker.io/library/mysql:8.4
+    environment:
+      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-password}
+      MYSQL_DATABASE: matrixhub
+      MYSQL_USER: matrixhub
+      MYSQL_PASSWORD: ${MYSQL_PASSWORD:-changeme}
+    ports:
+      - "3306:3306"
+    volumes:
+      - ./data/mysql:/var/lib/mysql
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-uroot", "-p${MYSQL_ROOT_PASSWORD:-password}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    restart: unless-stopped
+
+  s3:
+    image: quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      - MINIO_ROOT_USER=adminadmin
+      - MINIO_ROOT_PASSWORD=adminadmin
+    entrypoint:
+      - sh
+      - -c
+    command:
+      - |
+        mkdir -p /data/matrixhub && \
+        /usr/bin/docker-entrypoint.sh server /data/ --console-address ":9001"
+    volumes:
+      - ./data/s3:/data/
+    restart: unless-stopped
+
+  matrixhub:
+    # image: ghcr.io/matrixhub-ai/matrixhub:${MATRIXHUB_IMAGE_TAG:-latest}
+    build:
+      context: ../
+      dockerfile: ./deploy/docker/matrixhub/Dockerfile
+    privileged: true
+    working_dir: /workspace
+    command:
+      - /usr/local/bin/matrixhub
+      - apiserver
+      - -c
+      - /etc/matrixhub/config.yaml
+    ports:
+      - "3001:3001"
+    volumes:
+      - ./data/tmp:/data/tmp/
+      - ./config/config.s3.yaml:/etc/matrixhub/config.yaml:ro
+    environment:
+      - MATRIXHUB_DATABASE_DSN=matrixhub:${MYSQL_PASSWORD:-changeme}@tcp(mysql:3306)/matrixhub?charset=utf8mb4&multiStatements=true&parseTime=true
+    depends_on:
+      mysql:
+        condition: service_healthy
+      s3:
+        condition: service_started
+    restart: unless-stopped

deploy/docker/matrixhub/Dockerfile

@@ -58,7 +58,7 @@ ARG HTTP_PROXY
 ARG HTTPS_PROXY
 
 RUN --mount=type=cache,target=/var/cache/apk \
-    apk add ca-certificates git && \
+    apk add ca-certificates git s3fs-fuse && \
     update-ca-certificates
 
 COPY --from=builder /matrixhub /usr/local/bin/matrixhub

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.6
 	github.com/jackc/pgconn v1.14.3
 	github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6
-	github.com/matrixhub-ai/hfd v0.0.0-20260522103102-73bc92d77d19
+	github.com/matrixhub-ai/hfd v0.0.0-20260622073022-7b4b945e39d4
 	github.com/onsi/ginkgo/v2 v2.22.1
 	github.com/onsi/gomega v1.36.2
 	github.com/pkg/errors v0.9.1
@@ -29,6 +29,7 @@ require (
 	go.uber.org/zap v1.27.1
 	golang.org/x/crypto v0.50.0
 	golang.org/x/oauth2 v0.36.0
+	golang.org/x/sync v0.20.0
 	google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409
 	google.golang.org/grpc v1.78.0
 	google.golang.org/protobuf v1.36.11
@@ -126,17 +127,16 @@ require (
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.3.1 // indirect
 	github.com/wzshiming/dl v0.0.0-20260427082752-19804485f005 // indirect
-	github.com/wzshiming/httpseek v0.6.0 // indirect
+	github.com/wzshiming/httpseek v0.6.1 // indirect
 	github.com/wzshiming/ioswmr v0.2.0 // indirect
-	github.com/wzshiming/xet v0.0.0-20260515075456-afcac81072aa // indirect
+	github.com/wzshiming/xet v0.0.0-20260617081805-02c9cfb4deca // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/zeebo/blake3 v0.2.4 // indirect
 	go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/arch v0.22.0 // indirect
 	golang.org/x/net v0.53.0 // indirect
-	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/sys v0.43.0 // indirect
 	golang.org/x/text v0.36.0 // indirect
 	golang.org/x/tools v0.43.0 // indirect

go.sum

@@ -240,8 +240,8 @@ github.com/leonelquinteros/gotext v1.5.0 h1:ODY7LzLpZWWSJdAHnzhreOr6cwLXTAmc914F
 github.com/leonelquinteros/gotext v1.5.0/go.mod h1:OCiUVHuhP9LGFBQ1oAmdtNCHJCiHiQA8lf4nAifHkr0=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/matrixhub-ai/hfd v0.0.0-20260522103102-73bc92d77d19 h1:OrxRrJPrAnROotcuRpDPmRVmGBMdQlZpvGTlZF5WMBQ=
-github.com/matrixhub-ai/hfd v0.0.0-20260522103102-73bc92d77d19/go.mod h1:SWICq+Zuz3sUe9nHigwRdr8e9cOZOq7MHCjsjIgGcyI=
+github.com/matrixhub-ai/hfd v0.0.0-20260622073022-7b4b945e39d4 h1:ANGcUdDqAwHw0k99RCfRiP5awiZJmqBueUs8Va8WX6s=
+github.com/matrixhub-ai/hfd v0.0.0-20260622073022-7b4b945e39d4/go.mod h1:1ttFYcNAx1YrqcFImxFfmk9xORuGmTNtHtXM0JGpA5g=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
@@ -345,12 +345,12 @@ github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY
 github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4=
 github.com/wzshiming/dl v0.0.0-20260427082752-19804485f005 h1:Nac8aZwtkW8GTkxh8mWGF08nVLO6Ze68FWH2I+1BpwI=
 github.com/wzshiming/dl v0.0.0-20260427082752-19804485f005/go.mod h1:jO3MKwJvcFUxlK9Pd8HpQ0YToDWTB61nvklu/NuYPqw=
-github.com/wzshiming/httpseek v0.6.0 h1:RWisrPz5mef1rIp//WXRMlq4m1RN11G9D6YdgtXrkhM=
-github.com/wzshiming/httpseek v0.6.0/go.mod h1:YoZhlLIwNjTBDXIT8NpK5zRjOgZouRXPaBfjVXdqMMs=
+github.com/wzshiming/httpseek v0.6.1 h1:nmrpxl7SNtj8KmmZ8XRMCnKVfOvj1yNAGXNYtPfeRoA=
+github.com/wzshiming/httpseek v0.6.1/go.mod h1:YoZhlLIwNjTBDXIT8NpK5zRjOgZouRXPaBfjVXdqMMs=
 github.com/wzshiming/ioswmr v0.2.0 h1:PWPtSII1hLz/XIhTxUaEQFe8Vwyb/BXXMg5fUKOpnyw=
 github.com/wzshiming/ioswmr v0.2.0/go.mod h1:TwwDyS1wnJG3AvKliA+PPB0kliN3yEjsabtH4o7xySQ=
-github.com/wzshiming/xet v0.0.0-20260515075456-afcac81072aa h1:RWJKY+ZmhVSiBxe6JGg7Kb+Yk5b44p3bnh4IPi5VQU8=
-github.com/wzshiming/xet v0.0.0-20260515075456-afcac81072aa/go.mod h1:AvhOcT0RB0pEwNZvwn5SxGj8MiZBVF6NPHL7OoumYJ8=
+github.com/wzshiming/xet v0.0.0-20260617081805-02c9cfb4deca h1:XPJAxtUWWn2ZsQC/V8toPPEEWINBzscEFQFy6eECUHI=
+github.com/wzshiming/xet v0.0.0-20260617081805-02c9cfb4deca/go.mod h1:oC+etpyWxYsBQPxBfOtX/YWFKFHLkUsYO0c6zv9+Au8=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=

internal/apiserver/apiserver.go

@@ -37,6 +37,7 @@ import (
 	"github.com/matrixhub-ai/hfd/pkg/mirror"
 	"github.com/matrixhub-ai/hfd/pkg/permission"
 	"github.com/matrixhub-ai/hfd/pkg/receive"
+	hfds3fs "github.com/matrixhub-ai/hfd/pkg/s3fs"
 	hfdssh "github.com/matrixhub-ai/hfd/pkg/ssh"
 	gitstorage "github.com/matrixhub-ai/hfd/pkg/storage"
 	"github.com/soheilhy/cmux"
@@ -251,7 +252,51 @@ func (server *APIServer) initGitStorage() {
 		gitstorage.WithRootDir(server.config.DataDir),
 	)
 
-	lfsStorage := lfs.NewLocal(storage.LFSDir())
+	// Initialize LFS storage based on configuration
+	var lfsStorage lfs.Storage
+	storageConfig := server.config.APIServer.Storage
+	if storageConfig.Mode == "s3" && storageConfig.S3.Endpoint != "" {
+		s3Config := storageConfig.S3
+
+		// Mount S3 bucket for repositories if enabled
+		if s3Config.Repositories {
+			repositoriesDir := storage.RepositoriesDir()
+			log.Info("mounting S3 bucket for repositories", "bucket", s3Config.Bucket, "path", repositoriesDir)
+			err := hfds3fs.Mount(
+				context.Background(),
+				repositoriesDir,
+				s3Config.Endpoint,
+				s3Config.AccessKey,
+				s3Config.SecretKey,
+				s3Config.Bucket,
+				"/repositories/",
+				s3Config.UsePathStyle,
+			)
+			if err != nil {
+				log.Error("error mounting S3 bucket for repositories", "bucket", s3Config.Bucket, "path", repositoriesDir, "error", err)
+				// Don't exit, continue with local repositories
+			}
+			// Note: In production, you should handle unmount on shutdown
+		}
+
+		signEndpoint := s3Config.SignEndpoint
+		if signEndpoint == "" {
+			signEndpoint = s3Config.Endpoint
+		}
+		lfsStorage = lfs.NewS3(
+			storage.LFSDir(),
+			s3Config.Endpoint,
+			s3Config.AccessKey,
+			s3Config.SecretKey,
+			s3Config.Bucket,
+			s3Config.UsePathStyle,
+			signEndpoint,
+		)
+		log.Info("using S3 storage", "endpoint", s3Config.Endpoint, "bucket", s3Config.Bucket)
+	} else {
+		lfsStorage = lfs.NewLocal(storage.LFSDir())
+		log.Info("using local storage", "path", storage.LFSDir())
+	}
 
 	mirrorSourceFunc := server.gitHooks.mirrorSourceFunc
 	mirrorDestinationFunc := server.gitHooks.mirrorDestinationFunc

internal/infra/config/config.go

@@ -108,6 +108,36 @@ type APIServerConfig struct {
 	// Empty means "not configured" and the API returns an empty string so the
 	// UI can hide affected panels.
 	ExternalURL string `yaml:"externalURL"`
+	// Storage is the storage configuration for git/LFS data
+	Storage StorageConfig `yaml:"storage"`
+}
+
+// StorageConfig defines the storage backend configuration
+type StorageConfig struct {
+	// Mode is the storage mode: "local" for filesystem, "s3" for S3-compatible object storage
+	Mode string `yaml:"mode"`
+	// S3 is the S3 storage configuration (used when mode is "s3")
+	S3 S3Config `yaml:"s3"`
+}
+
+// S3Config defines S3-compatible object storage configuration
+type S3Config struct {
+	// Endpoint is the S3 endpoint URL (e.g., http://localhost:9000)
+	Endpoint string `yaml:"endpoint"`
+	// Bucket is the S3 bucket name
+	Bucket string `yaml:"bucket"`
+	// Region is the S3 region
+	Region string `yaml:"region"`
+	// AccessKey is the S3 access key
+	AccessKey string `yaml:"accessKey"`
+	// SecretKey is the S3 secret key
+	SecretKey string `yaml:"secretKey"`
+	// UsePathStyle enables path-style addressing (required for MinIO)
+	UsePathStyle bool `yaml:"usePathStyle"`
+	// SignEndpoint is the endpoint for generating presigned URLs (defaults to Endpoint if empty)
+	SignEndpoint string `yaml:"signEndpoint"`
+	// Repositories enables storing git repositories in S3 using s3fs (requires privileged mode)
+	Repositories bool `yaml:"repositories"`
 }
 
 type UIConfig struct {