diff --git a/config/config.yaml b/config/config.yaml index 137a253a..55f26e41 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -15,6 +15,27 @@ database: apiServer: port: 3001 + # S3 storage configuration (optional) + storage: + # storage mode: local (local filesystem), s3 (S3-compatible object storage) + mode: local + s3: + # S3 endpoint URL (e.g., http://localhost:9000) + endpoint: "" + # S3 bucket name + bucket: "" + # S3 region + region: "" + # S3 access key + accessKey: "" + # S3 secret key + secretKey: "" + # use path-style addressing (required for MinIO) + usePathStyle: true + # S3 signing endpoint (optional, for presigned URLs) + signEndpoint: "" + # Store git repositories in S3 using s3fs (requires privileged mode) + repositories: false jobServer: enabled: true diff --git a/deploy/config/config.s3.yaml b/deploy/config/config.s3.yaml new file mode 100644 index 00000000..cd7e0d87 --- /dev/null +++ b/deploy/config/config.s3.yaml @@ -0,0 +1,49 @@ +# config for local dev env +debug: true +migrationPath: /etc/matrixhub/migrations +log: + level: debug +database: + # dsn: "matrixhub:changeme@tcp(127.0.0.1:30007)/matrixhub?charset=utf8mb4&multiStatements=true&parseTime=true" + driver: mysql + accessType: readwrite + maxOpenConns: 100 + maxIdleConns: 10 + connMaxLifetimeSeconds: 3600 + connMaxIdleSeconds: 1800 + migrate: true + +apiServer: + port: 3001 + # S3 storage configuration (optional) + storage: + mode: s3 + s3: + endpoint: http://s3:9000 + bucket: matrixhub + region: us-east-1 + accessKey: adminadmin + secretKey: adminadmin + usePathStyle: true + signEndpoint: http://127.0.0.1:9000 + repositories: true + +jobServer: + enabled: true + shutdownGrace: 30s + syncPolicy: + pollInterval: 30s + maxConcurrent: 5 + taskMaxDuration: 2h + syncTask: + pollInterval: 30s + maxConcurrent: 5 + taskMaxDuration: 2h + syncJob: + pollInterval: 30s + maxConcurrent: 5 + taskMaxDuration: 2h + logDir: "" + +ui: + staticDir: /app diff --git a/deploy/docker-compose.s3.yml b/deploy/docker-compose.s3.yml new file mode 100644 index 00000000..fb26a567 --- /dev/null +++ b/deploy/docker-compose.s3.yml @@ -0,0 +1,64 @@ +# S3 storage deployment. Run with: docker compose -f docker-compose.yml -f docker-compose.s3.yml up -d +services: + mysql: + image: docker.io/library/mysql:8.4 + environment: + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-password} + MYSQL_DATABASE: matrixhub + MYSQL_USER: matrixhub + MYSQL_PASSWORD: ${MYSQL_PASSWORD:-changeme} + ports: + - "3306:3306" + volumes: + - ./data/mysql:/var/lib/mysql + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-uroot", "-p${MYSQL_ROOT_PASSWORD:-password}"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + s3: + image: quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z + ports: + - "9000:9000" + - "9001:9001" + environment: + - MINIO_ROOT_USER=adminadmin + - MINIO_ROOT_PASSWORD=adminadmin + entrypoint: + - sh + - -c + command: + - | + mkdir -p /data/matrixhub && \ + /usr/bin/docker-entrypoint.sh server /data/ --console-address ":9001" + volumes: + - ./data/s3:/data/ + restart: unless-stopped + + matrixhub: + # image: ghcr.io/matrixhub-ai/matrixhub:${MATRIXHUB_IMAGE_TAG:-latest} + build: + context: ../ + dockerfile: ./deploy/docker/matrixhub/Dockerfile + privileged: true + working_dir: /workspace + command: + - /usr/local/bin/matrixhub + - apiserver + - -c + - /etc/matrixhub/config.yaml + ports: + - "3001:3001" + volumes: + - ./data/tmp:/data/tmp/ + - ./config/config.s3.yaml:/etc/matrixhub/config.yaml:ro + environment: + - MATRIXHUB_DATABASE_DSN=matrixhub:${MYSQL_PASSWORD:-changeme}@tcp(mysql:3306)/matrixhub?charset=utf8mb4&multiStatements=true&parseTime=true + depends_on: + mysql: + condition: service_healthy + s3: + condition: service_started + restart: unless-stopped diff --git a/deploy/docker/matrixhub/Dockerfile b/deploy/docker/matrixhub/Dockerfile index 7b48f789..13a0bcc8 100644 --- a/deploy/docker/matrixhub/Dockerfile +++ b/deploy/docker/matrixhub/Dockerfile @@ -58,7 +58,7 @@ ARG HTTP_PROXY ARG HTTPS_PROXY RUN --mount=type=cache,target=/var/cache/apk \ - apk add ca-certificates git && \ + apk add ca-certificates git s3fs-fuse && \ update-ca-certificates COPY --from=builder /matrixhub /usr/local/bin/matrixhub diff --git a/go.mod b/go.mod index 5480faae..4307b61d 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.6 github.com/jackc/pgconn v1.14.3 github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 - github.com/matrixhub-ai/hfd v0.0.0-20260522103102-73bc92d77d19 + github.com/matrixhub-ai/hfd v0.0.0-20260622073022-7b4b945e39d4 github.com/onsi/ginkgo/v2 v2.22.1 github.com/onsi/gomega v1.36.2 github.com/pkg/errors v0.9.1 @@ -29,6 +29,7 @@ require ( go.uber.org/zap v1.27.1 golang.org/x/crypto v0.50.0 golang.org/x/oauth2 v0.36.0 + golang.org/x/sync v0.20.0 google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 google.golang.org/grpc v1.78.0 google.golang.org/protobuf v1.36.11 @@ -126,9 +127,9 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.1 // indirect github.com/wzshiming/dl v0.0.0-20260427082752-19804485f005 // indirect - github.com/wzshiming/httpseek v0.6.0 // indirect + github.com/wzshiming/httpseek v0.6.1 // indirect github.com/wzshiming/ioswmr v0.2.0 // indirect - github.com/wzshiming/xet v0.0.0-20260515075456-afcac81072aa // indirect + github.com/wzshiming/xet v0.0.0-20260617081805-02c9cfb4deca // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/zeebo/blake3 v0.2.4 // indirect go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect @@ -136,7 +137,6 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.22.0 // indirect golang.org/x/net v0.53.0 // indirect - golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.43.0 // indirect golang.org/x/text v0.36.0 // indirect golang.org/x/tools v0.43.0 // indirect diff --git a/go.sum b/go.sum index a1d4544b..25edf5af 100644 --- a/go.sum +++ b/go.sum @@ -240,8 +240,8 @@ github.com/leonelquinteros/gotext v1.5.0 h1:ODY7LzLpZWWSJdAHnzhreOr6cwLXTAmc914F github.com/leonelquinteros/gotext v1.5.0/go.mod h1:OCiUVHuhP9LGFBQ1oAmdtNCHJCiHiQA8lf4nAifHkr0= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/matrixhub-ai/hfd v0.0.0-20260522103102-73bc92d77d19 h1:OrxRrJPrAnROotcuRpDPmRVmGBMdQlZpvGTlZF5WMBQ= -github.com/matrixhub-ai/hfd v0.0.0-20260522103102-73bc92d77d19/go.mod h1:SWICq+Zuz3sUe9nHigwRdr8e9cOZOq7MHCjsjIgGcyI= +github.com/matrixhub-ai/hfd v0.0.0-20260622073022-7b4b945e39d4 h1:ANGcUdDqAwHw0k99RCfRiP5awiZJmqBueUs8Va8WX6s= +github.com/matrixhub-ai/hfd v0.0.0-20260622073022-7b4b945e39d4/go.mod h1:1ttFYcNAx1YrqcFImxFfmk9xORuGmTNtHtXM0JGpA5g= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= @@ -345,12 +345,12 @@ github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/wzshiming/dl v0.0.0-20260427082752-19804485f005 h1:Nac8aZwtkW8GTkxh8mWGF08nVLO6Ze68FWH2I+1BpwI= github.com/wzshiming/dl v0.0.0-20260427082752-19804485f005/go.mod h1:jO3MKwJvcFUxlK9Pd8HpQ0YToDWTB61nvklu/NuYPqw= -github.com/wzshiming/httpseek v0.6.0 h1:RWisrPz5mef1rIp//WXRMlq4m1RN11G9D6YdgtXrkhM= -github.com/wzshiming/httpseek v0.6.0/go.mod h1:YoZhlLIwNjTBDXIT8NpK5zRjOgZouRXPaBfjVXdqMMs= +github.com/wzshiming/httpseek v0.6.1 h1:nmrpxl7SNtj8KmmZ8XRMCnKVfOvj1yNAGXNYtPfeRoA= +github.com/wzshiming/httpseek v0.6.1/go.mod h1:YoZhlLIwNjTBDXIT8NpK5zRjOgZouRXPaBfjVXdqMMs= github.com/wzshiming/ioswmr v0.2.0 h1:PWPtSII1hLz/XIhTxUaEQFe8Vwyb/BXXMg5fUKOpnyw= github.com/wzshiming/ioswmr v0.2.0/go.mod h1:TwwDyS1wnJG3AvKliA+PPB0kliN3yEjsabtH4o7xySQ= -github.com/wzshiming/xet v0.0.0-20260515075456-afcac81072aa h1:RWJKY+ZmhVSiBxe6JGg7Kb+Yk5b44p3bnh4IPi5VQU8= -github.com/wzshiming/xet v0.0.0-20260515075456-afcac81072aa/go.mod h1:AvhOcT0RB0pEwNZvwn5SxGj8MiZBVF6NPHL7OoumYJ8= +github.com/wzshiming/xet v0.0.0-20260617081805-02c9cfb4deca h1:XPJAxtUWWn2ZsQC/V8toPPEEWINBzscEFQFy6eECUHI= +github.com/wzshiming/xet v0.0.0-20260617081805-02c9cfb4deca/go.mod h1:oC+etpyWxYsBQPxBfOtX/YWFKFHLkUsYO0c6zv9+Au8= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 51429953..4ba11eca 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -37,6 +37,7 @@ import ( "github.com/matrixhub-ai/hfd/pkg/mirror" "github.com/matrixhub-ai/hfd/pkg/permission" "github.com/matrixhub-ai/hfd/pkg/receive" + hfds3fs "github.com/matrixhub-ai/hfd/pkg/s3fs" hfdssh "github.com/matrixhub-ai/hfd/pkg/ssh" gitstorage "github.com/matrixhub-ai/hfd/pkg/storage" "github.com/soheilhy/cmux" @@ -251,7 +252,51 @@ func (server *APIServer) initGitStorage() { gitstorage.WithRootDir(server.config.DataDir), ) - lfsStorage := lfs.NewLocal(storage.LFSDir()) + // Initialize LFS storage based on configuration + var lfsStorage lfs.Storage + storageConfig := server.config.APIServer.Storage + if storageConfig.Mode == "s3" && storageConfig.S3.Endpoint != "" { + s3Config := storageConfig.S3 + + // Mount S3 bucket for repositories if enabled + if s3Config.Repositories { + repositoriesDir := storage.RepositoriesDir() + log.Info("mounting S3 bucket for repositories", "bucket", s3Config.Bucket, "path", repositoriesDir) + err := hfds3fs.Mount( + context.Background(), + repositoriesDir, + s3Config.Endpoint, + s3Config.AccessKey, + s3Config.SecretKey, + s3Config.Bucket, + "/repositories/", + s3Config.UsePathStyle, + ) + if err != nil { + log.Error("error mounting S3 bucket for repositories", "bucket", s3Config.Bucket, "path", repositoriesDir, "error", err) + // Don't exit, continue with local repositories + } + // Note: In production, you should handle unmount on shutdown + } + + signEndpoint := s3Config.SignEndpoint + if signEndpoint == "" { + signEndpoint = s3Config.Endpoint + } + lfsStorage = lfs.NewS3( + storage.LFSDir(), + s3Config.Endpoint, + s3Config.AccessKey, + s3Config.SecretKey, + s3Config.Bucket, + s3Config.UsePathStyle, + signEndpoint, + ) + log.Info("using S3 storage", "endpoint", s3Config.Endpoint, "bucket", s3Config.Bucket) + } else { + lfsStorage = lfs.NewLocal(storage.LFSDir()) + log.Info("using local storage", "path", storage.LFSDir()) + } mirrorSourceFunc := server.gitHooks.mirrorSourceFunc mirrorDestinationFunc := server.gitHooks.mirrorDestinationFunc diff --git a/internal/infra/config/config.go b/internal/infra/config/config.go index ad3ec32c..bbc82930 100644 --- a/internal/infra/config/config.go +++ b/internal/infra/config/config.go @@ -108,6 +108,36 @@ type APIServerConfig struct { // Empty means "not configured" and the API returns an empty string so the // UI can hide affected panels. ExternalURL string `yaml:"externalURL"` + // Storage is the storage configuration for git/LFS data + Storage StorageConfig `yaml:"storage"` +} + +// StorageConfig defines the storage backend configuration +type StorageConfig struct { + // Mode is the storage mode: "local" for filesystem, "s3" for S3-compatible object storage + Mode string `yaml:"mode"` + // S3 is the S3 storage configuration (used when mode is "s3") + S3 S3Config `yaml:"s3"` +} + +// S3Config defines S3-compatible object storage configuration +type S3Config struct { + // Endpoint is the S3 endpoint URL (e.g., http://localhost:9000) + Endpoint string `yaml:"endpoint"` + // Bucket is the S3 bucket name + Bucket string `yaml:"bucket"` + // Region is the S3 region + Region string `yaml:"region"` + // AccessKey is the S3 access key + AccessKey string `yaml:"accessKey"` + // SecretKey is the S3 secret key + SecretKey string `yaml:"secretKey"` + // UsePathStyle enables path-style addressing (required for MinIO) + UsePathStyle bool `yaml:"usePathStyle"` + // SignEndpoint is the endpoint for generating presigned URLs (defaults to Endpoint if empty) + SignEndpoint string `yaml:"signEndpoint"` + // Repositories enables storing git repositories in S3 using s3fs (requires privileged mode) + Repositories bool `yaml:"repositories"` } type UIConfig struct {