diff --git a/.github/workflows/build-renku-release.yml b/.github/workflows/build-renku-release.yml new file mode 100644 index 0000000000000..5a3b8833f097a --- /dev/null +++ b/.github/workflows/build-renku-release.yml @@ -0,0 +1,197 @@ +name: Build rclone release for Renku + +# Trigger the workflow on tag pushes or manually +on: + push: + tags: + - '**' + workflow_dispatch: + +jobs: + build: + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os'] + + include: + - job_name: linux + os: ubuntu-latest + go: '>=1.23.0-rc.1' + gotags: cmount + build_flags: '-include "^linux/"' + check: true + quicktest: true + racequicktest: true + librclonetest: true + deploy: true + + - job_name: linux_386 + os: ubuntu-latest + go: '>=1.23.0-rc.1' + goarch: 386 + gotags: cmount + quicktest: true + + - job_name: mac_amd64 + os: macos-latest + go: '>=1.23.0-rc.1' + gotags: 'cmount' + build_flags: '-include "^darwin/amd64" -cgo' + quicktest: true + racequicktest: true + deploy: true + + - job_name: mac_arm64 + os: macos-latest + go: '>=1.23.0-rc.1' + gotags: 'cmount' + build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib' + deploy: true + + - job_name: windows + os: windows-latest + go: '>=1.23.0-rc.1' + gotags: cmount + cgo: '0' + build_flags: '-include "^windows/"' + build_args: '-buildmode exe' + quicktest: true + deploy: true + + - job_name: other_os + os: ubuntu-latest + go: '>=1.23.0-rc.1' + build_flags: '-exclude "^(windows/|darwin/|linux/)"' + compile_all: true + deploy: true + + name: ${{ matrix.job_name }} + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go }} + check-latest: true + + - name: Set environment variables + shell: bash + run: | + echo 'GOTAGS=${{ matrix.gotags }}' >> $GITHUB_ENV + echo 'BUILD_FLAGS=${{ matrix.build_flags }}' >> $GITHUB_ENV + echo 'BUILD_ARGS=${{ matrix.build_args }}' >> $GITHUB_ENV + if [[ "${{ matrix.goarch }}" != "" ]]; then echo 'GOARCH=${{ matrix.goarch }}' >> $GITHUB_ENV ; fi + if [[ "${{ matrix.cgo }}" != "" ]]; then echo 'CGO_ENABLED=${{ matrix.cgo }}' >> $GITHUB_ENV ; fi + + - name: Install Libraries on Linux + shell: bash + run: | + sudo modprobe fuse + sudo chmod 666 /dev/fuse + sudo chown root:$USER /etc/fuse.conf + sudo apt-get update + sudo apt-get install -y fuse3 libfuse-dev rpm pkg-config git-annex git-annex-remote-rclone nfs-common + if: matrix.os == 'ubuntu-latest' + + - name: Install Libraries on macOS + shell: bash + run: | + # https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788 + # https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008 + unset HOMEBREW_NO_INSTALL_FROM_API + brew untap --force homebrew/core + brew untap --force homebrew/cask + brew update + brew install --cask macfuse + brew install git-annex git-annex-remote-rclone + if: matrix.os == 'macos-latest' + + - name: Install Libraries on Windows + shell: powershell + run: | + $ProgressPreference = 'SilentlyContinue' + choco install -y winfsp zip + echo "CPATH=C:\Program Files\WinFsp\inc\fuse;C:\Program Files (x86)\WinFsp\inc\fuse" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + if ($env:GOARCH -eq "386") { + choco install -y mingw --forcex86 --force + echo "C:\\ProgramData\\chocolatey\\lib\\mingw\\tools\\install\\mingw32\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + } + # Copy mingw32-make.exe to make.exe so the same command line + # can be used on Windows as on macOS and Linux + $path = (get-command mingw32-make.exe).Path + Copy-Item -Path $path -Destination (Join-Path (Split-Path -Path $path) 'make.exe') + if: matrix.os == 'windows-latest' + + - name: Print Go version and environment + shell: bash + run: | + printf "Using go at: $(which go)\n" + printf "Go version: $(go version)\n" + printf "\n\nGo environment:\n\n" + go env + printf "\n\nRclone environment:\n\n" + make vars + printf "\n\nSystem environment:\n\n" + env + + - name: Build rclone + shell: bash + run: | + make + + - name: Rclone version + shell: bash + run: | + rclone version + + - name: Run tests + shell: bash + run: | + make quicktest + if: matrix.quicktest + + - name: Race test + shell: bash + run: | + make racequicktest + if: matrix.racequicktest + + - name: Run librclone tests + shell: bash + run: | + make -C librclone/ctest test + make -C librclone/ctest clean + librclone/python/test_rclone.py + if: matrix.librclonetest + + - name: Compile all architectures test + shell: bash + run: | + make + make compile_all + if: matrix.compile_all + + - name: Build binaries for release + shell: bash + run: | + if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi + make ci_gha + ls -al build/ + if: matrix.deploy + + - name: Upload artifacts to GitHub + uses: actions/upload-artifact@v4 + with: + name: build-${{ matrix.job_name }} + path: build + retention-days: 7 + if: matrix.deploy diff --git a/.github/workflows/build-renku.yml b/.github/workflows/build-renku.yml new file mode 100644 index 0000000000000..66719c58661a0 --- /dev/null +++ b/.github/workflows/build-renku.yml @@ -0,0 +1,237 @@ +name: Build rclone for Renku + +# Trigger the workflow on push or pull request +on: + push: + branches: + - "**" + tags: + - "**" + pull_request: + workflow_dispatch: + inputs: + manual: + description: Manual run (bypass default conditions) + type: boolean + default: true + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + if: inputs.manual || (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name) + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + job_name: ["linux"] + + include: + - job_name: linux + os: ubuntu-latest + go: ">=1.23.0-rc.1" + gotags: cmount + build_flags: '-include "linux/amd64|linux/arm64"' + deploy: true + + name: ${{ matrix.job_name }} + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go }} + check-latest: true + + - name: Set environment variables + shell: bash + run: | + echo 'GOTAGS=${{ matrix.gotags }}' >> $GITHUB_ENV + echo 'BUILD_FLAGS=${{ matrix.build_flags }}' >> $GITHUB_ENV + echo 'BUILD_ARGS=${{ matrix.build_args }}' >> $GITHUB_ENV + if [[ "${{ matrix.goarch }}" != "" ]]; then echo 'GOARCH=${{ matrix.goarch }}' >> $GITHUB_ENV ; fi + if [[ "${{ matrix.cgo }}" != "" ]]; then echo 'CGO_ENABLED=${{ matrix.cgo }}' >> $GITHUB_ENV ; fi + + - name: Install Libraries on Linux + shell: bash + run: | + sudo modprobe fuse + sudo chmod 666 /dev/fuse + sudo chown root:$USER /etc/fuse.conf + sudo apt-get update + sudo apt-get install -y fuse3 libfuse-dev rpm pkg-config git-annex git-annex-remote-rclone nfs-common + if: matrix.os == 'ubuntu-latest' + + - name: Install Libraries on macOS + shell: bash + run: | + # https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788 + # https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008 + unset HOMEBREW_NO_INSTALL_FROM_API + brew untap --force homebrew/core + brew untap --force homebrew/cask + brew update + brew install --cask macfuse + brew install git-annex git-annex-remote-rclone + if: matrix.os == 'macos-latest' + + - name: Install Libraries on Windows + shell: powershell + run: | + $ProgressPreference = 'SilentlyContinue' + choco install -y winfsp zip + echo "CPATH=C:\Program Files\WinFsp\inc\fuse;C:\Program Files (x86)\WinFsp\inc\fuse" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + if ($env:GOARCH -eq "386") { + choco install -y mingw --forcex86 --force + echo "C:\\ProgramData\\chocolatey\\lib\\mingw\\tools\\install\\mingw32\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + } + # Copy mingw32-make.exe to make.exe so the same command line + # can be used on Windows as on macOS and Linux + $path = (get-command mingw32-make.exe).Path + Copy-Item -Path $path -Destination (Join-Path (Split-Path -Path $path) 'make.exe') + if: matrix.os == 'windows-latest' + + - name: Print Go version and environment + shell: bash + run: | + printf "Using go at: $(which go)\n" + printf "Go version: $(go version)\n" + printf "\n\nGo environment:\n\n" + go env + printf "\n\nRclone environment:\n\n" + make vars + printf "\n\nSystem environment:\n\n" + env + + - name: Build rclone + shell: bash + run: | + make + + - name: Rclone version + shell: bash + run: | + rclone version + + - name: Run tests + shell: bash + run: | + make quicktest + if: matrix.quicktest + + - name: Race test + shell: bash + run: | + make racequicktest + if: matrix.racequicktest + + - name: Run librclone tests + shell: bash + run: | + make -C librclone/ctest test + make -C librclone/ctest clean + librclone/python/test_rclone.py + if: matrix.librclonetest + + - name: Compile all architectures test + shell: bash + run: | + make + make compile_all + if: matrix.compile_all + + - name: Build binaries for release + shell: bash + run: | + if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi + make ci_gha + ls -al build/ + if: matrix.deploy + + - name: Upload artifacts to GitHub + uses: actions/upload-artifact@v4 + with: + name: build-${{ matrix.job_name }} + path: build + retention-days: 1 + if: matrix.deploy + + image: + if: inputs.manual || (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name) + timeout-minutes: 60 + runs-on: ubuntu-latest + needs: [build] + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: build-linux + path: rclone + + - name: List downloaded files + run: ls -R rclone + + - name: Extract rclone binary - amd64 + run: | + unzip "rclone/*-amd64.zip" -d rclone-unzip + ls -R rclone-unzip + mkdir -p linux/amd64 + mv rclone-unzip/*/rclone linux/amd64/rclone + - name: Extract rclone binary - arm64 + run: | + unzip "rclone/*-arm64.zip" -d rclone-unzip + ls -R rclone-unzip + mkdir -p linux/arm64 + mv rclone-unzip/*/rclone linux/arm64/rclone + - name: Docker image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: type=sha + + - name: Extract Docker image name + id: docker_image + env: + IMAGE_TAGS: ${{ steps.meta.outputs.tags }} + run: | + IMAGE=$(echo "$IMAGE_TAGS" | cut -d" " -f1) + IMAGE_REPOSITORY=$(echo "$IMAGE" | cut -d":" -f1) + IMAGE_TAG=$(echo "$IMAGE" | cut -d":" -f2) + echo "image=$IMAGE" >> "$GITHUB_OUTPUT" + echo "image_repository=$IMAGE_REPOSITORY" >> "$GITHUB_OUTPUT" + echo "image_tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT" + - name: Set up Docker buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up Docker + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: . + file: .renku/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a5afa9e1f10bc..feae79bc9ea57 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,7 +21,7 @@ on: jobs: build: - if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) + if: inputs.manual || (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name) timeout-minutes: 60 defaults: run: @@ -29,7 +29,7 @@ jobs: strategy: fail-fast: false matrix: - job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.24'] + job_name: ['linux', 'linux_386', 'other_os', 'go1.24'] include: - job_name: linux @@ -50,31 +50,31 @@ jobs: gotags: cmount quicktest: true - - job_name: mac_amd64 - os: macos-latest - go: '>=1.25.0-rc.1' - gotags: 'cmount' - build_flags: '-include "^darwin/amd64" -cgo' - quicktest: true - racequicktest: true - deploy: true - - - job_name: mac_arm64 - os: macos-latest - go: '>=1.25.0-rc.1' - gotags: 'cmount' - build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib' - deploy: true - - - job_name: windows - os: windows-latest - go: '>=1.25.0-rc.1' - gotags: cmount - cgo: '0' - build_flags: '-include "^windows/"' - build_args: '-buildmode exe' - quicktest: true - deploy: true + # - job_name: mac_amd64 + # os: macos-latest + # go: '>=1.25.0-rc.1' + # gotags: 'cmount' + # build_flags: '-include "^darwin/amd64" -cgo' + # quicktest: true + # racequicktest: true + # deploy: true + + # - job_name: mac_arm64 + # os: macos-latest + # go: '>=1.25.0-rc.1' + # gotags: 'cmount' + # build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib' + # deploy: true + + # - job_name: windows + # os: windows-latest + # go: '>=1.25.0-rc.1' + # gotags: cmount + # cgo: '0' + # build_flags: '-include "^windows/"' + # build_args: '-buildmode exe' + # quicktest: true + # deploy: true - job_name: other_os os: ubuntu-latest @@ -192,18 +192,18 @@ jobs: make compile_all if: matrix.compile_all - - name: Deploy built binaries - run: | - if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi - make ci_beta - env: - RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }} - # working-directory: '$(modulePath)' - # Deploy binaries if enabled in config && not a PR && not a fork - if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone' + # - name: Deploy built binaries + # run: | + # if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi + # make ci_beta + # env: + # RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }} + # # working-directory: '$(modulePath)' + # # Deploy binaries if enabled in config && not a PR && not a fork + # if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone' lint: - if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) + if: inputs.manual || (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name) timeout-minutes: 30 name: "lint" runs-on: ubuntu-latest diff --git a/.renku/Dockerfile b/.renku/Dockerfile new file mode 100644 index 0000000000000..a3e34b8007186 --- /dev/null +++ b/.renku/Dockerfile @@ -0,0 +1,3 @@ +FROM scratch +ARG TARGETPLATFORM +COPY ${TARGETPLATFORM}/rclone rclone diff --git a/Makefile b/Makefile index cbec6d16c1938..5473603d82c4b 100644 --- a/Makefile +++ b/Makefile @@ -239,6 +239,9 @@ ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),) endif @echo Beta release ready at $(BETA_URL) +ci_gha: + go run bin/cross-compile.go $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG) + # Fetch the binary builds from GitHub actions fetch_binaries: rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/ diff --git a/vfs/vfs.go b/vfs/vfs.go index 1d2f71dd109fc..33ce1151fcd30 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -352,7 +352,7 @@ func (vfs *VFS) SetCacheMode(cacheMode vfscommon.CacheMode) { vfs.cache = nil if cacheMode > vfscommon.CacheModeOff { ctx, cancel := context.WithCancel(context.Background()) - cache, err := vfscache.New(ctx, vfs.f, &vfs.Opt, vfs.AddVirtual) // FIXME pass on context or get from Opt? + cache, err := vfscache.New(ctx, vfs.f, &vfs.Opt, vfs.AddVirtual, totalCacheUsed) // FIXME pass on context or get from Opt? if err != nil { fs.Errorf(nil, "Failed to create vfs cache - disabling: %v", err) vfs.Opt.CacheMode = vfscommon.CacheModeOff @@ -849,6 +849,18 @@ func (vfs *VFS) AddVirtual(remote string, size int64, isDir bool) (err error) { return nil } +// totalCacheUsed returns the total size used by VFS caches +func totalCacheUsed() (size int64) { + activeMu.Lock() + for _, vfses := range active { + for _, vfs := range vfses { + size += vfs.cache.Used() + } + } + activeMu.Unlock() + return size +} + // Readlink returns the destination of the named symbolic link. // If there is an error, it will be of type *PathError. func (vfs *VFS) Readlink(name string) (s string, err error) { diff --git a/vfs/vfscache/cache.go b/vfs/vfscache/cache.go index 0ffa30c76ecb4..562cc03225712 100644 --- a/vfs/vfscache/cache.go +++ b/vfs/vfscache/cache.go @@ -51,6 +51,7 @@ type Cache struct { hashOption *fs.HashesOption // corresponding OpenOption writeback *writeback.WriteBack // holds Items for writeback avFn AddVirtualFn // if set, can be called to add dir entries + tuFn TotalUsedFn // if set, can be called to check quotas mu sync.Mutex // protects the following variables cond sync.Cond // cond lock for synchronous cache cleaning @@ -72,11 +73,14 @@ type Cache struct { // go into the directory tree. type AddVirtualFn func(remote string, size int64, isDir bool) error +// TotalUsedFn can be used to check if the total size of caches exceeds the allowed quota. +type TotalUsedFn func() (size int64) + // New creates a new cache hierarchy for fremote // // This starts background goroutines which can be cancelled with the // context passed in. -func New(ctx context.Context, fremote fs.Fs, opt *vfscommon.Options, avFn AddVirtualFn) (*Cache, error) { +func New(ctx context.Context, fremote fs.Fs, opt *vfscommon.Options, avFn AddVirtualFn, tuFn TotalUsedFn) (*Cache, error) { // Get cache root path. // We need it in two variants: OS path as an absolute path with UNC prefix, // OS-specific path separators, and encoded with OS-specific encoder. Standard path @@ -127,6 +131,7 @@ func New(ctx context.Context, fremote fs.Fs, opt *vfscommon.Options, avFn AddVir hashOption: hashOption, writeback: writeback.New(ctx, opt), avFn: avFn, + tuFn: tuFn, } // load in the cache and metadata off disk @@ -739,16 +744,39 @@ func (c *Cache) maxSizeQuotaOK() bool { return c.used <= int64(c.opt.CacheMaxSize) } +// Check the available quota for a disk is in limits wrt. DiskSpaceTotalSize. +// +// must be called with mu held. +func (c *Cache) totalSizeQuotaOK() bool { + if c.opt.DiskSpaceTotalSize <= 0 { + return true + } + var margin int64 + if c.opt.CacheMinFreeSpace > 0 { + margin = int64(c.opt.CacheMinFreeSpace) + } + if (c.used + margin) > int64(c.opt.DiskSpaceTotalSize) { + return false + } + if c.tuFn != nil { + totalUsed := c.tuFn() + if (totalUsed + margin) > int64(c.opt.DiskSpaceTotalSize) { + return false + } + } + return true +} + // Check the available quotas for a disk is in limits. // // must be called with mu held. func (c *Cache) quotasOK() bool { - return c.maxSizeQuotaOK() && c.minFreeSpaceQuotaOK() + return c.maxSizeQuotaOK() && c.minFreeSpaceQuotaOK() && c.totalSizeQuotaOK() } // Return true if any quotas set func (c *Cache) haveQuotas() bool { - return c.opt.CacheMaxSize > 0 || c.opt.CacheMinFreeSpace > 0 + return c.opt.CacheMaxSize > 0 || c.opt.CacheMinFreeSpace > 0 || c.opt.DiskSpaceTotalSize > 0 } // Remove clean cache files that are not open until the total space @@ -899,3 +927,8 @@ func (c *Cache) AddVirtual(remote string, size int64, isDir bool) error { } return c.avFn(remote, size, isDir) } + +// Used returns the current bytes used by this cache +func (c *Cache) Used() (size int64) { + return c.used +} diff --git a/vfs/vfscache/cache_test.go b/vfs/vfscache/cache_test.go index 8c5510f6d37f2..1e11eb478d0be 100644 --- a/vfs/vfscache/cache_test.go +++ b/vfs/vfscache/cache_test.go @@ -91,7 +91,7 @@ func newTestCacheOpt(t *testing.T, opt vfscommon.Options) (r *fstest.Run, c *Cac ctx, cancel := context.WithCancel(context.Background()) avInfos = nil - c, err := New(ctx, r.Fremote, &opt, addVirtual) + c, err := New(ctx, r.Fremote, &opt, addVirtual, nil) require.NoError(t, err) t.Cleanup(func() {