Updating examples to use nativelink

TraceMachina · Sep 18, 2024 · bc21dfa · bc21dfa
1 parent 097e83c
commit bc21dfa
Show file tree

Hide file tree

Showing 13 changed files with 269 additions and 342 deletions.
diff --git a/.github/actions/build_buck2_nativelink/action.yml b/.github/actions/build_buck2_nativelink/action.yml
@@ -0,0 +1,53 @@
+name: build_buck2_nativelink
+inputs:
+  NATIVELINK_ENGINE_ADDRESS_SECRET:
+    description: ''
+    required: true
+  NATIVELINK_AC_ADDRESS_SECRET:
+    description: ''
+    required: true
+  NATIVELINK_CAS_ADDRESS_SECRET:
+    description: ''
+    required: true
+  NATIVELINK_HEADER_RW_KEY_SECRET:
+    description: ''
+    required: true
+runs:
+  using: composite
+  steps:
+  - name:
+    run: |-
+      DEBIAN_FRONTEND=noninteractive sudo apt-get install -y clang-format clang-tidy clang-tools clang clangd libc++-dev libc++1 libc++abi-dev libc++abi1 libclang-dev libclang1 liblldb-dev libllvm-ocaml-dev libomp-dev libomp5 lld lldb llvm-dev llvm-runtime llvm python3-clang protobuf-compiler libprotobuf-dev
+    shell: bash
+  - name: Build `buck2` with `buck2` using remote execution
+    run: |-
+      echo "[buck2_re_client]
+      engine_address       = $NATIVELINK_ENGINE_ADDRESS
+      action_cache_address = $NATIVELINK_AC_ADDRESS
+      cas_address          = $NATIVELINK_CAS_ADDRESS
+      http_headers         = x-nativelink-api-key:$NATIVELINK_HEADER_RW_KEY
+      tls = true
+      instance_name = main
+      enabled = true
+      capabilities = true
+      [build]
+      execution_platforms = prelude//platforms:default" > .buckconfig.local
+      cargo install --locked --git https://github.com/facebookincubator/reindeer reindeer
+      nix develop --impure --command $RUNNER_TEMP/artifacts/buck2 clean
+      nix develop --impure --command $RUNNER_TEMP/artifacts/buck2 killall
+      rm -f shim/third-party/rust/Cargo.lock shim/third-party/rust/BUCK
+      reindeer --third-party-dir shim/third-party/rust buckify
+      # nix develop --impure --command $RUNNER_TEMP/artifacts/buck2 debug paranoid enable
+      export BUCK2_RE_DOWNLOAD_CONCURRENCY=5
+      nix develop --impure --command $RUNNER_TEMP/artifacts/buck2 build //:buck2 -v 1 --show-simple-output --ui re --ui io --ui dice --ui debugevents --num-threads 5
+    env:
+      NATIVELINK_ENGINE_ADDRESS: ${{ inputs.NATIVELINK_ENGINE_ADDRESS_SECRET }}
+      NATIVELINK_AC_ADDRESS: ${{ inputs.NATIVELINK_AC_ADDRESS_SECRET }}
+      NATIVELINK_CAS_ADDRESS: ${{ inputs.NATIVELINK_CAS_ADDRESS_SECRET }}
+      NATIVELINK_HEADER_RW_KEY: ${{ inputs.NATIVELINK_HEADER_RW_KEY_SECRET }}
+    shell: bash
+  - name: Buck2 failure
+    if: failure()
+    run: |-
+      nix develop --impure --command $RUNNER_TEMP/artifacts/buck2 log what-failed -v 4
+    shell: bash
diff --git a/.github/actions/build_example_nativelink/action.yml b/.github/actions/build_example_nativelink/action.yml
@@ -0,0 +1,39 @@
+name: build_example_nativelink
+inputs:
+  NATIVELINK_ENGINE_ADDRESS_SECRET:
+    description: ''
+    required: true
+  NATIVELINK_AC_ADDRESS_SECRET:
+    description: ''
+    required: true
+  NATIVELINK_CAS_ADDRESS_SECRET:
+    description: ''
+    required: true
+  NATIVELINK_HEADER_RW_KEY_SECRET:
+    description: ''
+    required: true
+runs:
+  using: composite
+  steps:
+  - name: Build examples/remote_execution/nativelink directory
+    run: |-
+      echo "[buck2_re_client]
+      engine_address       = $NATIVELINK_ENGINE_ADDRESS
+      action_cache_address = $NATIVELINK_AC_ADDRESS
+      cas_address          = $NATIVELINK_CAS_ADDRESS
+      http_headers         = x-nativelink-api-key:$NATIVELINK_HEADER_RW_KEY
+      tls = true
+      instance_name = main
+      enabled = true
+      capabilities = true
+      [build]
+      execution_platforms = root//platforms:platforms" > examples/remote_execution/nativelink/.buckconfig.local
+      cd examples/remote_execution/nativelink
+      export BUCK2_RE_DOWNLOAD_CONCURRENCY=25
+      $RUNNER_TEMP/artifacts/buck2 build //... -v 1 --show-simple-output --ui re --ui io --ui dice --ui debugevents --num-threads 10
+    env:
+      NATIVELINK_ENGINE_ADDRESS: ${{ inputs.NATIVELINK_ENGINE_ADDRESS_SECRET }}
+      NATIVELINK_AC_ADDRESS: ${{ inputs.NATIVELINK_AC_ADDRESS_SECRET }}
+      NATIVELINK_CAS_ADDRESS: ${{ inputs.NATIVELINK_CAS_ADDRESS_SECRET }}
+      NATIVELINK_HEADER_RW_KEY: ${{ inputs.NATIVELINK_HEADER_RW_KEY_SECRET }}
+    shell: bash
diff --git a/.github/actions/build_example_no_prelude/action.yml b/.github/actions/build_example_no_prelude/action.yml
@@ -1,9 +1,39 @@
 name: build_example_no_prelude
+inputs:
+  NATIVELINK_ENGINE_ADDRESS_SECRET:
+    description: ''
+    required: true
+  NATIVELINK_AC_ADDRESS_SECRET:
+    description: ''
+    required: true
+  NATIVELINK_CAS_ADDRESS_SECRET:
+    description: ''
+    required: true
+  NATIVELINK_HEADER_RW_KEY_SECRET:
+    description: ''
+    required: true
 runs:
   using: composite
   steps:
   - name: Build example/no_prelude directory
     run: |-
+      echo "[buck2_re_client]
+      engine_address       = $NATIVELINK_ENGINE_ADDRESS
+      action_cache_address = $NATIVELINK_AC_ADDRESS
+      cas_address          = $NATIVELINK_CAS_ADDRESS
+      http_headers         = x-nativelink-api-key:$NATIVELINK_HEADER_RW_KEY
+      tls = true
+      instance_name = main
+      enabled = true
+      capabilities = true
+      [build]
+      execution_platforms = root//platforms:platforms" > examples/no_prelude/.buckconfig.local
       cd examples/no_prelude
-      $RUNNER_TEMP/artifacts/buck2 build //... -v 2
+      export BUCK2_RE_DOWNLOAD_CONCURRENCY=25
+      $RUNNER_TEMP/artifacts/buck2 build //... -v 1 --show-simple-output --ui re --ui io --ui dice --ui debugevents --num-threads 10
+    env:
+      NATIVELINK_ENGINE_ADDRESS: ${{ inputs.NATIVELINK_ENGINE_ADDRESS_SECRET }}
+      NATIVELINK_AC_ADDRESS: ${{ inputs.NATIVELINK_AC_ADDRESS_SECRET }}
+      NATIVELINK_CAS_ADDRESS: ${{ inputs.NATIVELINK_CAS_ADDRESS_SECRET }}
+      NATIVELINK_HEADER_RW_KEY: ${{ inputs.NATIVELINK_HEADER_RW_KEY_SECRET }}
     shell: bash
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -1,93 +1,57 @@
-name: Build and test
+name: Build and test nativelink
 on:
   push:
-  pull_request:
+  workflow_dispatch:
+  workflow_call:
+  workflow_run:
+    workflows: ["rebase upstream manual", "rebase upstream cron"]
+    types:
+      - completed
 jobs:
-  linux-build-and-test:
-    runs-on: 4-core-ubuntu
-    steps:
-    - uses: actions/[email protected]
-    - uses: ./.github/actions/setup_linux_env
-    - uses: ./.github/actions/build_debug
-    - uses: ./.github/actions/run_test_py
-  macos-build-and-test:
-    runs-on: macos-latest
-    steps:
-    - uses: maxim-lobanov/[email protected]
-      with:
-        xcode-version: 15.3.0
-    - uses: actions/[email protected]
-    - uses: ./.github/actions/setup_macos_env
-    - uses: ./.github/actions/build_debug
-    - uses: ./.github/actions/run_test_py
-  windows-build-and-test:
-    runs-on: windows-latest
-    steps:
-    - uses: actions/[email protected]
-    - uses: ./.github/actions/setup_windows_env
-    - uses: ./.github/actions/build_debug
-    - uses: ./.github/actions/run_test_py
-  macos-build-examples:
-    runs-on: macos-latest
-    steps:
-    - uses: maxim-lobanov/[email protected]
-      with:
-        xcode-version: 15.3.0
-    - uses: actions/[email protected]
-    - uses: ./.github/actions/setup_macos_env
-    - uses: ./.github/actions/init_opam
-    - uses: ./.github/actions/build_release
-    - name: Setup the 'example/with_prelude' project
-      run: |-
-        cd examples/with_prelude
-        ./haskell-setup.sh
-        ./ocaml-setup.sh
-    - name: Build the 'example/with_prelude' project
-      run: |-
-        cd examples/with_prelude
-        $RUNNER_TEMP/artifacts/buck2 build //... -v 2
-        $RUNNER_TEMP/artifacts/buck2 test //... -v 2
-    - uses: ./.github/actions/build_example_no_prelude
-    - uses: ./.github/actions/setup_reindeer
-    - uses: ./.github/actions/build_bootstrap
   linux-build-examples:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/[email protected]
     - uses: ./.github/actions/setup_linux_env
     - uses: ./.github/actions/init_opam
-    - uses: ./.github/actions/build_release
-    - name: Setup the 'example/with_prelude' project
-      run: |-
-        cd examples/with_prelude
-        ./haskell-setup.sh
-        ./ocaml-setup.sh
-    - name: Build the 'example/with_prelude' project
-      run: |-
-        cd examples/with_prelude
-        $RUNNER_TEMP/artifacts/buck2 build //... -v 2
-        $RUNNER_TEMP/artifacts/buck2 test //... -v 2
-    - uses: ./.github/actions/build_example_conan
+    - uses: ./.github/actions/build_debug
+    - uses: ./.github/actions/build_example_nativelink
+      with:
+        NATIVELINK_ENGINE_ADDRESS_SECRET: ${{ secrets.NATIVELINK_ENGINE_ADDRESS_SECRET }}
+        NATIVELINK_AC_ADDRESS_SECRET: ${{ secrets.NATIVELINK_AC_ADDRESS_SECRET }}
+        NATIVELINK_CAS_ADDRESS_SECRET: ${{ secrets.NATIVELINK_CAS_ADDRESS_SECRET }}
+        NATIVELINK_HEADER_RW_KEY_SECRET: ${{ secrets.NATIVELINK_HEADER_RW_KEY_SECRET }}
     - uses: ./.github/actions/build_example_no_prelude
+      with:
+        NATIVELINK_ENGINE_ADDRESS_SECRET: ${{ secrets.NATIVELINK_ENGINE_ADDRESS_SECRET }}
+        NATIVELINK_AC_ADDRESS_SECRET: ${{ secrets.NATIVELINK_AC_ADDRESS_SECRET }}
+        NATIVELINK_CAS_ADDRESS_SECRET: ${{ secrets.NATIVELINK_CAS_ADDRESS_SECRET }}
+        NATIVELINK_HEADER_RW_KEY_SECRET: ${{ secrets.NATIVELINK_HEADER_RW_KEY_SECRET }}
     - uses: ./.github/actions/setup_reindeer
-    - uses: ./.github/actions/build_bootstrap
-  windows-build-examples:
-    runs-on: windows-latest
-    steps:
-      - uses: actions/[email protected]
-      - uses: ./.github/actions/setup_windows_env
-      - uses: ./.github/actions/build_release
-      - name: Build example/prelude directory
-        run: |-
-          cd examples/with_prelude
-          & $Env:RUNNER_TEMP/artifacts/buck2 build //... -v 2
-          & $Env:RUNNER_TEMP/artifacts/buck2 test //... -v 2
-      - uses: ./.github/actions/build_example_no_prelude
-      - name: Configure CARGO_HOME
-        run: |-
-          echo CARGO_HOME=$GITHUB_WORKSPACE/.cargo >> $GITHUB_ENV
-          echo $GITHUB_WORKSPACE/.cargo/bin >> $GITHUB_PATH
-        shell:
-          bash
-      - uses: ./.github/actions/setup_reindeer
-      - uses: ./.github/actions/build_bootstrap
+    #- uses: ./.github/actions/build_bootstrap
+    - uses: DeterminateSystems/nix-installer-action@de22e16c4711fca50c816cc9081563429d1cf563
+      name: Install Nix
+    - uses: endersonmenezes/free-disk-space@3f9ec39ebae520864ac93467ee395f5237585c21
+      name: Free disk space
+      with:
+        remove_android: true
+        remove_dotnet: true
+        remove_haskell: true
+        remove_tool_cache: false
+    - uses: DeterminateSystems/magic-nix-cache-action@fc6aaceb40b9845a02b91e059ec147e78d1b4e41
+      name: Cache Nix derivations
+    # Required due to the mismatch of protoc and nix prebuild bins
+    # - name: Install Protoc
+    #   uses: arduino/setup-protoc@v3
+    #   with:
+    #     version: "21.4"
+    - uses: ./.github/actions/build_buck2_nativelink
+      with:
+        NATIVELINK_ENGINE_ADDRESS_SECRET: ${{ secrets.NATIVELINK_ENGINE_ADDRESS_SECRET }}
+        NATIVELINK_AC_ADDRESS_SECRET: ${{ secrets.NATIVELINK_AC_ADDRESS_SECRET }}
+        NATIVELINK_CAS_ADDRESS_SECRET: ${{ secrets.NATIVELINK_CAS_ADDRESS_SECRET }}
+        NATIVELINK_HEADER_RW_KEY_SECRET: ${{ secrets.NATIVELINK_HEADER_RW_KEY_SECRET }}
+    # - name: what-ran
+    #   run: |-
+    #     $RUNNER_TEMP/artifacts/buck2 log what-ran --format json | jq .
+    #   shell: bash
diff --git a/.github/workflows/rebase-upstream-cron.yml b/.github/workflows/rebase-upstream-cron.yml
@@ -0,0 +1,24 @@
+name: rebase upstream cron
+on:
+  schedule:
+    - cron:  '0 15 * * *'
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Rebase
+      run: |
+        git config --local user.email "[email protected]"
+        git config --local user.name "github-actions"
+        git remote add upstream http://github.com/facebook/buck2.git
+        git fetch upstream
+        git rebase upstream/main 
+    - name: Push changes
+      uses: ad-m/github-push-action@master
+      with:
+        force_with_lease: true
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        branch: ${{ github.ref }}
diff --git a/.github/workflows/rebase-upstream-manual.yml b/.github/workflows/rebase-upstream-manual.yml
@@ -0,0 +1,22 @@
+name: rebase upstream manual
+on: workflow_dispatch
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Rebase
+      run: |
+        git config --local user.email "[email protected]"
+        git config --local user.name "github-actions"
+        git remote add upstream http://github.com/facebook/buck2.git
+        git fetch upstream
+        git rebase upstream/main 
+    - name: Push changes
+      uses: ad-m/github-push-action@master
+      with:
+        force_with_lease: true
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        branch: ${{ github.ref }}