diff --git a/.github/workflows/bk-ci.yml b/.github/workflows/bk-ci.yml index c38a2680ac1..38b066f24fb 100644 --- a/.github/workflows/bk-ci.yml +++ b/.github/workflows/bk-ci.yml @@ -38,65 +38,10 @@ concurrency: cancel-in-progress: true jobs: - build-and-license-check: - name: PR Validation - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: | - echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT - - - name: Cache local Maven repository - if: steps.check_changes.outputs.docs_only != 'true' - id: cache - uses: actions/cache@v3 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/bookkeeper - !~/.m2/repository/org/apache/distributedlog - key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} - - - name: Set up JDK 11 - if: steps.check_changes.outputs.docs_only != 'true' - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 11 - - - name: Validate pull request - if: steps.check_changes.outputs.docs_only != 'true' - run: mvn clean -T 1C -B -nsu apache-rat:check checkstyle:check spotbugs:check package -Ddistributedlog -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: Check license files - if: steps.check_changes.outputs.docs_only != 'true' - run: dev/check-all-licenses - - - name: Generate Javadoc - if: steps.check_changes.outputs.docs_only != 'true' - run: mvn -B -nsu -am -pl bookkeeper-common,bookkeeper-server,:bookkeeper-stats-api,:bookkeeper-stats-providers,:codahale-metrics-provider,:prometheus-metrics-provider javadoc:aggregate -DskipTests -Pdelombok -Dchesktyle.skip -Dspotbugs.skip - unit-tests: name: ${{ matrix.step_name }} runs-on: ubuntu-latest timeout-minutes: ${{ matrix.timeout || 60 }} - needs: [ 'build-and-license-check' ] - if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }} strategy: fail-fast: false matrix: @@ -104,29 +49,7 @@ jobs: - step_name: Bookie Tests module: bookkeeper-server flag: bookie - test_args: "-Dtest='org.apache.bookkeeper.bookie.**'" - - step_name: Client Tests - module: bookkeeper-server - flag: client - test_args: "-Dtest='org.apache.bookkeeper.client.**'" - - step_name: Remaining Tests - module: bookkeeper-server - flag: remaining - test_args: "-Dtest='org.apache.bookkeeper.replication.**'" - - step_name: Replication Tests - module: bookkeeper-server - flag: replication - test_args: "-Dtest='!org.apache.bookkeeper.client.**,!org.apache.bookkeeper.bookie.**,!org.apache.bookkeeper.replication.**,!org.apache.bookkeeper.tls.**'" - - step_name: TLS Tests - module: bookkeeper-server - flag: tls - test_args: "-Dtest='org.apache.bookkeeper.tls.**'" - - step_name: StreamStorage Tests - test_args: "-f stream/pom.xml -DstreamTests" - flag: stream - - step_name: Shell tests - module: tests/scripts - flag: shell + test_args: "-Dtest='org.apache.bookkeeper.bookie.LedgerCacheTest'" steps: - name: checkout @@ -152,6 +75,11 @@ jobs: distribution: 'temurin' java-version: 11 + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + with: + maven-version: 3.8.7 + - name: Build run: | projects_list= @@ -194,244 +122,7 @@ jobs: path: surefire-reports retention-days: 7 - integration-tests: - name: Integration Tests - runs-on: ubuntu-latest - timeout-minutes: 60 - needs: [ 'build-and-license-check' ] - if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }} - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Cache local Maven repository - id: cache - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/bookkeeper - !~/.m2/repository/org/apache/distributedlog - key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 11 - - - name: Build with Maven - run: mvn -B -nsu clean install -Pdocker -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: Run metadata driver tests - run: mvn -B -nsu -f metadata-drivers/pom.xml test -DintegrationTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: Run all integration tests - run: mvn -B -nsu -f tests/pom.xml test -DintegrationTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: ./dev/ci-tool print_thread_dumps - - backward-compatibility-tests: - name: Backward compatibility tests - runs-on: ubuntu-latest - timeout-minutes: 60 - needs: [ 'build-and-license-check' ] - if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }} - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Cache local Maven repository - id: cache - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/bookkeeper - !~/.m2/repository/org/apache/distributedlog - key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} - - - name: Set up JDK 8 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 8 - - - name: Build with Maven - run: mvn -B -nsu clean install -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: Test current server with old clients - run: mvn -B -nsu -DintegrationTests -pl :backward-compat-current-server-old-clients test - - - name: Test progressive upgrade - run: mvn -B -nsu -DintegrationTests -pl :upgrade test - - - name: Other tests - run: | - mvn -B -nsu -DintegrationTests -pl :bc-non-fips,:hierarchical-ledger-manager,:hostname-bookieid,:old-cookie-new-cluster,:recovery-no-password,:upgrade-direct,:yahoo-custom-version test - - windows-build: - name: Build with windows on JDK 11 - runs-on: windows-latest - timeout-minutes: 30 - needs: [ 'build-and-license-check' ] - if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }} - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Install mingw - run: choco install mingw - - - name: Cache local Maven repository - id: cache - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/bookkeeper - !~/.m2/repository/org/apache/distributedlog - key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} - - - - name: Set up JDK 11 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 11 - - - name: mvn package - run: mvn -B -nsu clean package -DskipTests - - macos-build: - name: Build with macos on JDK 11 - runs-on: macos-latest - timeout-minutes: 30 - needs: [ 'build-and-license-check' ] - if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }} - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Cache local Maven repository - id: cache - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/bookkeeper - !~/.m2/repository/org/apache/distributedlog - key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} - - - name: Set up JDK 11 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 11 - - - name: mvn package - run: mvn -B -nsu clean package -DskipTests - - jdk-compatibility-checks: - name: ${{ matrix.step_name }} - runs-on: ubuntu-latest - timeout-minutes: ${{ matrix.timeout || 60 }} - needs: [ 'build-and-license-check' ] - if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }} - strategy: - fail-fast: false - matrix: - include: - - step_name: Compatibility Check Java8 - jdk_version: 8 - - step_name: Compatibility Check Java11 - jdk_version: 11 - - step_name: Compatibility Check Java17 - jdk_version: 17 - - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Cache local Maven repository - id: cache - uses: actions/cache@v3 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/bookkeeper - !~/.m2/repository/org/apache/distributedlog - key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} - - - - name: Set up JDK ${{ matrix.jdk_version }} - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: ${{ matrix.jdk_version }} - - - name: Build with Maven - run: mvn clean package -B -nsu -DskipBookKeeperServerTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: ./dev/ci-tool print_thread_dumps - - owasp-dependency-check: - name: OWASP Dependency Check - runs-on: ubuntu-latest - timeout-minutes: 60 - needs: [ 'build-and-license-check' ] - if: ${{ needs.build-and-license-check.outputs.need_owasp == 'true' }} - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Cache local Maven repository - id: cache - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/bookkeeper - !~/.m2/repository/org/apache/distributedlog - key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} - - - name: Set up JDK 11 - uses: actions/setup-java@v3 - with: - java-version: 11 - - - name: run "clean install verify" to trigger dependency check - # excluding dlfs because it includes hadoop lib with - # CVEs that we cannot patch up anyways - run: mvn -q -B -ntp clean install verify -Powasp-dependency-check -DskipTests -pl '!stream/distributedlog/io/dlfs' + - name: Setup tmate session + if: failure() + uses: mxschmitt/action-tmate@v3 - - name: Upload report - uses: actions/upload-artifact@v3 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: dependency report - path: target/dependency-check-report.html - retention-days: 7 diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java index 05f740d33a3..394c961cbc9 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java @@ -23,6 +23,7 @@ import static org.apache.bookkeeper.proto.BookieProtocol.FLAG_RECOVERY_ADD; import com.google.common.collect.ImmutableMap; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.netty.buffer.ByteBuf; import io.netty.util.Recycler; import io.netty.util.Recycler.Handle; @@ -80,6 +81,7 @@ class PendingAddOp implements WriteCallback { boolean allowFailFast = false; List ensemble; + @SuppressFBWarnings("IS2_INCONSISTENT_SYNC") static PendingAddOp create(LedgerHandle lh, ClientContext clientCtx, List ensemble, ByteBuf payload, EnumSet writeFlags, diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/PageCacheUtil.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/PageCacheUtil.java index 5dc1a5d8cce..08bf9cfb385 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/PageCacheUtil.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/PageCacheUtil.java @@ -95,7 +95,7 @@ public static void bestEffortRemoveFromPageCache(int fd, long offset, long len) } try { NATIVE_IO.posix_fadvise(fd, offset, len, POSIX_FADV_DONTNEED); - } catch (Exception e) { + } catch (Throwable e) { log.warn("Failed to perform posix_fadvise: {}", e.getMessage()); fadvisePossible = false; } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java index 85c3bca8d7c..6ecd3ca0f9c 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -797,13 +798,16 @@ public void testSortedLedgerFlushFailure() throws Exception { // after flush failure, the bookie is set to readOnly assertTrue("Bookie is expected to be in Read mode", bookie.isReadOnly()); // write fail - bookie.addEntry(generateEntry(1, 3), false, new BookkeeperInternalCallbacks.WriteCallback(){ - public void writeComplete(int rc, long ledgerId, long entryId, BookieId addr, Object ctx){ - LOG.info("fail write to bk"); - assertTrue(rc != OK); + CountDownLatch latch = new CountDownLatch(1); + bookie.addEntry(generateEntry(1, 3), false, new BookkeeperInternalCallbacks.WriteCallback() { + public void writeComplete(int rc, long ledgerId, long entryId, BookieId addr, Object ctx) { + LOG.info("write to bk succeed due to the bookie readOnly mode check is in the request process step"); + assertTrue(rc == OK); + latch.countDown(); } }, null, "passwd".getBytes()); + latch.await(); bookie.shutdown(); } diff --git a/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml b/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml index 09daf196569..dec4118b58a 100644 --- a/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml +++ b/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml @@ -317,6 +317,11 @@ + + + + + diff --git a/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java b/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java index 8df5691ae70..b51fc758b6d 100644 --- a/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java +++ b/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java @@ -23,6 +23,7 @@ import java.util.Arrays; import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.bookie.storage.ldb.ArrayGroupSort; import org.openjdk.jmh.annotations.Benchmark; @@ -55,7 +56,7 @@ public static class TestState { private long[] items; public TestState() { - Random r = new Random(); + Random r = ThreadLocalRandom.current(); for (int i = 0; i < (N * 4); i++) { randomItems[i] = r.nextLong(); } diff --git a/pom.xml b/pom.xml index 43e1217e9ff..d8bb81f571f 100644 --- a/pom.xml +++ b/pom.xml @@ -205,7 +205,7 @@ 1.4.1.Final 0.6.1 9.3 - 4.6.0.0 + 4.7.3.2 1 4.0.0 3.0.1 @@ -1317,6 +1317,7 @@ --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED + --add-opens java.base/java.lang.invoke=ALL-UNNAMED --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.nio.channels.spi=ALL-UNNAMED