Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
327 changes: 9 additions & 318 deletions .github/workflows/bk-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,95 +38,18 @@ concurrency:
cancel-in-progress: true

jobs:
build-and-license-check:
name: PR Validation
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- name: Checkout
uses: actions/checkout@v2

- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm

- name: Detect changed files
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
filters: .github/changes-filter.yaml
list-files: csv

- name: Check changed files
id: check_changes
run: |
echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT

- name: Cache local Maven repository
if: steps.check_changes.outputs.docs_only != 'true'
id: cache
uses: actions/cache@v3
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/bookkeeper
!~/.m2/repository/org/apache/distributedlog
key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}

- name: Set up JDK 11
if: steps.check_changes.outputs.docs_only != 'true'
uses: actions/setup-java@v2
with:
distribution: 'temurin'
java-version: 11

- name: Validate pull request
if: steps.check_changes.outputs.docs_only != 'true'
run: mvn clean -T 1C -B -nsu apache-rat:check checkstyle:check spotbugs:check package -Ddistributedlog -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO

- name: Check license files
if: steps.check_changes.outputs.docs_only != 'true'
run: dev/check-all-licenses

- name: Generate Javadoc
if: steps.check_changes.outputs.docs_only != 'true'
run: mvn -B -nsu -am -pl bookkeeper-common,bookkeeper-server,:bookkeeper-stats-api,:bookkeeper-stats-providers,:codahale-metrics-provider,:prometheus-metrics-provider javadoc:aggregate -DskipTests -Pdelombok -Dchesktyle.skip -Dspotbugs.skip

unit-tests:
name: ${{ matrix.step_name }}
runs-on: ubuntu-latest
timeout-minutes: ${{ matrix.timeout || 60 }}
needs: [ 'build-and-license-check' ]
if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
strategy:
fail-fast: false
matrix:
include:
- step_name: Bookie Tests
module: bookkeeper-server
flag: bookie
test_args: "-Dtest='org.apache.bookkeeper.bookie.**'"
- step_name: Client Tests
module: bookkeeper-server
flag: client
test_args: "-Dtest='org.apache.bookkeeper.client.**'"
- step_name: Remaining Tests
module: bookkeeper-server
flag: remaining
test_args: "-Dtest='org.apache.bookkeeper.replication.**'"
- step_name: Replication Tests
module: bookkeeper-server
flag: replication
test_args: "-Dtest='!org.apache.bookkeeper.client.**,!org.apache.bookkeeper.bookie.**,!org.apache.bookkeeper.replication.**,!org.apache.bookkeeper.tls.**'"
- step_name: TLS Tests
module: bookkeeper-server
flag: tls
test_args: "-Dtest='org.apache.bookkeeper.tls.**'"
- step_name: StreamStorage Tests
test_args: "-f stream/pom.xml -DstreamTests"
flag: stream
- step_name: Shell tests
module: tests/scripts
flag: shell
test_args: "-Dtest='org.apache.bookkeeper.bookie.LedgerCacheTest'"

steps:
- name: checkout
Expand All @@ -152,6 +75,11 @@ jobs:
distribution: 'temurin'
java-version: 11

- name: Set up Maven
uses: apache/pulsar-test-infra/setup-maven@master
with:
maven-version: 3.8.7

- name: Build
run: |
projects_list=
Expand Down Expand Up @@ -194,244 +122,7 @@ jobs:
path: surefire-reports
retention-days: 7

integration-tests:
name: Integration Tests
runs-on: ubuntu-latest
timeout-minutes: 60
needs: [ 'build-and-license-check' ]
if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
steps:
- name: Checkout
uses: actions/checkout@v2

- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm

- name: Cache local Maven repository
id: cache
uses: actions/cache@v2
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/bookkeeper
!~/.m2/repository/org/apache/distributedlog
key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}

- name: Set up JDK 11
uses: actions/setup-java@v2
with:
distribution: 'temurin'
java-version: 11

- name: Build with Maven
run: mvn -B -nsu clean install -Pdocker -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO

- name: Run metadata driver tests
run: mvn -B -nsu -f metadata-drivers/pom.xml test -DintegrationTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO

- name: Run all integration tests
run: mvn -B -nsu -f tests/pom.xml test -DintegrationTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO

- name: print JVM thread dumps when cancelled
if: cancelled()
run: ./dev/ci-tool print_thread_dumps

backward-compatibility-tests:
name: Backward compatibility tests
runs-on: ubuntu-latest
timeout-minutes: 60
needs: [ 'build-and-license-check' ]
if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
steps:
- name: Checkout
uses: actions/checkout@v2

- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm

- name: Cache local Maven repository
id: cache
uses: actions/cache@v2
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/bookkeeper
!~/.m2/repository/org/apache/distributedlog
key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}

- name: Set up JDK 8
uses: actions/setup-java@v2
with:
distribution: 'temurin'
java-version: 8

- name: Build with Maven
run: mvn -B -nsu clean install -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO

- name: Test current server with old clients
run: mvn -B -nsu -DintegrationTests -pl :backward-compat-current-server-old-clients test

- name: Test progressive upgrade
run: mvn -B -nsu -DintegrationTests -pl :upgrade test

- name: Other tests
run: |
mvn -B -nsu -DintegrationTests -pl :bc-non-fips,:hierarchical-ledger-manager,:hostname-bookieid,:old-cookie-new-cluster,:recovery-no-password,:upgrade-direct,:yahoo-custom-version test

windows-build:
name: Build with windows on JDK 11
runs-on: windows-latest
timeout-minutes: 30
needs: [ 'build-and-license-check' ]
if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
steps:
- name: Checkout
uses: actions/checkout@v3

- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm

- name: Install mingw
run: choco install mingw

- name: Cache local Maven repository
id: cache
uses: actions/cache@v2
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/bookkeeper
!~/.m2/repository/org/apache/distributedlog
key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}


- name: Set up JDK 11
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: 11

- name: mvn package
run: mvn -B -nsu clean package -DskipTests

macos-build:
name: Build with macos on JDK 11
runs-on: macos-latest
timeout-minutes: 30
needs: [ 'build-and-license-check' ]
if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
steps:
- name: Checkout
uses: actions/checkout@v3

- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm

- name: Cache local Maven repository
id: cache
uses: actions/cache@v2
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/bookkeeper
!~/.m2/repository/org/apache/distributedlog
key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}

- name: Set up JDK 11
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: 11

- name: mvn package
run: mvn -B -nsu clean package -DskipTests

jdk-compatibility-checks:
name: ${{ matrix.step_name }}
runs-on: ubuntu-latest
timeout-minutes: ${{ matrix.timeout || 60 }}
needs: [ 'build-and-license-check' ]
if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
strategy:
fail-fast: false
matrix:
include:
- step_name: Compatibility Check Java8
jdk_version: 8
- step_name: Compatibility Check Java11
jdk_version: 11
- step_name: Compatibility Check Java17
jdk_version: 17

steps:
- name: checkout
uses: actions/checkout@v3

- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm

- name: Cache local Maven repository
id: cache
uses: actions/cache@v3
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/bookkeeper
!~/.m2/repository/org/apache/distributedlog
key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}


- name: Set up JDK ${{ matrix.jdk_version }}
uses: actions/setup-java@v2
with:
distribution: 'temurin'
java-version: ${{ matrix.jdk_version }}

- name: Build with Maven
run: mvn clean package -B -nsu -DskipBookKeeperServerTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO

- name: print JVM thread dumps when cancelled
if: cancelled()
run: ./dev/ci-tool print_thread_dumps

owasp-dependency-check:
name: OWASP Dependency Check
runs-on: ubuntu-latest
timeout-minutes: 60
needs: [ 'build-and-license-check' ]
if: ${{ needs.build-and-license-check.outputs.need_owasp == 'true' }}
steps:
- name: Checkout
uses: actions/checkout@v2

- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm

- name: Cache local Maven repository
id: cache
uses: actions/cache@v2
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/bookkeeper
!~/.m2/repository/org/apache/distributedlog
key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}

- name: Set up JDK 11
uses: actions/setup-java@v3
with:
java-version: 11

- name: run "clean install verify" to trigger dependency check
# excluding dlfs because it includes hadoop lib with
# CVEs that we cannot patch up anyways
run: mvn -q -B -ntp clean install verify -Powasp-dependency-check -DskipTests -pl '!stream/distributedlog/io/dlfs'
- name: Setup tmate session
if: failure()
uses: mxschmitt/action-tmate@v3

- name: Upload report
uses: actions/upload-artifact@v3
if: ${{ cancelled() || failure() }}
continue-on-error: true
with:
name: dependency report
path: target/dependency-check-report.html
retention-days: 7
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import static org.apache.bookkeeper.proto.BookieProtocol.FLAG_RECOVERY_ADD;

import com.google.common.collect.ImmutableMap;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.netty.buffer.ByteBuf;
import io.netty.util.Recycler;
import io.netty.util.Recycler.Handle;
Expand Down Expand Up @@ -80,6 +81,7 @@ class PendingAddOp implements WriteCallback {
boolean allowFailFast = false;
List<BookieId> ensemble;

@SuppressFBWarnings("IS2_INCONSISTENT_SYNC")
static PendingAddOp create(LedgerHandle lh, ClientContext clientCtx,
List<BookieId> ensemble,
ByteBuf payload, EnumSet<WriteFlag> writeFlags,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ public static void bestEffortRemoveFromPageCache(int fd, long offset, long len)
}
try {
NATIVE_IO.posix_fadvise(fd, offset, len, POSIX_FADV_DONTNEED);
} catch (Exception e) {
} catch (Throwable e) {
log.warn("Failed to perform posix_fadvise: {}", e.getMessage());
fadvisePossible = false;
}
Expand Down
Loading