diff --git a/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json b/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json index 1bd74515152c..f1ba03a243ee 100644 --- a/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json +++ b/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run", - "modification": 4 -} \ No newline at end of file + "modification": 5 +} diff --git a/runners/google-cloud-dataflow-java/worker/build.gradle b/runners/google-cloud-dataflow-java/worker/build.gradle index 21879861e9d6..44a2d40f944c 100644 --- a/runners/google-cloud-dataflow-java/worker/build.gradle +++ b/runners/google-cloud-dataflow-java/worker/build.gradle @@ -154,6 +154,22 @@ applyJavaNature( /******************************************************************************/ // Configure the worker root project +tasks.withType(Test).configureEach { + // WindmillStateTestUtils.assertNoReference walks every object reachable from the Windmill + // state cache (Guava Cache / ConcurrentHashMap) to ensure no per-work-item WindmillStateReader + // leaks into the global cache. It uses reflection (Field.setAccessible) into JDK internals + // (e.g. Integer.value, AtomicReferenceArray.array, ReentrantLock.sync, ReferenceQueue.head), + // which requires --add-opens on Java 17+. + jvmArgs '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.lang.ref=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/java.io=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.util.concurrent=ALL-UNNAMED', + '--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED', + '--add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED' +} + configurations { sourceFile diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowOperationContextTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowOperationContextTest.java index 34c3b3d5373c..6692f06d75d4 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowOperationContextTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowOperationContextTest.java @@ -305,11 +305,13 @@ private void verifyLullLog(boolean hasFullThreadDump) throws IOException { String infoLines = Joiner.on("\n").join(Iterables.filter(lines, line -> line.contains("\"INFO\""))); + // Match on the thread name rather than the full Thread.toString() prefix: JDK 21+ inserts + // the thread id (Thread[#51,backgroundThread,...] vs Thread[backgroundThread,...]). if (hasFullThreadDump) { assertThat( infoLines, Matchers.allOf( - Matchers.containsString("Thread[backgroundThread,"), + Matchers.containsString("backgroundThread,"), Matchers.containsString( "org.apache.beam.runners.dataflow.worker.DataflowOperationContext"), Matchers.not(Matchers.containsString(SimpleDoFnRunner.class.getName())))); @@ -318,7 +320,7 @@ private void verifyLullLog(boolean hasFullThreadDump) throws IOException { infoLines, Matchers.not( Matchers.anyOf( - Matchers.containsString("Thread[backgroundThread,"), + Matchers.containsString("backgroundThread,"), Matchers.containsString( "org.apache.beam.runners.dataflow.worker.DataflowOperationContext")))); } diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/status/ThreadzServletTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/status/ThreadzServletTest.java index 7737f7d405b5..1c2352954bd0 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/status/ThreadzServletTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/status/ThreadzServletTest.java @@ -36,13 +36,18 @@ public class ThreadzServletTest { @Test public void testDeduping() throws Exception { + // Use Thread.toString() rather than hard-coded strings: JDK 21+ includes the thread id + // (e.g. Thread[#42,Thread1,5,main] vs Thread[Thread1,5,main]). + Thread thread1 = new Thread("Thread1"); + Thread thread2 = new Thread("Thread2"); + Thread thread3 = new Thread("Thread3"); Map stacks = ImmutableMap.of( - new Thread("Thread1"), + thread1, new StackTraceElement[] {new StackTraceElement("Class", "Method1", "File", 11)}, - new Thread("Thread2"), + thread2, new StackTraceElement[] {new StackTraceElement("Class", "Method1", "File", 11)}, - new Thread("Thread3"), + thread3, new StackTraceElement[] {new StackTraceElement("Class", "Method2", "File", 17)}); Map> deduped = ThreadzServlet.deduplicateThreadStacks(stacks); @@ -54,13 +59,13 @@ public void testDeduping() throws Exception { new Stack( new StackTraceElement[] {new StackTraceElement("Class", "Method1", "File", 11)}, Thread.State.NEW), - Arrays.asList("Thread[Thread1,5,main]", "Thread[Thread2,5,main]"))); + Arrays.asList(thread1.toString(), thread2.toString()))); assertThat( deduped, Matchers.hasEntry( new Stack( new StackTraceElement[] {new StackTraceElement("Class", "Method2", "File", 17)}, Thread.State.NEW), - Arrays.asList("Thread[Thread3,5,main]"))); + Arrays.asList(thread3.toString()))); } } diff --git a/sdks/java/io/hcatalog/build.gradle b/sdks/java/io/hcatalog/build.gradle index d3bdd8f10765..b44487ab6ec4 100644 --- a/sdks/java/io/hcatalog/build.gradle +++ b/sdks/java/io/hcatalog/build.gradle @@ -40,6 +40,10 @@ hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")} def hive_version = "4.0.1" +tasks.withType(Test).configureEach { + jvmArgs '--add-opens=java.base/java.net=ALL-UNNAMED' +} + dependencies { implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow")