Update bundled JDK to JDK-24

Signed-off-by: Andriy Redko <[email protected]>

Author: reta

DEVELOPER_GUIDE.md

@@ -76,7 +76,7 @@ Fork [opensearch-project/OpenSearch](https://github.com/opensearch-project/OpenS
 
 #### JDK
 
-OpenSearch recommends building with the [Temurin/Adoptium](https://adoptium.net/temurin/releases/) distribution. JDK 11 is the minimum supported, and JDK-23 is the newest supported. You must have a supported JDK installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-21`. 
+OpenSearch recommends building with the [Temurin/Adoptium](https://adoptium.net/temurin/releases/) distribution. JDK 11 is the minimum supported, and JDK-24 is the newest supported. You must have a supported JDK installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-21`. 
 
 Download Java 11 from [here](https://adoptium.net/releases.html?variant=openjdk11). 
 
@@ -87,11 +87,11 @@ In addition, certain backward compatibility tests check out and compile the prev
 ./gradlew check -Dorg.gradle.warning.mode=none
 ```
 
-By default, the test tasks use bundled JDK runtime, configured in version catalog [gradle/libs.versions.toml](gradle/libs.versions.toml), and set to JDK 23 (non-LTS).
+By default, the test tasks use bundled JDK runtime, configured in version catalog [gradle/libs.versions.toml](gradle/libs.versions.toml), and set to JDK 24 (non-LTS).
 
 ```
 bundled_jdk_vendor = adoptium
-bundled_jdk = 23.0.1+11
+bundled_jdk = 24.0.1+9
 ```
 
 #### Custom Runtime JDK

distribution/tools/plugin-cli/build.gradle

@@ -37,6 +37,7 @@ base {
 dependencies {
   compileOnly project(":server")
   compileOnly project(":libs:opensearch-cli")
+  api project(":libs:agent-sm:agent-policy")
   api "org.bouncycastle:bc-fips:${versions.bouncycastle_jce}"
   api "org.bouncycastle:bcpg-fips:${versions.bouncycastle_pg}"
   testImplementation project(":test:framework")

distribution/tools/plugin-cli/src/main/java/org/opensearch/tools/cli/plugin/PluginSecurity.java

@@ -37,16 +37,15 @@
 import org.opensearch.cli.Terminal.Verbosity;
 import org.opensearch.cli.UserException;
 import org.opensearch.common.util.io.IOUtils;
+import org.opensearch.secure_sm.policy.PolicyFile;
 
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.security.NoSuchAlgorithmException;
 import java.security.Permission;
 import java.security.PermissionCollection;
 import java.security.Permissions;
 import java.security.Policy;
-import java.security.URIParameter;
 import java.security.UnresolvedPermission;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -143,22 +142,12 @@ static Set<String> parsePermissions(Path file, Path tmpDir) throws IOException {
         // 2. read permission to the code itself (e.g. jar file of the code)
 
         Path emptyPolicyFile = Files.createTempFile(tmpDir, "empty", "tmp");
-        final Policy emptyPolicy;
-        try {
-            emptyPolicy = Policy.getInstance("JavaPolicy", new URIParameter(emptyPolicyFile.toUri()));
-        } catch (NoSuchAlgorithmException e) {
-            throw new RuntimeException(e);
-        }
+        final Policy emptyPolicy = new PolicyFile(emptyPolicyFile.toUri().toURL());
         IOUtils.rm(emptyPolicyFile);
 
         // parse the plugin's policy file into a set of permissions
-        final Policy policy;
-        try {
-            policy = Policy.getInstance("JavaPolicy", new URIParameter(file.toUri()));
-        } catch (NoSuchAlgorithmException e) {
-            throw new RuntimeException(e);
-        }
-        PermissionCollection permissions = policy.getPermissions(PluginSecurity.class.getProtectionDomain());
+        final Policy policy = new PolicyFile(file.toUri().toURL());
+        final PermissionCollection permissions = policy.getPermissions(PluginSecurity.class.getProtectionDomain());
         // this method is supported with the specific implementation we use, but just check for safety.
         if (permissions == Policy.UNSUPPORTED_EMPTY_COLLECTION) {
             throw new UnsupportedOperationException("JavaPolicy implementation does not support retrieving permissions");

gradle/code-coverage.gradle

@@ -19,7 +19,7 @@ repositories {
 
 allprojects {
   plugins.withId('jacoco') {
-    jacoco.toolVersion = '0.8.12'
+    jacoco.toolVersion = '0.8.13'
   }
 }
 

gradle/libs.versions.toml

@@ -3,7 +3,7 @@ opensearch        = "3.1.0"
 lucene            = "10.1.0"
 
 bundled_jdk_vendor = "adoptium"
-bundled_jdk = "21.0.7+6"
+bundled_jdk = "24.0.1+9"
 
 # optional dependencies
 spatial4j         = "0.7"
@@ -56,6 +56,7 @@ commonsio         = "2.16.0"
 aws               = "2.30.31"
 awscrt            = "0.35.0"
 reactivestreams   = "1.0.4"
+hadoop3           = "3.4.1"
 
 # when updating this version, you need to ensure compatibility with:
 #  - plugins/ingest-attachment (transitive dependency, check the upstream POM)

libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/PolicyFile.java

@@ -125,8 +125,32 @@ private void addGrantEntry(GrantEntry grantEntry, List<PolicyEntry> entries) thr
         entries.add(new PolicyEntry(codesource, permissions));
     }
 
+    /**
+     * Expands known system properties like ${java.home} and ${user.home} to their absolute
+     * path equivalents.
+     */
+    private static String expandKnownSystemProperty(final String property, final String value) {
+        final int index = value.indexOf("${" + property + "}/");
+        final String path = System.getProperty(property);
+        if (path.endsWith(File.pathSeparator)) {
+            return path + value.substring(index + property.length() + 4 /* replace the path separator */);
+        } else {
+            return path + value.substring(index + property.length() + 3 /* keep the path separator */);
+        }
+    }
+
     private static PermissionEntry expandPermissionName(PermissionEntry pe) {
-        if (pe.name() == null || !pe.name().contains("${{")) {
+        if (pe.name() == null) {
+            return pe;
+        }
+
+        if (pe.name().startsWith("${java.home}")) {
+            return new PermissionEntry(pe.permission(), expandKnownSystemProperty("java.home", pe.name()), pe.action());
+        } else if (pe.name().startsWith("${user.home}")) {
+            return new PermissionEntry(pe.permission(), expandKnownSystemProperty("user.home", pe.name()), pe.action());
+        }
+
+        if (!pe.name().contains("${{")) {
             return pe;
         }
 

libs/agent-sm/agent/src/test/java/org/opensearch/javaagent/FileInterceptorIntegTests.java

@@ -22,6 +22,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
+import java.security.Permission;
 import java.security.PermissionCollection;
 import java.security.Permissions;
 import java.security.Policy;
@@ -56,6 +57,17 @@ public PermissionCollection getPermissions(ProtectionDomain domain) {
                 permissions.add(new FilePermission(System.getProperty("user.dir") + "/-", "read,write,delete"));
                 return permissions;
             }
+
+            @Override
+            public boolean implies(ProtectionDomain domain, Permission permission) {
+                final PermissionCollection pc = getPermissions(domain);
+
+                if (pc == null) {
+                    return false;
+                }
+
+                return pc.implies(permission);
+            }
         };
         AgentPolicy.setPolicy(policy);
         Files.createDirectories(getTestDir());

plugins/repository-hdfs/build.gradle

@@ -47,10 +47,6 @@ opensearchplugin {
   classname = 'org.opensearch.repositories.hdfs.HdfsPlugin'
 }
 
-versions << [
-  'hadoop3': '3.3.6'
-]
-
 testFixtures.useFixture ":test:fixtures:krb5kdc-fixture", "hdfs"
 
 configurations {
@@ -138,7 +134,8 @@ for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture',
     executable = "${BuildParams.runtimeJavaHome}/bin/java"
     env 'CLASSPATH', "${-> configurations.hdfsFixture.asPath}"
     maxWaitInSeconds = 60
-    onlyIf { BuildParams.inFipsJvm == false }
+    // See please https://issues.apache.org/jira/browse/HADOOP-19486
+    onlyIf { BuildParams.inFipsJvm == false && BuildParams.runtimeJavaVersion < JavaVersion.VERSION_24 }
     waitCondition = { fixture, ant ->
       // the hdfs.MiniHDFS fixture writes the ports file when
       // it's ready, so we can just wait for the file to exist
@@ -197,7 +194,8 @@ for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSec
       }
     }
 
-      onlyIf { BuildParams.inFipsJvm == false }
+      // See please https://issues.apache.org/jira/browse/HADOOP-19486
+      onlyIf { BuildParams.inFipsJvm == false && BuildParams.runtimeJavaVersion < JavaVersion.VERSION_24 }
       if (integTestTaskName.contains("Ha")) {
         Path portsFile
         File portsFileDir = file("${workingDir}/hdfsFixture")
@@ -271,7 +269,8 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
     }
   }
 } else {
-  fixtureSupported = true
+  // See please https://issues.apache.org/jira/browse/HADOOP-19486
+  fixtureSupported = BuildParams.runtimeJavaVersion < JavaVersion.VERSION_24
 }
 
 boolean legalPath = rootProject.rootDir.toString().contains(" ") == false
@@ -282,7 +281,8 @@ if (legalPath == false) {
 // Always ignore HA integration tests in the normal integration test runner, they are included below as
 // part of their own HA-specific integration test tasks.
 integTest {
-  onlyIf { BuildParams.inFipsJvm == false }
+  // See please https://issues.apache.org/jira/browse/HADOOP-19486
+  onlyIf { BuildParams.inFipsJvm == false && BuildParams.runtimeJavaVersion < JavaVersion.VERSION_24 }
   exclude('**/Ha*TestSuiteIT.class')
 }
 
@@ -371,7 +371,6 @@ thirdPartyAudit {
     'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
     'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
     'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
-    'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',
     'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64',
     'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64$1',
     'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64$Cell',
@@ -383,6 +382,9 @@ thirdPartyAudit {
     'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$1',
     'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$JvmMemoryAccessor',
     'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$MemoryAccessor',
+    'org.apache.hadoop.thirdparty.protobuf.MessageSchema',
+    'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android32MemoryAccessor',
+    'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android64MemoryAccessor',
 
     'org.apache.hadoop.shaded.com.google.common.cache.Striped64',
     'org.apache.hadoop.shaded.com.google.common.cache.Striped64$1',

plugins/repository-hdfs/licenses/hadoop-client-api-3.3.6.jar.sha1

@@ -0,0 +1 @@
+7e4cfae7f5c85cffdbc21fdf749262fc78d4463a

plugins/repository-hdfs/licenses/hadoop-client-api-3.4.1.jar.sha1

@@ -0,0 +1 @@
+30ec07ceccb224b2ef17af34fbb593bff3e9e071

plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.6.jar.sha1

@@ -0,0 +1 @@
+8545078b39e33416cb70ccef1bb22f2c88fb6b6c

plugins/repository-hdfs/licenses/hadoop-client-runtime-3.4.1.jar.sha1

@@ -55,6 +55,8 @@
 import org.opensearch.indices.recovery.RecoverySettings;
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
 
+import javax.security.auth.Subject;
+
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.net.InetAddress;
@@ -212,6 +214,10 @@ private UserGroupInformation login(Configuration hadoopConfiguration, Settings r
                 logger.debug("Using kerberos principal [{}] and keytab located at [{}]", principal, keytab);
                 return UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab);
             }
+            final Subject subject = Subject.current();
+            if (subject == null || subject.getPrincipals() == null) {
+                return UserGroupInformation.getLoginUser();
+            }
             return UserGroupInformation.getCurrentUser();
         } catch (IOException e) {
             throw new UncheckedIOException("Could not retrieve the current user information", e);

plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.6.jar.sha1

@@ -119,6 +119,7 @@ private FileContext createContext(URI uri) {
         });
     }
 
+    @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/HADOOP-19486")
     public void testReadOnly() throws Exception {
         FileContext fileContext = createTestContext();
         // Constructor will not create dir if read only

plugins/repository-hdfs/licenses/hadoop-hdfs-3.4.1.jar.sha1

@@ -33,11 +33,13 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 
+import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.repositories.blobstore.OpenSearchBlobStoreRepositoryIntegTestCase;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
+import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 
@@ -66,4 +68,58 @@ protected Settings repositorySettings() {
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Collections.singletonList(HdfsPlugin.class);
     }
+
+    @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/HADOOP-19486")
+    @Override
+    public void testContainerCreationAndDeletion() throws IOException {
+        super.testContainerCreationAndDeletion();
+    }
+
+    @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/HADOOP-19486")
+    @Override
+    public void testDeleteBlobs() throws IOException {
+        super.testDeleteBlobs();
+    }
+
+    @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/HADOOP-19486")
+    @Override
+    public void testIndicesDeletedFromRepository() throws Exception {
+        super.testIndicesDeletedFromRepository();
+    }
+
+    @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/HADOOP-19486")
+    @Override
+    public void testList() throws IOException {
+        super.testList();
+    }
+
+    @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/HADOOP-19486")
+    @Override
+    public void testMultipleSnapshotAndRollback() throws Exception {
+        super.testMultipleSnapshotAndRollback();
+    }
+
+    @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/HADOOP-19486")
+    @Override
+    public void testReadNonExistingPath() throws IOException {
+        super.testReadNonExistingPath();
+    }
+
+    @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/HADOOP-19486")
+    @Override
+    public void testReadRange() throws IOException {
+        super.testReadRange();
+    }
+
+    @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/HADOOP-19486")
+    @Override
+    public void testSnapshotAndRestore() throws Exception {
+        super.testSnapshotAndRestore();
+    }
+
+    @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/HADOOP-19486")
+    @Override
+    public void testWriteRead() throws IOException {
+        super.testWriteRead();
+    }
 }

plugins/repository-hdfs/src/main/java/org/opensearch/repositories/hdfs/HdfsRepository.java

@@ -33,6 +33,7 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 
+import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 import org.opensearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse;
 import org.opensearch.common.settings.MockSecureSettings;
 import org.opensearch.common.settings.SecureSettings;
@@ -79,4 +80,22 @@ protected void assertCleanupResponse(CleanupRepositoryResponse response, long by
             assertThat(response.result().blobs(), equalTo(0L));
         }
     }
+
+    @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/HADOOP-19486")
+    @Override
+    public void testCleanup() throws Exception {
+        super.testCleanup();
+    }
+
+    @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/HADOOP-19486")
+    @Override
+    public void testCreateSnapshot() {
+        super.testCreateSnapshot();
+    }
+
+    @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/HADOOP-19486")
+    @Override
+    public void testListChildren() throws Exception {
+        super.testListChildren();
+    }
 }

plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsBlobStoreContainerTests.java

@@ -33,6 +33,7 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 
+import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.opensearch.cluster.ClusterState;
@@ -61,6 +62,7 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(HdfsPlugin.class);
     }
 
+    @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/HADOOP-19486")
     public void testSimpleWorkflow() {
         Client client = client();
         Settings.Builder settings = Settings.builder()