diff --git a/BUILDING.txt b/BUILDING.txt
index 1e2a1fef1a098..173cc0bcad49e 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -385,6 +385,50 @@ Create a local staging version of the website (in /tmp/hadoop-site)
Note that the site needs to be built in a second pass after other artifacts.
+----------------------------------------------------------------------------------
+Including Cloud Connector Dependencies in Distributions:
+
+Hadoop distributions include the hadoop modules needed to work with data and services
+on cloud infrastructure
+
+However, dependencies are omitted for all cloud connectors except hadoop-azure
+(abfs:// and wasb://) and possibly hadoop-gcp (gs://) and hadoop-tos (tos://).
+For the latter two modules, it depends on shading options.
+
+For hadoop-aws the AWS SDK bundle.jar is omitted, but everything else is included.
+
+Excluding the extra binaries:
+* Keeps release artifact size below the limit of the ASF distribution network.
+* Reduces download and size overhead in docker usage.
+* Reduces the CVE attack surface and audit-related complaints about those same CVEs.
+* Reduces the risk of classpath conflict.
+
+To produce a build with the specific desired dependencies, the build must be executed
+with the relevant profile of ${module}-package alongside the -Pdist profile.
+
+For example, a build with the hadoop-aws and hadoop-azure-datalake dependencies,
+run with
+
+ mvn package -Pdist -DskipTests -Dhadoop-aws-package -Dhadoop-azure-datalake-package
+
+Available package profiles:
+ hadoop-aliyun-package
+ hadoop-aws-package
+ hadoop-azure-datalake-package
+ hadoop-cos-package
+ hadoop-huaweicloud-package
+
+To build a complete distribution then with all cloud dependencies included:
+
+mvn package -Pdist -DskipTests -Dtar -Dmaven.javadoc.skip=true \
+ -Dhadoop-aliyun-package \
+ -Dhadoop-aws-package \
+ -Dhadoop-azure-datalake-package \
+ -Dhadoop-cos-package \
+ -Dhadoop-huaweicloud-package
+
+The resulting tar file will be too large to be distributable through ASF infrastructure.
+
----------------------------------------------------------------------------------
Installing Hadoop
diff --git a/LICENSE-binary b/LICENSE-binary
index 5c09abd9c76a9..385515fd1f112 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -203,7 +203,10 @@
--------------------------------------------------------------------------------
This project bundles some components that are also licensed under the Apache
-License Version 2.0:
+License Version 2.0.
+Note: some of the listed artifacts may not be included in a given build of the binary
+distribution; it depends on the build options. This list intends
+to be inclusive of all which may be included:
hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.* (css and js files)
@@ -211,10 +214,12 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/data
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java
ch.qos.reload4j:reload4j:1.2.22
+com.aliyun:aliyun-java-core:0.2.11-beta
com.aliyun:aliyun-java-sdk-core:4.5.10
com.aliyun:aliyun-java-sdk-kms:2.11.0
com.aliyun:aliyun-java-sdk-ram:3.1.0
com.aliyun:aliyun-java-sdk-sts:3.0.0
+com.aliyun:java-trace-api:0.2.11-beta
com.aliyun.oss:aliyun-sdk-oss:3.13.2
com.cedarsoftware:java-util:1.9.0
com.cedarsoftware:json-io:2.5.1
@@ -241,6 +246,8 @@ com.google.guava:guava:33.4.8-jre
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
com.microsoft.azure:azure-storage:7.0.0
com.nimbusds:nimbus-jose-jwt:10.4
+com.squareup.okhttp3:okhttp:jar:3.14.2
+com.squareup.okio:okio:jar:1.17.2
com.zaxxer:HikariCP:4.0.3
commons-beanutils:commons-beanutils:1.9.4
commons-cli:commons-cli:1.9.0
@@ -289,6 +296,9 @@ io.netty:netty-transport-native-kqueue:4.1.127.Final
io.netty:netty-resolver-dns-native-macos:4.1.127.Final
io.opencensus:opencensus-api:0.12.3
io.opencensus:opencensus-contrib-grpc-metrics:0.12.3
+io.opentracing:opentracing-api:0.33.0.jar
+io.opentracing:opentracing-noop:0.33.0.jar
+io.opentracing:opentracing-util:0.33.0.jar
io.reactivex:rxjava:1.3.8
io.reactivex:rxjava-string:1.1.1
io.reactivex:rxnetty:0.4.20
@@ -316,6 +326,8 @@ org.apache.htrace:htrace-core:3.1.0-incubating
org.apache.htrace:htrace-core4:4.1.0-incubating
org.apache.httpcomponents:httpclient:4.5.13
org.apache.httpcomponents:httpcore:4.4.13
+org.apache.httpcomponents.client5:httpclient5:5.5
+org.apache.httpcomponents.core5:httpcore5:5.5
org.apache.kafka:kafka-clients:3.9.0
org.apache.kerby:kerb-admin:2.0.3
org.apache.kerby:kerb-client:2.0.3
@@ -432,6 +444,7 @@ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage
bootstrap v3.3.6
broccoli-asset-rev v2.4.2
broccoli-funnel v1.0.1
+cos_api-bundle-5.6.19.jar
datatables v1.11.5
em-helpers v0.5.13
em-table v0.1.6
@@ -477,7 +490,7 @@ com.microsoft.azure:azure-cosmosdb:2.4.5
com.microsoft.azure:azure-cosmosdb-commons:2.4.5
com.microsoft.azure:azure-cosmosdb-direct:2.4.5
com.microsoft.azure:azure-cosmosdb-gateway:2.4.5
-com.microsoft.azure:azure-data-lake-store-sdk:2.3.3
+com.microsoft.azure:azure-data-lake-store-sdk:2.3.9
com.microsoft.azure:azure-keyvault-core:1.0.0
com.microsoft.sqlserver:mssql-jdbc:6.2.1.jre7
org.bouncycastle:bcpkix-jdk18on:1.82
@@ -536,3 +549,8 @@ Public Domain
-------------
aopalliance:aopalliance:1.0
+
+Dom4J license
+-------------
+
+org.dom4j:dom4j:2.1.4.jar
\ No newline at end of file
diff --git a/dev-support/bin/dist-layout-stitching b/dev-support/bin/dist-layout-stitching
index d4bfd8aaada3b..e19b2e4f1d621 100755
--- a/dev-support/bin/dist-layout-stitching
+++ b/dev-support/bin/dist-layout-stitching
@@ -130,6 +130,10 @@ run cp -p "${ROOT}/README.txt" .
run copy "${ROOT}/hadoop-common-project/hadoop-common/target/hadoop-common-${VERSION}" .
run copy "${ROOT}/hadoop-common-project/hadoop-nfs/target/hadoop-nfs-${VERSION}" .
run copy "${ROOT}/hadoop-common-project/hadoop-registry/target/hadoop-registry-${VERSION}" .
+
+# cloud connectors go into common
+run copy "${ROOT}/hadoop-cloud-storage-project/hadoop-cloud-storage-dist/target/hadoop-cloud-storage-dist-${VERSION}" .
+
run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${VERSION}" .
run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-nfs/target/hadoop-hdfs-nfs-${VERSION}" .
run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-${VERSION}" .
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-cloud-storage.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-cloud-storage.xml
new file mode 100644
index 0000000000000..03f62dcb4b8ab
--- /dev/null
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-cloud-storage.xml
@@ -0,0 +1,59 @@
+
+
+ hadoop-cloud-storage
+
+ dir
+
+ false
+
+
+
+
+ ../../hadoop-tools/hadoop-aws/src/main/bin
+ /bin
+ 0755
+
+
+ ./../hadoop-tools/hadoop-aws/src/main/shellprofile.d
+
+ *
+
+ /libexec/shellprofile.d
+ 0755
+
+
+
+
+
+ /share/hadoop/common/lib
+ false
+ runtime
+ false
+
+
+ org.apache.hadoop:hadoop-annotations
+ org.apache.hadoop.thirdparty:hadoop-shaded-guava
+
+
+
+
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-dynamometer.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-dynamometer.xml
index b2ce562231c5a..508faa9c85ec2 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-dynamometer.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-dynamometer.xml
@@ -9,7 +9,7 @@
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
- distributed under the Li2cense is distributed on an "AS IS" BASIS,
+ distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml
index 871694209393f..7895f4e57142f 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml
@@ -57,6 +57,7 @@
**/file:/**
**/SecurityAuth.audit*
patchprocess/**
+ **/auth-keys.xml
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
index c01d9c4282089..4276820a7b822 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
@@ -9,7 +9,7 @@
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
- distributed under the Li2cense is distributed on an "AS IS" BASIS,
+ distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
diff --git a/hadoop-cloud-storage-project/hadoop-cloud-storage-dist/pom.xml b/hadoop-cloud-storage-project/hadoop-cloud-storage-dist/pom.xml
new file mode 100644
index 0000000000000..10bfdeb6f0b42
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-cloud-storage-dist/pom.xml
@@ -0,0 +1,282 @@
+
+
+
+ 4.0.0
+
+ org.apache.hadoop
+ hadoop-project
+ 3.4.3-SNAPSHOT
+ ../../hadoop-project
+
+ hadoop-cloud-storage-dist
+ 3.4.3-SNAPSHOT
+ jar
+
+ Apache Hadoop Cloud Storage Distribution
+ Apache Hadoop Cloud Storage Distribution
+
+
+
+ cloud-storage
+
+
+
+
+ org.apache.hadoop
+ hadoop-annotations
+ provided
+
+
+ org.apache.hadoop
+ hadoop-common
+ provided
+
+
+
+
+ org.apache.hadoop
+ hadoop-aws
+ compile
+
+
+ software.amazon.awssdk
+ *
+
+
+
+
+
+ org.apache.hadoop
+ hadoop-azure
+ compile
+
+
+ org.apache.hadoop
+ hadoop-azure-datalake
+ compile
+
+
+ *
+ *
+
+
+
+
+ org.apache.hadoop
+ hadoop-cos
+ compile
+
+
+ *
+ *
+
+
+
+
+ org.apache.hadoop
+ hadoop-huaweicloud
+ compile
+
+
+ *
+ *
+
+
+
+
+
+
+
+
+ maven-deploy-plugin
+
+ true
+
+
+
+ org.apache.rat
+ apache-rat-plugin
+
+
+
+
+
+
+
+
+
+
+ dist
+
+ false
+
+
+
+
+ org.apache.maven.plugins
+ maven-assembly-plugin
+
+
+ org.apache.hadoop
+ hadoop-assemblies
+ ${project.version}
+
+
+
+
+ dist
+ prepare-package
+
+ single
+
+
+ false
+ false
+ ${project.artifactId}-${project.version}
+
+ hadoop-cloud-storage
+
+
+
+
+
+
+
+
+
+
+
+ hadoop-aliyun-package
+
+ hadoop-aliyun-package
+
+
+
+ org.apache.hadoop
+ hadoop-aliyun
+ compile
+
+
+
+
+
+
+ hadoop-aws-package
+
+ hadoop-aws-package
+
+
+
+ org.apache.hadoop
+ hadoop-aws
+ compile
+
+
+
+
+
+
+ hadoop-azure-datalake-package
+
+ hadoop-azure-datalake-package
+
+
+
+ org.apache.hadoop
+ hadoop-azure-datalake
+ compile
+
+
+
+
+
+
+ hadoop-cos-package
+
+ hadoop-cos-package
+
+
+
+ org.apache.hadoop
+ hadoop-cos
+ compile
+
+
+
+
+
+
+ hadoop-huaweicloud-package
+
+ hadoop-huaweicloud-package
+
+
+
+ org.apache.hadoop
+ hadoop-huaweicloud
+ compile
+
+
+ com.fasterxml.jackson
+ *
+
+
+
+
+
+
+
+
diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml b/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml
index 3b7c81404a3f1..f7b648ae45331 100755
--- a/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml
+++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml
@@ -169,6 +169,10 @@
log4j-api
org.apache.logging.log4j
+
+ com.fasterxml.jackson.core
+ *
+
diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
deleted file mode 100644
index e77425ab52989..0000000000000
--- a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-org.apache.hadoop.fs.obs.OBSFileSystem
diff --git a/hadoop-cloud-storage-project/pom.xml b/hadoop-cloud-storage-project/pom.xml
index 6e32bc22188ec..70904617036e1 100644
--- a/hadoop-cloud-storage-project/pom.xml
+++ b/hadoop-cloud-storage-project/pom.xml
@@ -34,7 +34,6 @@
hadoop-cos
hadoop-huaweicloud
-
@@ -51,5 +50,18 @@
-
+
+
+ dist
+
+ false
+
+ tar
+
+
+
+ hadoop-cloud-storage-dist
+
+
+
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 3e1351c3c88e5..db548e1e692f1 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -4438,4 +4438,10 @@ The switch to turn S3A auditing on or off.
The AbstractFileSystem for Ozone FileSystem o3fs uri
+
+ fs.obs.impl
+ org.apache.hadoop.fs.obs.OBSFileSystem
+ The FileSystem for obs: uris.
+
+
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
index 210b36f2dbd1f..c0e8b4690cb40 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
@@ -250,5 +250,7 @@ public void initializeMemberVariables() {
xmlPropsToSkipCompare.add("io.seqfile.local.dir");
xmlPropsToSkipCompare.add("hadoop.http.sni.host.check.enabled");
+
+ xmlPrefixToSkipCompare.add("fs.obs.");
}
}
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 65211d64daebc..1850ca1221913 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -75,6 +75,8 @@
4.5.13
4.4.13
+ 5.5
+ 5.3.6
1.7.36
@@ -840,6 +842,28 @@
httpcore
${httpcore.version}
+
+ org.apache.httpcomponents.client5
+ httpclient5
+ ${httpclient5.version}
+
+
+ org.slf4j
+ *
+
+
+
+
+ org.apache.httpcomponents.core5
+ httpcore5
+ ${httpcore5.version}
+
+
+ org.apache.logging.log4j
+ *
+
+
+
commons-codec
commons-codec
diff --git a/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem b/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
deleted file mode 100644
index 1938bdefa4107..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-org.apache.hadoop.fs.s3native.NativeS3FileSystem
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 2f064d82401dc..1da0df465cac6 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -92,30 +92,12 @@
pom
${project.version}
-
- org.apache.hadoop
- hadoop-aws
- compile
- ${project.version}
-
org.apache.hadoop
hadoop-kafka
compile
${project.version}
-
- org.apache.hadoop
- hadoop-azure
- compile
- ${project.version}
-
-
- org.apache.hadoop
- hadoop-aliyun
- compile
- ${project.version}
-
org.apache.hadoop
hadoop-sls
@@ -127,12 +109,6 @@
${project.version}
compile
-
- org.apache.hadoop
- hadoop-azure-datalake
- compile
- ${project.version}
-
org.apache.hadoop
hadoop-fs2img