Skip to content

Rework Hadoop #904

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 8 commits into
base: main
Choose a base branch
from
Draft
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 33 additions & 22 deletions hadoop/Dockerfile
Original file line number Diff line number Diff line change
@@ -19,27 +19,34 @@ WORKDIR /stackable/jmx
# At the same time a new HDFS Operator will still work with older images which do not have the symlink to the versionless jar.
# After one of our next releases (23.11 or 24.x) we should update the operator to point at the non-versioned symlink (jmx_prometheus_javaagent.jar)
# And then we can also remove the symlink to 0.16.1 from this Dockerfile.
RUN curl "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -o "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" && \
chmod -x "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" && \
ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar && \
ln -s /stackable/jmx/jmx_prometheus_javaagent.jar /stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar
RUN <<EOF
curl "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -o "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar"
chmod -x "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar"
ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar
EOF

WORKDIR /stackable

RUN ARCH="${TARGETARCH/amd64/x64}" && \
curl "https://repo.stackable.tech/repository/packages/async-profiler/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}.tar.gz" | tar -xzC . && \
ln -s "/stackable/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}" /stackable/async-profiler
RUN <<EOF
export ARCH="${TARGETARCH/amd64/x64}"
curl "https://repo.stackable.tech/repository/packages/async-profiler/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}.tar.gz" | tar -xzC .
ln -s "/stackable/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}" /stackable/async-profiler
EOF

# This Protobuf version is the exact version as used in the Hadoop Dockerfile
# See https://github.com/apache/hadoop/blob/trunk/dev-support/docker/pkg-resolver/install-protobuf.sh
# (this was hardcoded in the Dockerfile in earlier versions of Hadoop, make sure to look at the exact version in Github)
WORKDIR /opt/protobuf-src
RUN curl https://repo.stackable.tech/repository/packages/protobuf/protobuf-java-${PROTOBUF}.tar.gz -o /opt/protobuf.tar.gz && \
tar xzf /opt/protobuf.tar.gz --strip-components 1 --no-same-owner && \
./configure --prefix=/opt/protobuf && \
make "-j$(nproc)" && \
make install && \
rm -rf /opt/protobuf-src
RUN <<EOF
mkdir /opt/protobuf-src
cd /opt/protobuf-src
curl https://repo.stackable.tech/repository/packages/protobuf/protobuf-java-${PROTOBUF}.tar.gz -o /opt/protobuf.tar.gz
tar xzf /opt/protobuf.tar.gz --strip-components 1 --no-same-owner
./configure --prefix=/opt/protobuf
make "-j$(nproc)"
make install
rm -rf /opt/protobuf-src
EOF

ENV PROTOBUF_HOME=/opt/protobuf
ENV PATH="${PATH}:/opt/protobuf/bin"
@@ -61,15 +68,19 @@ COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/patches /stackable/patches
# Also skip building the yarn, mapreduce and minicluster modules: this will result in the modules being excluded but not all
# jar files will be stripped if they are needed elsewhere e.g. share/hadoop/yarn will not be part of the build, but yarn jars
# will still exist in share/hadoop/tools as they would be needed by the resource estimator tool. Such jars are removed in a later step.
RUN curl "https://repo.stackable.tech/repository/packages/hadoop/hadoop-${PRODUCT}-src.tar.gz" | tar -xzC . && \
patches/apply_patches.sh ${PRODUCT} && \
cd hadoop-${PRODUCT}-src && \
mvn --batch-mode --no-transfer-progress clean package -Pdist,native -pl '!hadoop-tools/hadoop-pipes,!hadoop-yarn-project,!hadoop-mapreduce-project,!hadoop-minicluster' -Drequire.fuse=true -DskipTests -Dmaven.javadoc.skip=true && \
cp -r hadoop-dist/target/hadoop-${PRODUCT} /stackable/hadoop-${PRODUCT} && \
mv hadoop-dist/target/bom.json /stackable/hadoop-${PRODUCT}/hadoop-${PRODUCT}.cdx.json && \
# HDFS fuse-dfs is not part of the regular dist output, so we need to copy it in ourselves
cp hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs/fuse_dfs /stackable/hadoop-${PRODUCT}/bin && \
rm -rf /stackable/hadoop-${PRODUCT}-src
RUN <<EOF
curl "https://repo.stackable.tech/repository/packages/hadoop/hadoop-${PRODUCT}-src.tar.gz" | tar -xzC .
patches/apply_patches.sh ${PRODUCT}
cd hadoop-${PRODUCT}-src
mvn --batch-mode --no-transfer-progress clean package -Pdist,native -pl '!hadoop-tools/hadoop-pipes,!hadoop-yarn-project,!hadoop-mapreduce-project,!hadoop-minicluster' -Drequire.fuse=true -DskipTests -Dmaven.javadoc.skip=true

cp -r hadoop-dist/target/hadoop-${PRODUCT} /stackable/hadoop-${PRODUCT}
mv hadoop-dist/target/bom.json /stackable/hadoop-${PRODUCT}/hadoop-${PRODUCT}.cdx.json

# HDFS fuse-dfs is not part of the regular dist output, so we need to copy it in ourselves
cp hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs/fuse_dfs /stackable/hadoop-${PRODUCT}/bin
rm -rf /stackable/hadoop-${PRODUCT}-src
EOF

# For earlier versions this script removes the .class file that contains the
# vulnerable code.
2,194 changes: 2,194 additions & 0 deletions hadoop/stackable/patches/3.4.0/01-remove-hadoop-archive-logs.patch

Large diffs are not rendered by default.

2,315 changes: 2,315 additions & 0 deletions hadoop/stackable/patches/3.4.0/02-remove-hadoop-archives.patch

Large diffs are not rendered by default.

470 changes: 470 additions & 0 deletions hadoop/stackable/patches/3.4.0/03-remove-hadoop-benchmark.patch

Large diffs are not rendered by default.

1,631 changes: 1,631 additions & 0 deletions hadoop/stackable/patches/3.4.0/04-remove-hadoop-datajoin.patch

Large diffs are not rendered by default.

16,095 changes: 16,095 additions & 0 deletions hadoop/stackable/patches/3.4.0/05-remove-hadoop-dynamometer.patch

Large diffs are not rendered by default.

1,420 changes: 1,420 additions & 0 deletions hadoop/stackable/patches/3.4.0/06-remove-hadoop-extras.patch

Large diffs are not rendered by default.

20,311 changes: 20,311 additions & 0 deletions hadoop/stackable/patches/3.4.0/07-remove-hadoop-gridmix.patch

Large diffs are not rendered by default.

572 changes: 572 additions & 0 deletions hadoop/stackable/patches/3.4.0/08-remove-hadoop-kafka.patch

Large diffs are not rendered by default.

103 changes: 103 additions & 0 deletions hadoop/stackable/patches/3.4.0/09-remove-hadoop-openstack.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
Entirely remove hadoop-openstack

From: Lars Francke <git@lars-francke.de>


---
hadoop-project/pom.xml | 7 ----
hadoop-tools/hadoop-openstack/pom.xml | 55 ---------------------------------
hadoop-tools/pom.xml | 1 -
3 files changed, 63 deletions(-)
delete mode 100644 hadoop-tools/hadoop-openstack/pom.xml

diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 6015bb3ef10..1776a9499bd 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -682,13 +682,6 @@
<version>${hadoop.version}</version>
</dependency>

- <!-- This is empty; retained only for downstream app build compatibility. -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-openstack</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-azure</artifactId>
diff --git a/hadoop-tools/hadoop-openstack/pom.xml b/hadoop-tools/hadoop-openstack/pom.xml
deleted file mode 100644
index 932ba7bdef0..00000000000
--- a/hadoop-tools/hadoop-openstack/pom.xml
+++ /dev/null
@@ -1,55 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-project</artifactId>
- <version>3.4.0</version>
- <relativePath>../../hadoop-project</relativePath>
- </parent>
- <artifactId>hadoop-openstack</artifactId>
- <version>3.4.0</version>
- <name>Apache Hadoop OpenStack support</name>
- <description>
- This module used to contain code to support integration with OpenStack.
- It has been deleted as unsupported; the JAR is still published so as to
- not break applications which declare an explicit maven/ivy/SBT dependency
- on the module.
- </description>
- <packaging>jar</packaging>
-
- <properties>
- <file.encoding>UTF-8</file.encoding>
- <downloadSources>true</downloadSources>
- </properties>
-
- <build>
- <plugins>
- <plugin>
- <groupId>com.github.spotbugs</groupId>
- <artifactId>spotbugs-maven-plugin</artifactId>
- <configuration>
- <xmlOutput>true</xmlOutput>
- <effort>Max</effort>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
-
-</project>
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index cdc305ecfd6..400a80cb803 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -36,7 +36,6 @@
<module>hadoop-rumen</module>
<module>hadoop-tools-dist</module>
<module>hadoop-pipes</module>
- <module>hadoop-openstack</module>
<module>hadoop-sls</module>
<module>hadoop-resourceestimator</module>
<module>hadoop-azure</module>
3,191 changes: 3,191 additions & 0 deletions hadoop/stackable/patches/3.4.0/10-remove-hadoop-pipes.patch

Large diffs are not rendered by default.

10,758 changes: 10,758 additions & 0 deletions hadoop/stackable/patches/3.4.0/11-remove-hadoop-resourceestimator.patch

Large diffs are not rendered by default.

59,834 changes: 59,834 additions & 0 deletions hadoop/stackable/patches/3.4.0/12-remove-hadoop-sls.patch

Large diffs are not rendered by default.

17,825 changes: 17,825 additions & 0 deletions hadoop/stackable/patches/3.4.0/13-remove-hadoop-rumen.patch

Large diffs are not rendered by default.

16,871 changes: 16,871 additions & 0 deletions hadoop/stackable/patches/3.4.0/14-remove-hadoop-streaming.patch

Large diffs are not rendered by default.

4,142 changes: 4,142 additions & 0 deletions hadoop/stackable/patches/3.4.0/15-hadoop-yarn-csi.patch

Large diffs are not rendered by default.

47,411 changes: 47,411 additions & 0 deletions hadoop/stackable/patches/3.4.0/16-remove-hadoop-yarn-ui.patch

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
Entirely remove hadoop-yarn-registry

From: Lars Francke <git@lars-francke.de>


---
.../hadoop-yarn/hadoop-yarn-registry/pom.xml | 39 --------------------
hadoop-yarn-project/hadoop-yarn/pom.xml | 1 -
2 files changed, 40 deletions(-)
delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
deleted file mode 100644
index f9fffb28ac5..00000000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
- https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>hadoop-yarn</artifactId>
- <groupId>org.apache.hadoop</groupId>
- <version>3.4.0</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
- <artifactId>hadoop-yarn-registry</artifactId>
- <version>3.4.0</version>
- <name>Apache Hadoop YARN Registry</name>
-
- <dependencies>
-
- <!-- The registry moved to Hadoop commons, this is just a stub pom. -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-registry</artifactId>
- </dependency>
-
- </dependencies>
-
-</project>
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml
index 87128835af5..38b7a872619 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -242,7 +242,6 @@
<module>hadoop-yarn-applications</module>
<module>hadoop-yarn-site</module>
<module>hadoop-yarn-client</module>
- <module>hadoop-yarn-registry</module>
</modules>
<!-- -->
</project>
10,509 changes: 10,509 additions & 0 deletions hadoop/stackable/patches/3.4.0/18-remove-hadoop-yarn-applications-catalog.patch

Large diffs are not rendered by default.

Large diffs are not rendered by default.

2,674 changes: 2,674 additions & 0 deletions hadoop/stackable/patches/3.4.0/20-remove-hadoop-yarn-applications-mawo.patch

Large diffs are not rendered by default.

Large diffs are not rendered by default.

39,503 changes: 39,503 additions & 0 deletions hadoop/stackable/patches/3.4.0/22-remove-hadoop-yarn-services.patch

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
Entirely remove hadoop-yarn-applications

From: Lars Francke <git@lars-francke.de>


---
.../hadoop-yarn/hadoop-yarn-applications/pom.xml | 53 --------------------
hadoop-yarn-project/hadoop-yarn/pom.xml | 1
2 files changed, 54 deletions(-)
delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
deleted file mode 100644
index 65ea6f55f1d..00000000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
+++ /dev/null
@@ -1,53 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
- https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>hadoop-yarn</artifactId>
- <groupId>org.apache.hadoop</groupId>
- <version>3.4.0</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
- <artifactId>hadoop-yarn-applications</artifactId>
- <version>3.4.0</version>
- <name>Apache Hadoop YARN Applications</name>
- <packaging>pom</packaging>
-
- <properties>
- <hadoop.common.build.dir>${basedir}/../../../../hadoop-common-project/hadoop-common/target</hadoop.common.build.dir>
- </properties>
-
- <!-- Do not add dependencies here, add them to the POM of the leaf module -->
-
- <profiles>
- <profile>
- <id>clover</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- <property>
- <name>clover</name>
- </property>
- </activation>
- <dependencies>
- <dependency>
- <groupId>com.cenqua.clover</groupId>
- <artifactId>clover</artifactId>
- </dependency>
- </dependencies>
- </profile>
-</profiles>
-</project>
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml
index 38b7a872619..200ff2cb770 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -239,7 +239,6 @@
<module>hadoop-yarn-api</module>
<module>hadoop-yarn-common</module>
<module>hadoop-yarn-server</module>
- <module>hadoop-yarn-applications</module>
<module>hadoop-yarn-site</module>
<module>hadoop-yarn-client</module>
</modules>
44,560 changes: 44,560 additions & 0 deletions hadoop/stackable/patches/3.4.0/24-remove-hadoop-yarn-server-router.patch

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

1,171 changes: 1,171 additions & 0 deletions hadoop/stackable/patches/3.4.0/29-remove-hadoop-client-minicluster.patch

Large diffs are not rendered by default.

Large diffs are not rendered by default.

31 changes: 31 additions & 0 deletions hadoop/stackable/patches/3.4.0/series
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
# This series applies on Git commit bd8b77f398f626bb7791783192ee7a5dfaeec760
01-remove-hadoop-archive-logs.patch
02-remove-hadoop-archives.patch
03-remove-hadoop-benchmark.patch
04-remove-hadoop-datajoin.patch
05-remove-hadoop-dynamometer.patch
06-remove-hadoop-extras.patch
07-remove-hadoop-gridmix.patch
08-remove-hadoop-kafka.patch
09-remove-hadoop-openstack.patch
10-remove-hadoop-pipes.patch
11-remove-hadoop-resourceestimator.patch
12-remove-hadoop-sls.patch
13-remove-hadoop-rumen.patch
14-remove-hadoop-streaming.patch
15-hadoop-yarn-csi.patch
16-remove-hadoop-yarn-ui.patch
17-remove-hadoop-yarn-registry.patch
18-remove-hadoop-yarn-applications-catalog.patch
19-remove-hadoop-yarn-applications-distributedshell.patch
20-remove-hadoop-yarn-applications-mawo.patch
21-remove-hadoop-yarn-applications-unmanaged-am-launcher.patch
22-remove-hadoop-yarn-services.patch
23-remove-hadoop-yarn-applications.patch
24-remove-hadoop-yarn-server-router.patch
25-remove-hadoop-yarn-server-globalpolicygenerator.patch
26-remove-hadoop-yarn-server-timeline-pluginstorage.patch
27-remove-hadoop-client-check-test-invariants.patch
28-remove-hadoop-client-integration-tests.patch
29-remove-hadoop-client-minicluster.patch
30-remove-hadoop-yarn-server-sharedcachemanager.patch
35 changes: 35 additions & 0 deletions hadoop/stackable/patches/3.4.1/001-patch-cyclonedx-plugin.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
diff --git a/pom.xml b/pom.xml
index 16a3733..c309dc6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -118,7 +118,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
<spotbugs-maven-plugin.version>4.2.0</spotbugs-maven-plugin.version>
<jsonschema2pojo-maven-plugin.version>1.1.1</jsonschema2pojo-maven-plugin.version>
<maven-compiler-plugin.version>3.10.1</maven-compiler-plugin.version>
- <cyclonedx.version>2.7.10</cyclonedx.version>
+ <cyclonedx.version>2.8.0</cyclonedx.version>

<shell-executable>bash</shell-executable>

@@ -770,6 +770,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
<groupId>org.cyclonedx</groupId>
<artifactId>cyclonedx-maven-plugin</artifactId>
<version>${cyclonedx.version}</version>
+ <configuration>
+ <projectType>application</projectType>
+ <schemaVersion>1.5</schemaVersion>
+ <skipNotDeployed>false</skipNotDeployed>
+ </configuration>
<executions>
<execution>
<phase>package</phase>
@@ -778,9 +783,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
</goals>
</execution>
</executions>
- <configuration>
- <outputFormat>xml</outputFormat>
- </configuration>
</plugin>
</plugins>
</build>
213 changes: 213 additions & 0 deletions hadoop/stackable/patches/3.4.1/002-update-dependencies.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java
index b2d2a8d100..97e281ba85 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java
@@ -33,7 +33,7 @@ public class TestAvroSerialization {
@Test
public void testSpecific() throws Exception {
AvroRecord before = new AvroRecord();
- before.intField = 5;
+ before.setIntField(5);
AvroRecord after = SerializationTestUtil.testSerialization(conf, before);
assertEquals(before, after);
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
index 66f3781239..aa70bbff62 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
@@ -31,8 +31,8 @@ public class JobQueueChangeEvent implements HistoryEvent {
private JobQueueChange datum = new JobQueueChange();

public JobQueueChangeEvent(JobID id, String queueName) {
- datum.jobid = new Utf8(id.toString());
- datum.jobQueueName = new Utf8(queueName);
+ datum.setJobid(new Utf8(id.toString()));
+ datum.setJobQueueName(new Utf8(queueName));
}

JobQueueChangeEvent() { }
@@ -54,13 +54,13 @@ public class JobQueueChangeEvent implements HistoryEvent {

/** Get the Job ID */
public JobID getJobId() {
- return JobID.forName(datum.jobid.toString());
+ return JobID.forName(datum.getJobid().toString());
}

/** Get the new Job queue name */
public String getJobQueueName() {
- if (datum.jobQueueName != null) {
- return datum.jobQueueName.toString();
+ if (datum.getJobQueueName() != null) {
+ return datum.getJobQueueName().toString();
}
return null;
}
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index bcba56eced..2d52e3ebb1 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -37,7 +37,7 @@
<!--Whether to proceed to next module if any test failures exist-->
<maven.test.failure.ignore>true</maven.test.failure.ignore>
<maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
- <jetty.version>9.4.53.v20231009</jetty.version>
+ <jetty.version>9.4.56.v20240826</jetty.version>
<test.exclude>_</test.exclude>
<test.exclude.pattern>_</test.exclude.pattern>

@@ -63,7 +63,7 @@
<java.security.egd>file:///dev/urandom</java.security.egd>

<!-- avro version -->
- <avro.version>1.9.2</avro.version>
+ <avro.version>1.11.4</avro.version>

<!-- jersey version -->
<jersey.version>1.19.4</jersey.version>
@@ -108,7 +108,7 @@
<findbugs.version>3.0.5</findbugs.version>
<dnsjava.version>3.6.1</dnsjava.version>

- <guava.version>27.0-jre</guava.version>
+ <guava.version>32.0.1-jre</guava.version>
<guice.version>4.2.3</guice.version>

<bouncycastle.version>1.78.1</bouncycastle.version>
@@ -143,7 +143,7 @@
<jna.version>5.2.0</jna.version>
<gson.version>2.9.0</gson.version>
<metrics.version>3.2.4</metrics.version>
- <netty4.version>4.1.100.Final</netty4.version>
+ <netty4.version>4.1.108.Final</netty4.version>
<snappy-java.version>1.1.10.4</snappy-java.version>
<lz4-java.version>1.7.1</lz4-java.version>

diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobBuilder.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobBuilder.java
index 1213e6a46f..603b248f6e 100644
--- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobBuilder.java
+++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobBuilder.java
@@ -460,7 +460,7 @@ public class JobBuilder {
}
task.setFinishTime(event.getFinishTime());
task.setTaskStatus(getPre21Value(event.getTaskStatus()));
- task.incorporateCounters(((TaskFinished) event.getDatum()).counters);
+ task.incorporateCounters(((TaskFinished) event.getDatum()).getCounters());
}

private void processTaskFailedEvent(TaskFailedEvent event) {
@@ -472,7 +472,7 @@ public class JobBuilder {
task.setFinishTime(event.getFinishTime());
task.setTaskStatus(getPre21Value(event.getTaskStatus()));
TaskFailed t = (TaskFailed)(event.getDatum());
- task.putDiagnosticInfo(t.error.toString());
+ task.putDiagnosticInfo(t.getError().toString());
// killed task wouldn't have failed attempt.
if (t.getFailedDueToAttempt() != null) {
task.putFailedDueToAttemptId(t.getFailedDueToAttempt().toString());
@@ -542,7 +542,7 @@ public class JobBuilder {
}
attempt.setFinishTime(event.getFinishTime());
attempt
- .incorporateCounters(((TaskAttemptFinished) event.getDatum()).counters);
+ .incorporateCounters(((TaskAttemptFinished) event.getDatum()).getCounters());
}

private void processReduceAttemptFinishedEvent(
@@ -568,7 +568,7 @@ public class JobBuilder {
attempt.setShuffleFinished(event.getShuffleFinishTime());
attempt.setSortFinished(event.getSortFinishTime());
attempt
- .incorporateCounters(((ReduceAttemptFinished) event.getDatum()).counters);
+ .incorporateCounters(((ReduceAttemptFinished) event.getDatum()).getCounters());
attempt.arraySetClockSplits(event.getClockSplits());
attempt.arraySetCpuUsages(event.getCpuUsages());
attempt.arraySetVMemKbytes(event.getVMemKbytes());
@@ -596,7 +596,7 @@ public class JobBuilder {
// is redundant, but making this will add future-proofing.
attempt.setFinishTime(event.getFinishTime());
attempt
- .incorporateCounters(((MapAttemptFinished) event.getDatum()).counters);
+ .incorporateCounters(((MapAttemptFinished) event.getDatum()).getCounters());
attempt.arraySetClockSplits(event.getClockSplits());
attempt.arraySetCpuUsages(event.getCpuUsages());
attempt.arraySetVMemKbytes(event.getVMemKbytes());
@@ -661,11 +661,11 @@ public class JobBuilder {

JobFinished job = (JobFinished)event.getDatum();
Map<String, Long> countersMap =
- JobHistoryUtils.extractCounters(job.totalCounters);
+ JobHistoryUtils.extractCounters(job.getTotalCounters());
result.putTotalCounters(countersMap);
- countersMap = JobHistoryUtils.extractCounters(job.mapCounters);
+ countersMap = JobHistoryUtils.extractCounters(job.getMapCounters());
result.putMapCounters(countersMap);
- countersMap = JobHistoryUtils.extractCounters(job.reduceCounters);
+ countersMap = JobHistoryUtils.extractCounters(job.getReduceCounters());
result.putReduceCounters(countersMap);
}

diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobHistoryUtils.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobHistoryUtils.java
index 6ae87bbd40..34ef95f337 100644
--- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobHistoryUtils.java
+++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobHistoryUtils.java
@@ -157,9 +157,9 @@ public class JobHistoryUtils {
static Map<String, Long> extractCounters(JhCounters counters) {
Map<String, Long> countersMap = new HashMap<String, Long>();
if (counters != null) {
- for (JhCounterGroup group : counters.groups) {
- for (JhCounter counter : group.counts) {
- countersMap.put(counter.name.toString(), counter.value);
+ for (JhCounterGroup group : counters.getGroups()) {
+ for (JhCounter counter : group.getCounts()) {
+ countersMap.put(counter.getName().toString(), counter.getValue());
}
}
}
diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTask.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTask.java
index 4ae33a7661..2308e58690 100644
--- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTask.java
+++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTask.java
@@ -268,11 +268,11 @@ public class LoggedTask implements DeepCompare {
String counterName) {
counterName = canonicalizeCounterName(counterName);

- for (JhCounterGroup group : counters.groups) {
- for (JhCounter counter : group.counts) {
+ for (JhCounterGroup group : counters.getGroups()) {
+ for (JhCounter counter : group.getCounts()) {
if (counterName
- .equals(canonicalizeCounterName(counter.name.toString()))) {
- thunk.set(counter.value);
+ .equals(canonicalizeCounterName(counter.getName().toString()))) {
+ thunk.set(counter.getValue());
return;
}
}
diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java
index 5c6abd372c..fae53b2926 100644
--- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java
+++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java
@@ -636,11 +636,11 @@ public class LoggedTaskAttempt implements DeepCompare {
String counterName) {
counterName = canonicalizeCounterName(counterName);

- for (JhCounterGroup group : counters.groups) {
- for (JhCounter counter : group.counts) {
+ for (JhCounterGroup group : counters.getGroups()) {
+ for (JhCounter counter : group.getCounts()) {
if (counterName
- .equals(canonicalizeCounterName(counter.name.toString()))) {
- thunk.set(counter.value);
+ .equals(canonicalizeCounterName(counter.getName().toString()))) {
+ thunk.set(counter.getValue());
return;
}
}
@@ -769,4 +769,4 @@ public class LoggedTaskAttempt implements DeepCompare {
compare1(vMemKbytes, other.vMemKbytes, loc, "vMemKbytes");
compare1(physMemKbytes, other.physMemKbytes, loc, "physMemKbytes");
}
-}
\ No newline at end of file
+}
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
index c01d9c42820..ee9e9040ec8 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
@@ -23,14 +23,6 @@
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
- <fileSet>
- <directory>../hadoop-archive-logs/src/main/shellprofile.d</directory>
- <includes>
- <include>*</include>
- </includes>
- <outputDirectory>/libexec/shellprofile.d</outputDirectory>
- <fileMode>0755</fileMode>
- </fileSet>
<fileSet>
<directory>../hadoop-archives/src/main/shellprofile.d</directory>
<includes>
@@ -98,13 +90,6 @@
<include>*-sources.jar</include>
</includes>
</fileSet>
- <fileSet>
- <directory>../hadoop-archive-logs/target</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
- <includes>
- <include>*-sources.jar</include>
- </includes>
- </fileSet>
<fileSet>
<directory>../hadoop-datajoin/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 2d52e3ebb12..84a618f8ab6 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -646,11 +646,6 @@
<artifactId>hadoop-archives</artifactId>
<version>${hadoop.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-archive-logs</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-distcp</artifactId>
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 8e85f379ef7..014df29226d 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -192,7 +192,6 @@
<menu name="Tools" inherit="top">
<item name="Hadoop Streaming" href="hadoop-streaming/HadoopStreaming.html"/>
<item name="Hadoop Archives" href="hadoop-archives/HadoopArchives.html"/>
- <item name="Hadoop Archive Logs" href="hadoop-archive-logs/HadoopArchiveLogs.html"/>
<item name="DistCp" href="hadoop-distcp/DistCp.html"/>
<item name="HDFS Federation Balance" href="hadoop-federation-balance/HDFSFederationBalance.html"/>
<item name="GridMix" href="hadoop-gridmix/GridMix.html"/>
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index afd7d33b494..68a5b04a0b2 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -60,11 +60,6 @@
<artifactId>hadoop-archives</artifactId>
<scope>compile</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-archive-logs</artifactId>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-rumen</artifactId>
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index 1818c65e313..64927f70b6e 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -35,7 +35,6 @@
<module>hadoop-federation-balance</module>
<module>hadoop-dynamometer</module>
<module>hadoop-archives</module>
- <module>hadoop-archive-logs</module>
<module>hadoop-rumen</module>
<module>hadoop-gridmix</module>
<module>hadoop-datajoin</module>
89 changes: 89 additions & 0 deletions hadoop/stackable/patches/3.4.1/004-remove-hadoop-archives.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
index ee9e9040ec8..67f1c9a4586 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
@@ -23,14 +23,6 @@
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
- <fileSet>
- <directory>../hadoop-archives/src/main/shellprofile.d</directory>
- <includes>
- <include>*</include>
- </includes>
- <outputDirectory>/libexec/shellprofile.d</outputDirectory>
- <fileMode>0755</fileMode>
- </fileSet>
<fileSet>
<directory>../hadoop-distcp/src/main/shellprofile.d</directory>
<includes>
@@ -83,13 +75,6 @@
</includes>
<outputDirectory>lib/native/examples</outputDirectory>
</fileSet>
- <fileSet>
- <directory>../hadoop-archives/target</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
- <includes>
- <include>*-sources.jar</include>
- </includes>
- </fileSet>
<fileSet>
<directory>../hadoop-datajoin/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 84a618f8ab6..a751b9dd84d 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -641,11 +641,6 @@
<artifactId>hadoop-streaming</artifactId>
<version>${hadoop.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-archives</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-distcp</artifactId>
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 014df29226d..d4fe2c03160 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -191,7 +191,6 @@

<menu name="Tools" inherit="top">
<item name="Hadoop Streaming" href="hadoop-streaming/HadoopStreaming.html"/>
- <item name="Hadoop Archives" href="hadoop-archives/HadoopArchives.html"/>
<item name="DistCp" href="hadoop-distcp/DistCp.html"/>
<item name="HDFS Federation Balance" href="hadoop-federation-balance/HDFSFederationBalance.html"/>
<item name="GridMix" href="hadoop-gridmix/GridMix.html"/>
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 68a5b04a0b2..470afb1eaa6 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -55,11 +55,6 @@
<artifactId>hadoop-federation-balance</artifactId>
<scope>compile</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-archives</artifactId>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-rumen</artifactId>
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index 64927f70b6e..6178bb15ecf 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -34,7 +34,6 @@
<module>hadoop-distcp</module>
<module>hadoop-federation-balance</module>
<module>hadoop-dynamometer</module>
- <module>hadoop-archives</module>
<module>hadoop-rumen</module>
<module>hadoop-gridmix</module>
<module>hadoop-datajoin</module>
12 changes: 12 additions & 0 deletions hadoop/stackable/patches/3.4.1/005-remove-hadoop-benchmark.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index 6178bb15ecf..16b2adeda68 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -49,7 +49,6 @@
<module>hadoop-azure-datalake</module>
<module>hadoop-aliyun</module>
<module>hadoop-fs2img</module>
- <module>hadoop-benchmark</module>
</modules>

<build>
62 changes: 62 additions & 0 deletions hadoop/stackable/patches/3.4.1/006-remove-hadoop-datajoin.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
index 67f1c9a4586..9121e90cdbd 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
@@ -75,13 +75,6 @@
</includes>
<outputDirectory>lib/native/examples</outputDirectory>
</fileSet>
- <fileSet>
- <directory>../hadoop-datajoin/target</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
- <includes>
- <include>*-sources.jar</include>
- </includes>
- </fileSet>
<fileSet>
<directory>../hadoop-distcp/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index a751b9dd84d..acc94c6e50e 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -663,11 +663,6 @@
<version>${hadoop.version}</version>
<type>test-jar</type>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-datajoin</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-rumen</artifactId>
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 470afb1eaa6..1c3afdfeb82 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -60,11 +60,6 @@
<artifactId>hadoop-rumen</artifactId>
<scope>compile</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-datajoin</artifactId>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-extras</artifactId>
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index 16b2adeda68..0f4b0954e1e 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -36,7 +36,6 @@
<module>hadoop-dynamometer</module>
<module>hadoop-rumen</module>
<module>hadoop-gridmix</module>
- <module>hadoop-datajoin</module>
<module>hadoop-tools-dist</module>
<module>hadoop-extras</module>
<module>hadoop-pipes</module>
36 changes: 36 additions & 0 deletions hadoop/stackable/patches/3.4.1/007-remove-hadoop-dynamometer.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
diff --git a/dev-support/bin/dist-layout-stitching b/dev-support/bin/dist-layout-stitching
index d4bfd8aaada..e769b26ff84 100755
--- a/dev-support/bin/dist-layout-stitching
+++ b/dev-support/bin/dist-layout-stitching
@@ -149,7 +149,6 @@ run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-runtime/target/hadoop-cli
run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-${VERSION}.jar" share/hadoop/client/

run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERSION}" .
-run copy "${ROOT}/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-dist/target/hadoop-dynamometer-dist-${VERSION}" .


echo
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index d4fe2c03160..4da6c2fdc20 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -198,7 +198,6 @@
<item name="Resource Estimator Service" href="hadoop-resourceestimator/ResourceEstimator.html"/>
<item name="Scheduler Load Simulator" href="hadoop-sls/SchedulerLoadSimulator.html"/>
<item name="Hadoop Benchmarking" href="hadoop-project-dist/hadoop-common/Benchmarking.html"/>
- <item name="Dynamometer" href="hadoop-dynamometer/Dynamometer.html"/>
</menu>

<menu name="Reference" inherit="top">
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index 0f4b0954e1e..01ef91f75f6 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -33,7 +33,6 @@
<module>hadoop-streaming</module>
<module>hadoop-distcp</module>
<module>hadoop-federation-balance</module>
- <module>hadoop-dynamometer</module>
<module>hadoop-rumen</module>
<module>hadoop-gridmix</module>
<module>hadoop-tools-dist</module>
77 changes: 77 additions & 0 deletions hadoop/stackable/patches/3.4.1/008-remove-hadoop-extras.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
index 9121e90cdbd..5d59374cbf0 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
@@ -39,14 +39,6 @@
<outputDirectory>/libexec/shellprofile.d</outputDirectory>
<fileMode>0755</fileMode>
</fileSet>
- <fileSet>
- <directory>../hadoop-extras/src/main/shellprofile.d</directory>
- <includes>
- <include>*</include>
- </includes>
- <outputDirectory>/libexec/shellprofile.d</outputDirectory>
- <fileMode>0755</fileMode>
- </fileSet>
<fileSet>
<directory>../hadoop-pipes/src/main/native/pipes/api/hadoop</directory>
<includes>
@@ -89,13 +81,6 @@
<include>*-sources.jar</include>
</includes>
</fileSet>
- <fileSet>
- <directory>../hadoop-extras/target</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
- <includes>
- <include>*-sources.jar</include>
- </includes>
- </fileSet>
<fileSet>
<directory>../hadoop-gridmix/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index acc94c6e50e..ffe72c5ac49 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -668,11 +668,6 @@
<artifactId>hadoop-rumen</artifactId>
<version>${hadoop.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-extras</artifactId>
- <version>${hadoop.version}</version>
- </dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 1c3afdfeb82..8abe0914853 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -60,11 +60,6 @@
<artifactId>hadoop-rumen</artifactId>
<scope>compile</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-extras</artifactId>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-gridmix</artifactId>
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index 01ef91f75f6..fdfccd6b274 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -36,7 +36,6 @@
<module>hadoop-rumen</module>
<module>hadoop-gridmix</module>
<module>hadoop-tools-dist</module>
- <module>hadoop-extras</module>
<module>hadoop-pipes</module>
<module>hadoop-openstack</module>
<module>hadoop-sls</module>
82 changes: 82 additions & 0 deletions hadoop/stackable/patches/3.4.1/009-remove-hadoop-gridmix.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
index 5d59374cbf0..f99ebc66e8a 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
@@ -81,21 +81,6 @@
<include>*-sources.jar</include>
</includes>
</fileSet>
- <fileSet>
- <directory>../hadoop-gridmix/target</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
- <includes>
- <include>*-sources.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>../hadoop-gridmix/src/main/shellprofile.d</directory>
- <includes>
- <include>*</include>
- </includes>
- <outputDirectory>/libexec/shellprofile.d</outputDirectory>
- <fileMode>0755</fileMode>
- </fileSet>
<fileSet>
<directory>../hadoop-rumen/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index ffe72c5ac49..056b64c46a1 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -630,11 +630,6 @@
<artifactId>hadoop-mapreduce-examples</artifactId>
<version>${hadoop.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-gridmix</artifactId>
- <version>${hadoop.version}</version>
- </dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 4da6c2fdc20..56b942953f2 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -193,7 +193,6 @@
<item name="Hadoop Streaming" href="hadoop-streaming/HadoopStreaming.html"/>
<item name="DistCp" href="hadoop-distcp/DistCp.html"/>
<item name="HDFS Federation Balance" href="hadoop-federation-balance/HDFSFederationBalance.html"/>
- <item name="GridMix" href="hadoop-gridmix/GridMix.html"/>
<item name="Rumen" href="hadoop-rumen/Rumen.html"/>
<item name="Resource Estimator Service" href="hadoop-resourceestimator/ResourceEstimator.html"/>
<item name="Scheduler Load Simulator" href="hadoop-sls/SchedulerLoadSimulator.html"/>
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 8abe0914853..01301855fcc 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -60,11 +60,6 @@
<artifactId>hadoop-rumen</artifactId>
<scope>compile</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-gridmix</artifactId>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-pipes</artifactId>
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index fdfccd6b274..ff98d837a7e 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -34,7 +34,6 @@
<module>hadoop-distcp</module>
<module>hadoop-federation-balance</module>
<module>hadoop-rumen</module>
- <module>hadoop-gridmix</module>
<module>hadoop-tools-dist</module>
<module>hadoop-pipes</module>
<module>hadoop-openstack</module>
29 changes: 29 additions & 0 deletions hadoop/stackable/patches/3.4.1/010-remove-hadoop-kafka.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 01301855fcc..fd38216ef5a 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -73,12 +73,6 @@
<scope>compile</scope>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-kafka</artifactId>
- <scope>compile</scope>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-azure</artifactId>
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index ff98d837a7e..74fc141ddf7 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -41,7 +41,6 @@
<module>hadoop-resourceestimator</module>
<module>hadoop-azure</module>
<module>hadoop-aws</module>
- <module>hadoop-kafka</module>
<module>hadoop-azure-datalake</module>
<module>hadoop-aliyun</module>
<module>hadoop-fs2img</module>
30 changes: 30 additions & 0 deletions hadoop/stackable/patches/3.4.1/011-remove-hadoop-openstack.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 056b64c46a1..253208f1eeb 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -682,13 +682,6 @@
<version>${hadoop.version}</version>
</dependency>

- <!-- This is empty; retained only for downstream app build compatibility. -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-openstack</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-azure</artifactId>
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index 74fc141ddf7..d0d152c3172 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -36,7 +36,6 @@
<module>hadoop-rumen</module>
<module>hadoop-tools-dist</module>
<module>hadoop-pipes</module>
- <module>hadoop-openstack</module>
<module>hadoop-sls</module>
<module>hadoop-resourceestimator</module>
<module>hadoop-azure</module>
78 changes: 78 additions & 0 deletions hadoop/stackable/patches/3.4.1/012-remove-hadoop-pipes.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
index f99ebc66e8a..76eb0568048 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
@@ -39,34 +39,6 @@
<outputDirectory>/libexec/shellprofile.d</outputDirectory>
<fileMode>0755</fileMode>
</fileSet>
- <fileSet>
- <directory>../hadoop-pipes/src/main/native/pipes/api/hadoop</directory>
- <includes>
- <include>*.hh</include>
- </includes>
- <outputDirectory>/include</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>../hadoop-pipes/src/main/native/utils/api/hadoop</directory>
- <includes>
- <include>*.hh</include>
- </includes>
- <outputDirectory>/include</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>../hadoop-pipes/target/native</directory>
- <includes>
- <include>*.a</include>
- </includes>
- <outputDirectory>lib/native</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>../hadoop-pipes/target/native/examples</directory>
- <includes>
- <include>*</include>
- </includes>
- <outputDirectory>lib/native/examples</outputDirectory>
- </fileSet>
<fileSet>
<directory>../hadoop-distcp/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
@@ -158,8 +130,6 @@
<exclude>org.apache.hadoop:hadoop-common</exclude>
<exclude>org.apache.hadoop:hadoop-hdfs</exclude>
<exclude>org.apache.hadoop:hadoop-client</exclude>
- <!-- pipes is native stuff, this just keeps pom from being package-->
- <exclude>org.apache.hadoop:hadoop-pipes</exclude>
<!-- use slf4j from common to avoid multiple binding warnings -->
<exclude>org.slf4j:slf4j-api</exclude>
<exclude>org.slf4j:slf4j-reload4j</exclude>
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index fd38216ef5a..a5dbad6cb7d 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -60,13 +60,6 @@
<artifactId>hadoop-rumen</artifactId>
<scope>compile</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-pipes</artifactId>
- <scope>compile</scope>
- <type>pom</type>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index d0d152c3172..6ddcc96f82f 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -35,7 +35,6 @@
<module>hadoop-federation-balance</module>
<module>hadoop-rumen</module>
<module>hadoop-tools-dist</module>
- <module>hadoop-pipes</module>
<module>hadoop-sls</module>
<module>hadoop-resourceestimator</module>
<module>hadoop-azure</module>
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 56b942953f2..45c1c19ea75 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -194,7 +194,6 @@
<item name="DistCp" href="hadoop-distcp/DistCp.html"/>
<item name="HDFS Federation Balance" href="hadoop-federation-balance/HDFSFederationBalance.html"/>
<item name="Rumen" href="hadoop-rumen/Rumen.html"/>
- <item name="Resource Estimator Service" href="hadoop-resourceestimator/ResourceEstimator.html"/>
<item name="Scheduler Load Simulator" href="hadoop-sls/SchedulerLoadSimulator.html"/>
<item name="Hadoop Benchmarking" href="hadoop-project-dist/hadoop-common/Benchmarking.html"/>
</menu>
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index a5dbad6cb7d..8781f0b9b42 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -83,12 +83,6 @@
<artifactId>hadoop-sls</artifactId>
<scope>compile</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-resourceestimator</artifactId>
- <version>${project.version}</version>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-azure-datalake</artifactId>
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index 6ddcc96f82f..48920bab35b 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -36,7 +36,6 @@
<module>hadoop-rumen</module>
<module>hadoop-tools-dist</module>
<module>hadoop-sls</module>
- <module>hadoop-resourceestimator</module>
<module>hadoop-azure</module>
<module>hadoop-aws</module>
<module>hadoop-azure-datalake</module>
78 changes: 78 additions & 0 deletions hadoop/stackable/patches/3.4.1/014-remove-hadoop-sls.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
index 76eb0568048..411ae8d6251 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
@@ -83,17 +83,6 @@
<outputDirectory>/libexec/shellprofile.d</outputDirectory>
<fileMode>0755</fileMode>
</fileSet>
- <fileSet>
- <directory>../hadoop-sls/target</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
- <includes>
- <include>*-sources.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>../hadoop-sls/target/hadoop-sls-${project.version}/sls</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/sls</outputDirectory>
- </fileSet>
<fileSet>
<directory>../hadoop-resourceestimator/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 253208f1eeb..4678ceba842 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1507,11 +1507,6 @@
<artifactId>metrics-core</artifactId>
<version>${metrics.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-sls</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-cloud-storage</artifactId>
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 45c1c19ea75..d212938b2e5 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -194,7 +194,6 @@
<item name="DistCp" href="hadoop-distcp/DistCp.html"/>
<item name="HDFS Federation Balance" href="hadoop-federation-balance/HDFSFederationBalance.html"/>
<item name="Rumen" href="hadoop-rumen/Rumen.html"/>
- <item name="Scheduler Load Simulator" href="hadoop-sls/SchedulerLoadSimulator.html"/>
<item name="Hadoop Benchmarking" href="hadoop-project-dist/hadoop-common/Benchmarking.html"/>
</menu>

diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 8781f0b9b42..eae261ade7a 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -78,11 +78,6 @@
<scope>compile</scope>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-sls</artifactId>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-azure-datalake</artifactId>
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index 48920bab35b..9e520eb1125 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -35,7 +35,6 @@
<module>hadoop-federation-balance</module>
<module>hadoop-rumen</module>
<module>hadoop-tools-dist</module>
- <module>hadoop-sls</module>
<module>hadoop-azure</module>
<module>hadoop-aws</module>
<module>hadoop-azure-datalake</module>
82 changes: 82 additions & 0 deletions hadoop/stackable/patches/3.4.1/015-remove-hadoop-rumen.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
index 411ae8d6251..89b474f9afc 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
@@ -53,21 +53,6 @@
<include>*-sources.jar</include>
</includes>
</fileSet>
- <fileSet>
- <directory>../hadoop-rumen/target</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
- <includes>
- <include>*-sources.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>../hadoop-rumen/src/main/shellprofile.d</directory>
- <includes>
- <include>*</include>
- </includes>
- <outputDirectory>/libexec/shellprofile.d</outputDirectory>
- <fileMode>0755</fileMode>
- </fileSet>
<fileSet>
<directory>../hadoop-streaming/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 4678ceba842..31d6da673ae 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -658,11 +658,6 @@
<version>${hadoop.version}</version>
<type>test-jar</type>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-rumen</artifactId>
- <version>${hadoop.version}</version>
- </dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index d212938b2e5..8be5e704e86 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -193,7 +193,6 @@
<item name="Hadoop Streaming" href="hadoop-streaming/HadoopStreaming.html"/>
<item name="DistCp" href="hadoop-distcp/DistCp.html"/>
<item name="HDFS Federation Balance" href="hadoop-federation-balance/HDFSFederationBalance.html"/>
- <item name="Rumen" href="hadoop-rumen/Rumen.html"/>
<item name="Hadoop Benchmarking" href="hadoop-project-dist/hadoop-common/Benchmarking.html"/>
</menu>

diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index eae261ade7a..35f9b0be70f 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -55,11 +55,6 @@
<artifactId>hadoop-federation-balance</artifactId>
<scope>compile</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-rumen</artifactId>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index 9e520eb1125..5b3ebd13a42 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -33,7 +33,6 @@
<module>hadoop-streaming</module>
<module>hadoop-distcp</module>
<module>hadoop-federation-balance</module>
- <module>hadoop-rumen</module>
<module>hadoop-tools-dist</module>
<module>hadoop-azure</module>
<module>hadoop-aws</module>
88 changes: 88 additions & 0 deletions hadoop/stackable/patches/3.4.1/016-remove-hadoop-streaming.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
index 89b474f9afc..d5ff61d9ffd 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
@@ -53,21 +53,6 @@
<include>*-sources.jar</include>
</includes>
</fileSet>
- <fileSet>
- <directory>../hadoop-streaming/target</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
- <includes>
- <include>*-sources.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>../hadoop-streaming/src/main/shellprofile.d</directory>
- <includes>
- <include>*</include>
- </includes>
- <outputDirectory>/libexec/shellprofile.d</outputDirectory>
- <fileMode>0755</fileMode>
- </fileSet>
<fileSet>
<directory>../hadoop-resourceestimator/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 31d6da673ae..3b6a3c97f9b 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -631,11 +631,6 @@
<version>${hadoop.version}</version>
</dependency>

- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-streaming</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-distcp</artifactId>
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 8be5e704e86..10202de9d17 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -190,7 +190,6 @@
</menu>

<menu name="Tools" inherit="top">
- <item name="Hadoop Streaming" href="hadoop-streaming/HadoopStreaming.html"/>
<item name="DistCp" href="hadoop-distcp/DistCp.html"/>
<item name="HDFS Federation Balance" href="hadoop-federation-balance/HDFSFederationBalance.html"/>
<item name="Hadoop Benchmarking" href="hadoop-project-dist/hadoop-common/Benchmarking.html"/>
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 35f9b0be70f..04033969353 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -34,17 +34,6 @@
</properties>

<dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-streaming</artifactId>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>org.projectlombok</groupId>
- <artifactId>lombok</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-distcp</artifactId>
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index 5b3ebd13a42..10fc262e242 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -30,7 +30,6 @@
<packaging>pom</packaging>

<modules>
- <module>hadoop-streaming</module>
<module>hadoop-distcp</module>
<module>hadoop-federation-balance</module>
<module>hadoop-tools-dist</module>
44 changes: 44 additions & 0 deletions hadoop/stackable/patches/3.4.1/017-remove-hadoop-yarn-csi.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index cd86ce4e417..71031593c3c 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -220,10 +220,6 @@
<directory>hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/target/lib</directory>
<outputDirectory>share/hadoop/${hadoop.component}/timelineservice/lib</outputDirectory>
</fileSet>
- <fileSet>
- <directory>hadoop-yarn/hadoop-yarn-csi/target/lib</directory>
- <outputDirectory>share/hadoop/${hadoop.component}/csi/lib</outputDirectory>
- </fileSet>
<fileSet>
<directory>hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/target/lib</directory>
<outputDirectory>share/hadoop/${hadoop.component}/timelineservice/lib</outputDirectory>
@@ -266,16 +262,6 @@
<unpack>false</unpack>
</binaries>
</moduleSet>
- <moduleSet>
- <includes>
- <include>org.apache.hadoop:hadoop-yarn-csi</include>
- </includes>
- <binaries>
- <outputDirectory>share/hadoop/${hadoop.component}/csi</outputDirectory>
- <includeDependencies>false</includeDependencies>
- <unpack>false</unpack>
- </binaries>
- </moduleSet>
<moduleSet>
<excludes>
<exclude>org.apache.hadoop:hadoop-yarn-server-timelineservice*</exclude>
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml
index f1131576132..239afc8daf6 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -244,7 +244,6 @@
<module>hadoop-yarn-client</module>
<module>hadoop-yarn-registry</module>
<module>hadoop-yarn-ui</module>
- <module>hadoop-yarn-csi</module>
</modules>
<!-- -->
</project>
30 changes: 30 additions & 0 deletions hadoop/stackable/patches/3.4.1/018-remove-hadoop-yarn-ui.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 71031593c3c..9230f0c7a75 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -203,13 +203,6 @@
<fileSet>
<directory>${project.build.directory}/site</directory>
<outputDirectory>/share/doc/hadoop/${hadoop.component}</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>hadoop-yarn/hadoop-yarn-ui/target/hadoop-yarn-ui-${project.version}</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/webapps/ui2</outputDirectory>
- <includes>
- <include>**/*</include>
- </includes>
</fileSet>
<!-- Copy dependecies from hadoop-yarn-server-timelineservice as well -->
<fileSet>
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml
index 239afc8daf6..40282e48a3b 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -243,7 +243,6 @@
<module>hadoop-yarn-site</module>
<module>hadoop-yarn-client</module>
<module>hadoop-yarn-registry</module>
- <module>hadoop-yarn-ui</module>
</modules>
<!-- -->
</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml
index 40282e48a3b..21a01043990 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -242,7 +242,6 @@
<module>hadoop-yarn-applications</module>
<module>hadoop-yarn-site</module>
<module>hadoop-yarn-client</module>
- <module>hadoop-yarn-registry</module>
</modules>
<!-- -->
</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 3b6a3c97f9b..9dea7400edb 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -605,13 +605,6 @@
<version>${hadoop.version}</version>
</dependency>

- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-applications-catalog-webapp</artifactId>
- <version>${hadoop.version}</version>
- <type>war</type>
- </dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
index 1e751a73d80..f453d0dadcf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
@@ -37,7 +37,6 @@
<module>hadoop-yarn-applications-distributedshell</module>
<module>hadoop-yarn-applications-unmanaged-am-launcher</module>
<module>hadoop-yarn-services</module>
- <module>hadoop-yarn-applications-catalog</module>
<module>hadoop-yarn-applications-mawo</module>
</modules>

diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml
index 564c051137b..0a3f39e41bc 100644
--- a/hadoop-yarn-project/pom.xml
+++ b/hadoop-yarn-project/pom.xml
@@ -85,11 +85,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-services-core</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-applications-catalog-webapp</artifactId>
- <type>war</type>
- </dependency>
</dependencies>

<build>
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 9230f0c7a75..7b0bf7219cc 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -79,13 +79,6 @@
<include>*-sources.jar</include>
</includes>
</fileSet>
- <fileSet>
- <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/target</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
- <includes>
- <include>*-sources.jar</include>
- </includes>
- </fileSet>
<fileSet>
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 9dea7400edb..cd9576b088f 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -555,12 +555,6 @@
<version>${hadoop.version}</version>
</dependency>

- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-applications-distributedshell</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-timeline-pluginstorage</artifactId>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
index f453d0dadcf..b550bf83deb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
@@ -34,7 +34,6 @@
<!-- Do not add dependencies here, add them to the POM of the leaf module -->

<modules>
- <module>hadoop-yarn-applications-distributedshell</module>
<module>hadoop-yarn-applications-unmanaged-am-launcher</module>
<module>hadoop-yarn-services</module>
<module>hadoop-yarn-applications-mawo</module>
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
index b550bf83deb..3933c52faad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
@@ -36,7 +36,6 @@
<modules>
<module>hadoop-yarn-applications-unmanaged-am-launcher</module>
<module>hadoop-yarn-services</module>
- <module>hadoop-yarn-applications-mawo</module>
</modules>

<profiles>
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 7b0bf7219cc..3c9cf4642ba 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -104,13 +104,6 @@
<include>*-sources.jar</include>
</includes>
</fileSet>
- <fileSet>
- <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/target</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
- <includes>
- <include>*-sources.jar</include>
- </includes>
- </fileSet>
<fileSet>
<directory>hadoop-yarn/hadoop-yarn-client/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
index 3933c52faad..b1c38d02b86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
@@ -34,7 +34,6 @@
<!-- Do not add dependencies here, add them to the POM of the leaf module -->

<modules>
- <module>hadoop-yarn-applications-unmanaged-am-launcher</module>
<module>hadoop-yarn-services</module>
</modules>

Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml
index 21a01043990..fbc1bc3304c 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -239,7 +239,6 @@
<module>hadoop-yarn-api</module>
<module>hadoop-yarn-common</module>
<module>hadoop-yarn-server</module>
- <module>hadoop-yarn-applications</module>
<module>hadoop-yarn-site</module>
<module>hadoop-yarn-client</module>
</modules>
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 3c9cf4642ba..5e5af377854 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -79,31 +79,6 @@
<include>*-sources.jar</include>
</includes>
</fileSet>
- <fileSet>
- <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/target</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
- <includes>
- <include>*-sources.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf</directory>
- <outputDirectory>etc/hadoop</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/yarn-service-examples</outputDirectory>
- <includes>
- <include>**/*</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/target</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
- <includes>
- <include>*-sources.jar</include>
- </includes>
- </fileSet>
<fileSet>
<directory>hadoop-yarn/hadoop-yarn-client/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index cd9576b088f..ad056195706 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -580,25 +580,6 @@
<version>${project.version}</version>
</dependency>

- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-services-core</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-services-core</artifactId>
- <version>${hadoop.version}</version>
- <type>test-jar</type>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-services-api</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
index b1c38d02b86..32bdf55d1d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
@@ -33,10 +33,6 @@

<!-- Do not add dependencies here, add them to the POM of the leaf module -->

- <modules>
- <module>hadoop-yarn-services</module>
- </modules>
-
<profiles>
<profile>
<id>clover</id>
diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml
index 0a3f39e41bc..e79e643a7f1 100644
--- a/hadoop-yarn-project/pom.xml
+++ b/hadoop-yarn-project/pom.xml
@@ -81,10 +81,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-globalpolicygenerator</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-services-core</artifactId>
- </dependency>
</dependencies>

<build>
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index ad056195706..9a1ae24385b 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -568,12 +568,6 @@
<version>${hadoop.version}</version>
</dependency>

- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-router</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-globalpolicygenerator</artifactId>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
index 1856e97a654..e62d38f14e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
@@ -45,7 +45,6 @@
<module>hadoop-yarn-server-timelineservice</module>
<module>hadoop-yarn-server-timelineservice-hbase</module>
<module>hadoop-yarn-server-timelineservice-hbase-tests</module>
- <module>hadoop-yarn-server-router</module>
<module>hadoop-yarn-server-timelineservice-documentstore</module>
<module>hadoop-yarn-server-globalpolicygenerator</module>
</modules>
diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml
index e79e643a7f1..2f5d50dc3fc 100644
--- a/hadoop-yarn-project/pom.xml
+++ b/hadoop-yarn-project/pom.xml
@@ -73,10 +73,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-router</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-globalpolicygenerator</artifactId>
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 9a1ae24385b..1cf40d06dfe 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -568,12 +568,6 @@
<version>${hadoop.version}</version>
</dependency>

- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-globalpolicygenerator</artifactId>
- <version>${project.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
index e62d38f14e9..7b0441dd616 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
@@ -46,6 +46,5 @@
<module>hadoop-yarn-server-timelineservice-hbase</module>
<module>hadoop-yarn-server-timelineservice-hbase-tests</module>
<module>hadoop-yarn-server-timelineservice-documentstore</module>
- <module>hadoop-yarn-server-globalpolicygenerator</module>
</modules>
</project>
diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml
index 2f5d50dc3fc..0ceeb5380de 100644
--- a/hadoop-yarn-project/pom.xml
+++ b/hadoop-yarn-project/pom.xml
@@ -73,10 +73,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-globalpolicygenerator</artifactId>
- </dependency>
</dependencies>

<build>
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 1cf40d06dfe..b623be5215f 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -555,19 +555,6 @@
<version>${hadoop.version}</version>
</dependency>

- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-timeline-pluginstorage</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-timeline-pluginstorage</artifactId>
- <type>test-jar</type>
- <version>${hadoop.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
index 7b0441dd616..ca533269634 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
@@ -41,7 +41,6 @@
<module>hadoop-yarn-server-sharedcachemanager</module>
<module>hadoop-yarn-server-tests</module>
<module>hadoop-yarn-server-applicationhistoryservice</module>
- <module>hadoop-yarn-server-timeline-pluginstorage</module>
<module>hadoop-yarn-server-timelineservice</module>
<module>hadoop-yarn-server-timelineservice-hbase</module>
<module>hadoop-yarn-server-timelineservice-hbase-tests</module>
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
diff --git a/hadoop-client-modules/pom.xml b/hadoop-client-modules/pom.xml
index 03e6c0d97df..6e044ae1181 100644
--- a/hadoop-client-modules/pom.xml
+++ b/hadoop-client-modules/pom.xml
@@ -37,7 +37,6 @@
<module>hadoop-client-minicluster</module>
<!-- Checks invariants above -->
<module>hadoop-client-check-invariants</module>
- <module>hadoop-client-check-test-invariants</module>
<!-- Attempt to use the created libraries -->
<module>hadoop-client-integration-tests</module>
</modules>
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index f035cc9692b..fa1878a1303 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -67,12 +67,6 @@
<type>pom</type>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client-check-test-invariants</artifactId>
- <type>pom</type>
- <scope>provided</scope>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-integration-tests</artifactId>
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index b623be5215f..abff3976b4f 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -285,12 +285,6 @@
<version>${hadoop.version}</version>
<type>pom</type>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client-check-test-invariants</artifactId>
- <version>${hadoop.version}</version>
- <type>pom</type>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-integration-tests</artifactId>
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
diff --git a/hadoop-client-modules/pom.xml b/hadoop-client-modules/pom.xml
index 6e044ae1181..1a858204c66 100644
--- a/hadoop-client-modules/pom.xml
+++ b/hadoop-client-modules/pom.xml
@@ -37,8 +37,6 @@
<module>hadoop-client-minicluster</module>
<!-- Checks invariants above -->
<module>hadoop-client-check-invariants</module>
- <!-- Attempt to use the created libraries -->
- <module>hadoop-client-integration-tests</module>
</modules>

</project>
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index fa1878a1303..d321f29be64 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -67,12 +67,6 @@
<type>pom</type>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client-integration-tests</artifactId>
- <scope>provided</scope>
- </dependency>
-
</dependencies>

<build>
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index abff3976b4f..b49ad05a1e0 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -285,11 +285,6 @@
<version>${hadoop.version}</version>
<type>pom</type>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client-integration-tests</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-runtime</artifactId>
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
diff --git a/dev-support/bin/dist-layout-stitching b/dev-support/bin/dist-layout-stitching
index e769b26ff84..1f6c1f79f5a 100755
--- a/dev-support/bin/dist-layout-stitching
+++ b/dev-support/bin/dist-layout-stitching
@@ -146,7 +146,6 @@ run cp -pr "${ROOT}/hadoop-common-project/hadoop-kms/target/hadoop-kms-${VERSION
run mkdir -p "share/hadoop/client"
run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-api/target/hadoop-client-api-${VERSION}.jar" share/hadoop/client/
run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-runtime/target/hadoop-client-runtime-${VERSION}.jar" share/hadoop/client/
-run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-${VERSION}.jar" share/hadoop/client/

run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERSION}" .

diff --git a/hadoop-client-modules/pom.xml b/hadoop-client-modules/pom.xml
index 1a858204c66..1b77a04bb4c 100644
--- a/hadoop-client-modules/pom.xml
+++ b/hadoop-client-modules/pom.xml
@@ -33,8 +33,6 @@
<module>hadoop-client-api</module>
<!-- Should be used at runtime scope for remaining classes necessary for hadoop-client-api to function -->
<module>hadoop-client-runtime</module>
- <!-- Should be used at test scope for those that need access to mini cluster that works with above api and runtime -->
- <module>hadoop-client-minicluster</module>
<!-- Checks invariants above -->
<module>hadoop-client-check-invariants</module>
</modules>
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index b49ad05a1e0..9ed26fbeb67 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -290,11 +290,6 @@
<artifactId>hadoop-client-runtime</artifactId>
<version>${hadoop.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client-minicluster</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
index ca533269634..0dffcee97f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
@@ -38,7 +38,6 @@
<module>hadoop-yarn-server-nodemanager</module>
<module>hadoop-yarn-server-web-proxy</module>
<module>hadoop-yarn-server-resourcemanager</module>
- <module>hadoop-yarn-server-sharedcachemanager</module>
<module>hadoop-yarn-server-tests</module>
<module>hadoop-yarn-server-applicationhistoryservice</module>
<module>hadoop-yarn-server-timelineservice</module>
2 changes: 2 additions & 0 deletions java-devel/Dockerfile
Original file line number Diff line number Diff line change
@@ -65,3 +65,5 @@ COPY --chown=stackable:0 java-devel/stackable/settings.xml /stackable/.m2/settin
# This variable is supported as of Log4j version 2.10 and
# disables the vulnerable feature
ENV LOG4J_FORMAT_MSG_NO_LOOKUPS=true

# COPY --chown=stackable:stackable shared/maven/settings.xml /stackable/.m2/
16 changes: 16 additions & 0 deletions renovate.json
Original file line number Diff line number Diff line change
@@ -3,5 +3,21 @@
"extends": [
"github>stackabletech/.github:renovate-config",
"docker:pinDigests"
],
"packageRules": [
{
"matchDatasources": [
"docker"
],
"registryUrls": [
"https://registry.access.redhat.com"
],
"packagePatterns": [
"^ubi9/ubi-minimal$"
],
"updateTypes": [
"version"
]
}
]
}
2 changes: 2 additions & 0 deletions stackable-base/Dockerfile
Original file line number Diff line number Diff line change
@@ -228,3 +228,5 @@ LABEL org.opencontainers.image.url="https://stackable.tech"
LABEL org.opencontainers.image.source="https://github.com/stackabletech/docker-images/"
LABEL org.opencontainers.image.vendor="Stackable GmbH"
LABEL org.opencontainers.image.licenses="Apache-2.0"

# COPY --chown=stackable:stackable shared/npm/.npmrc /stackable/.npmrc