From 80ef3efb4a12031c1d0a31ec6f9f29c65e6d6493 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Mon, 1 Jun 2026 19:57:44 +0530 Subject: [PATCH 1/5] HDDS-14946. Add CLI command for RewriteTablePathOzoneAction --- .../dev-support/bin/dist-layout-stitching | 8 + hadoop-ozone/dist/pom.xml | 6 + .../dist/src/shell/ozone/ozone-iceberg | 89 +++++++++++ hadoop-ozone/iceberg/pom.xml | 24 +-- .../iceberg/RewriteTablePathCommand.java | 139 ++++++++++++++++++ .../TestRewriteTablePathOzoneAction.java | 106 +++++++++---- 6 files changed, 337 insertions(+), 35 deletions(-) create mode 100644 hadoop-ozone/dist/src/shell/ozone/ozone-iceberg create mode 100644 hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathCommand.java diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching index 17723b208cf6..a08fcd126b12 100755 --- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching +++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching @@ -20,6 +20,9 @@ BASEDIR=$1 #hdds.version HDDS_VERSION=$2 +# true when ozone-iceberg is built (JDK 11+), false for JDK 8 builds +INCLUDE_ICEBERG=${3:-false} + ## @audience private ## @stability evolving function run() @@ -97,6 +100,11 @@ run cp -r "${ROOT}/hadoop-ozone/dist/src/main/dockerlibexec/." "libexec/" run cp "${ROOT}/hadoop-ozone/dist/src/shell/ozone/ozone" "bin/" +if [[ "${INCLUDE_ICEBERG}" == "true" ]]; then + run cp "${ROOT}/hadoop-ozone/dist/src/shell/ozone/ozone-iceberg" "bin/" + run chmod 755 "bin/ozone-iceberg" +fi + run cp "${ROOT}/hadoop-ozone/dist/src/shell/ozone/ozone-config.sh" "libexec/" run cp "${ROOT}/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh" "libexec/" diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml index 108ca1deb69d..176a57a51db8 100644 --- a/hadoop-ozone/dist/pom.xml +++ b/hadoop-ozone/dist/pom.xml @@ -37,6 +37,8 @@ -rocky true UTF-8 + + false true @@ -195,6 +197,7 @@ ${basedir}/dev-support/bin/dist-layout-stitching ${project.build.directory} ${hdds.version} + ${include.iceberg} @@ -279,6 +282,9 @@ [11,) + + true + org.apache.ozone diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone-iceberg b/hadoop-ozone/dist/src/shell/ozone/ozone-iceberg new file mode 100644 index 000000000000..3f9988d738fe --- /dev/null +++ b/hadoop-ozone/dist/src/shell/ozone/ozone-iceberg @@ -0,0 +1,89 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Entry point for the Iceberg CLI +#it’s the standard Ozone way to turn ozone-iceberg JAR + main class into a runnable command. +OZONE_SHELL_EXECNAME="ozone-iceberg" +MYNAME="${BASH_SOURCE-$0}" +bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) +JVM_PID="$$" + +# load functions +for dir in "${OZONE_LIBEXEC_DIR}" "${OZONE_HOME}/libexec" "${HADOOP_LIBEXEC_DIR}" "${HADOOP_HOME}/libexec" "${bin}/../libexec"; do + if [[ -e "${dir}/ozone-functions.sh" ]]; then + # shellcheck source=ozone-functions.sh + . "${dir}/ozone-functions.sh" + if declare -F ozone_bootstrap >& /dev/null; then + break + fi + fi +done + +if ! declare -F ozone_bootstrap >& /dev/null; then + echo "ERROR: Cannot find ozone-functions.sh." 2>&1 + exit 1 +fi + +## @description Usage for ozone-iceberg (used by ozone-config.sh / ozone_exit_with_usage). +## @audience public +function ozone_usage +{ + ozone_reset_usage + + ozone_add_subcommand "rewrite-path" client "rewrite Iceberg table paths for table migration (see ${OZONE_SHELL_EXECNAME} rewrite-path --help for tool options)" + + ozone_generate_usage "${OZONE_SHELL_EXECNAME}" false +} + +ozone_bootstrap +# shellcheck source=ozone-config.sh +. "${OZONE_LIBEXEC_DIR}/ozone-config.sh" + +MYNAME=$(ozone_abs "${MYNAME}") + +if [[ $# -eq 0 ]] || [[ "$1" != "rewrite-path" ]]; then + echo "Usage: ozone-iceberg rewrite-path [OPTIONS]" >&2 + exit 1 +fi +shift + +OZONE_CLASSNAME="org.apache.hadoop.ozone.iceberg.RewriteTablePathCommand" +OZONE_RUN_ARTIFACT_NAME="ozone-iceberg" +OZONE_SUBCMD="iceberg" +OZONE_SUBCMD_SUPPORTDAEMONIZATION=false +OZONE_SUBCMD_ARGS=("$@") + +ozone_validate_classpath + +if [[ -z "${OZONE_ORIGINAL_LOGLEVEL}" ]] && [[ -z "${OZONE_ORIGINAL_ROOT_LOGGER}" ]]; then + OZONE_LOGLEVEL=OFF + OZONE_ROOT_LOGGER="${OZONE_LOGLEVEL},console" + OZONE_OPTS="${OZONE_OPTS} -Dslf4j.internal.verbosity=ERROR" +fi + +ozone_assemble_classpath + +ozone_add_client_opts +ozone_add_server_opts + +ozone_subcommand_opts "ozone" "iceberg" + +ozone_add_default_gc_opts + +ozone_generic_java_subcmd_handler diff --git a/hadoop-ozone/iceberg/pom.xml b/hadoop-ozone/iceberg/pom.xml index d7b822c38608..53f5687b7a0a 100644 --- a/hadoop-ozone/iceberg/pom.xml +++ b/hadoop-ozone/iceberg/pom.xml @@ -32,6 +32,14 @@ + + info.picocli + picocli + + + org.apache.hadoop + hadoop-common + @@ -65,20 +73,18 @@ + + org.apache.ozone + hdds-common + org.slf4j slf4j-api - org.apache.hadoop - hadoop-common - test - - - org.apache.avro - avro - - + org.apache.ozone + ozone-filesystem + runtime diff --git a/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathCommand.java b/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathCommand.java new file mode 100644 index 000000000000..d034761ade2f --- /dev/null +++ b/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathCommand.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.iceberg; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.iceberg.Table; +import org.apache.iceberg.actions.RewriteTablePath; +import org.apache.iceberg.hadoop.HadoopTables; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +/** + * CLI to rewrite Iceberg table paths. The {@code ozone-iceberg} shell script strips the + * {@code rewrite-path} token before invoking this class (same pattern as {@code ozone debug}). + */ +@Command( + name = "rewrite-path", + description = "Rewrite Iceberg table paths for table migration", + mixinStandardHelpOptions = true +) +public class RewriteTablePathCommand implements Runnable { + + @Option( + names = {"-l", "--table-location"}, + required = true, + description = "The latest metadata.json file path of the table" + ) + private String tableLocation; + + @Option( + names = {"-s", "--source-prefix"}, + required = true, + description = "Source path prefix to replace" + ) + private String sourcePrefix; + + @Option( + names = {"-t", "--target-prefix"}, + required = true, + description = "Target path prefix" + ) + private String targetPrefix; + + @Option( + names = {"--staging"}, + description = "Staging location where all the rewritten files will be placed " + + "(Default is a new directory under the table's current metadata directory.)" + ) + private String stagingLocation; + + @Option( + names = {"--start-version"}, + description = "Start version metadata file name (optional, e.g., v1.metadata.json)" + ) + private String startVersion; + + @Option( + names = {"--end-version"}, + description = "End version metadata file name (optional, defaults to current)" + ) + private String endVersion; + + @Option( + names = {"--parallelism"}, + description = "Number of threads to use" + ) + private int parallelism; + + @Override + public void run() { + System.out.println("Starting Iceberg table path rewrite"); + System.out.println("Table location: " + tableLocation); + System.out.println("Source prefix: " + sourcePrefix); + System.out.println("Target prefix: " + targetPrefix); + + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem"); + + Table table = null; + if (tableLocation != null && !tableLocation.isBlank()) { + HadoopTables tables = new HadoopTables(conf); + table = tables.load(tableLocation.trim()); + System.out.println("Table loaded: " + table.location()); + } + + RewriteTablePathOzoneAction action; + if (parallelism > 0) { + action = new RewriteTablePathOzoneAction(table, parallelism); + } else { + action = new RewriteTablePathOzoneAction(table); + } + + RewriteTablePath rewriteAction = action.rewriteLocationPrefix(sourcePrefix, targetPrefix); + + if (stagingLocation != null && !stagingLocation.isBlank()) { + System.out.println("Staging location: " + stagingLocation); + rewriteAction.stagingLocation(stagingLocation); + } + + if (startVersion != null && !startVersion.isBlank()) { + System.out.println("Start version: " + startVersion); + rewriteAction.startVersion(startVersion); + } + + if (endVersion != null && !endVersion.isBlank()) { + System.out.println("End version: " + endVersion); + rewriteAction.endVersion(endVersion); + } + + RewriteTablePath.Result result = rewriteAction.execute(); + + System.out.println("\nRewrite completed successfully"); + System.out.println(" Latest version: " + result.latestVersion()); + System.out.println(" Staging location: " + result.stagingLocation()); + System.out.println("\nNext step: Copy files from source to target using the file list"); + System.out.println(" File list location: " + result.fileListLocation()); + } + + public static void main(String[] args) { + int exitCode = new CommandLine(new RewriteTablePathCommand()).execute(args); + System.exit(exitCode); + } +} diff --git a/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java b/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java index 87c9c665ff8a..01f42715aafc 100644 --- a/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java +++ b/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java @@ -23,18 +23,21 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.io.PrintStream; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.DeleteFile; @@ -62,10 +65,12 @@ import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.Pair; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.mockito.Mockito; +import picocli.CommandLine; /** * Testing path rewrite of iceberg table metadata files. @@ -84,6 +89,11 @@ class TestRewriteTablePathOzoneAction { private String targetPrefix = null; private Table table = null; + private ByteArrayOutputStream outContent; + private ByteArrayOutputStream errContent; + private PrintStream originalOut; + private PrintStream originalErr; + @TempDir private Path tableDir; @TempDir @@ -97,14 +107,24 @@ public void setupTableLocation() { this.table = createTable(tableLocation + "/"); this.sourcePrefix = tableLocation; this.targetPrefix = targetDir.toUri().toString().replaceFirst("^file:///", "file:/") + TABLE_NAME; + + outContent = new ByteArrayOutputStream(); + errContent = new ByteArrayOutputStream(); + originalOut = System.out; + originalErr = System.err; + System.setOut(new PrintStream(outContent, true, StandardCharsets.UTF_8)); + System.setErr(new PrintStream(errContent, true, StandardCharsets.UTF_8)); + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test void fullTablePathRewrite() throws Exception { - RewriteTablePath.Result result = new RewriteTablePathOzoneAction(table) - .rewriteLocationPrefix(sourcePrefix, targetPrefix) - .stagingLocation(stagingDir.toString() + "/") - .execute(); + String fileListLocation = executeRewriteCommand("--parallelism", "2"); List metadataPaths = metadataLogEntryPaths(table); Set expectedTargets = new HashSet<>(); @@ -112,7 +132,7 @@ void fullTablePathRewrite() throws Exception { expectedTargets.add(RewriteTablePathUtil.newPath(path, sourcePrefix, targetPrefix)); } - Set> csvPairs = readCsvPairs(table, result.fileListLocation()); + Set> csvPairs = readCsvPairs(table, fileListLocation); Set actualTargets = csvPairs.stream().map(Pair::second) .filter(p -> p.endsWith(".metadata.json")) .collect(Collectors.toSet()); @@ -128,11 +148,7 @@ void tablePathRewriteForStartAndNoEndVersionProvided() throws Exception { List metadataPaths = metadataLogEntryPaths(table); String startName = RewriteTablePathUtil.fileName(metadataPaths.get(2)); - RewriteTablePath.Result result = new RewriteTablePathOzoneAction(table) - .rewriteLocationPrefix(sourcePrefix, targetPrefix) - .stagingLocation(stagingDir.toString() + "/") - .startVersion(startName) - .execute(); + String fileListLocation = executeRewriteCommand("--start-version", startName); List expectedPaths = new ArrayList<>(); for (int i = metadataPaths.size() - 1; i >= 3; i--) { @@ -144,7 +160,7 @@ void tablePathRewriteForStartAndNoEndVersionProvided() throws Exception { expectedTargets.add(RewriteTablePathUtil.newPath(versionPath, sourcePrefix, targetPrefix)); } - Set> csvPairs = readCsvPairs(table, result.fileListLocation()); + Set> csvPairs = readCsvPairs(table, fileListLocation); Set actualTargets = csvPairs.stream().map(Pair::second) .filter(p -> p.endsWith(".metadata.json")) .collect(Collectors.toSet()); @@ -160,11 +176,7 @@ void tablePathRewriteForOnlyEndVersionProvided() throws Exception { List metadataPaths = metadataLogEntryPaths(table); String endName = RewriteTablePathUtil.fileName(metadataPaths.get(2)); - RewriteTablePath.Result result = new RewriteTablePathOzoneAction(table) - .rewriteLocationPrefix(sourcePrefix, targetPrefix) - .stagingLocation(stagingDir.toString() + "/") - .endVersion(endName) - .execute(); + String fileListLocation = executeRewriteCommand("--end-version", endName); List expectedPaths = new ArrayList<>(); for (int i = 2; i >= 0; i--) { @@ -176,7 +188,7 @@ void tablePathRewriteForOnlyEndVersionProvided() throws Exception { expectedTargets.add(RewriteTablePathUtil.newPath(versionPath, sourcePrefix, targetPrefix)); } - Set> csvPairs = readCsvPairs(table, result.fileListLocation()); + Set> csvPairs = readCsvPairs(table, fileListLocation); Set actualTargets = csvPairs.stream().map(Pair::second) .filter(p -> p.endsWith(".metadata.json")) .collect(Collectors.toSet()); @@ -193,12 +205,9 @@ void tablePathRewriteForStartAndEndVersionProvided() throws Exception { String startName = RewriteTablePathUtil.fileName(metadataPaths.get(1)); String endName = RewriteTablePathUtil.fileName(metadataPaths.get(3)); - RewriteTablePath.Result result = new RewriteTablePathOzoneAction(table) - .rewriteLocationPrefix(sourcePrefix, targetPrefix) - .stagingLocation(stagingDir.toString() + "/") - .startVersion(startName) - .endVersion(endName) - .execute(); + String fileListLocation = executeRewriteCommand( + "--start-version", startName, + "--end-version", endName); List expectedPaths = new ArrayList<>(); for (int i = 3; i >= 2; i--) { @@ -210,7 +219,7 @@ void tablePathRewriteForStartAndEndVersionProvided() throws Exception { expectedTargets.add(RewriteTablePathUtil.newPath(versionPath, sourcePrefix, targetPrefix)); } - Set> csvPairs = readCsvPairs(table, result.fileListLocation()); + Set> csvPairs = readCsvPairs(table, fileListLocation); Set actualTargets = csvPairs.stream().map(Pair::second) .filter(p -> p.endsWith(".metadata.json")) .collect(Collectors.toSet()); @@ -367,6 +376,51 @@ void statsFileCopyPlanReturnsBeforeToAfterPathPairs() { Pair.of("before-2.stats", "after-2.stats")), copyPlan); } + private CommandLine newRewritePathCommand() { + return new CommandLine(new RewriteTablePathCommand()); + } + + private String executeRewriteCommand(String... optionalArgs) { + List args = new ArrayList<>(); + args.add("-l"); + args.add(table.location()); + args.add("-s"); + args.add(sourcePrefix); + args.add("-t"); + args.add(targetPrefix); + args.add("--staging"); + args.add(stagingDir + "/"); + args.addAll(Arrays.asList(optionalArgs)); + + int exitCode = newRewritePathCommand().execute(args.toArray(new String[0])); + assertEquals(0, exitCode, + "Command failed.\nstdout:\n" + stdout() + "\nstderr:\n" + stderr()); + assertThat(stdout()) + .contains("Starting Iceberg table path rewrite") + .contains("Table loaded: " + table.location()) + .contains("Staging location: " + stagingDir + "/") + .contains("File list location:"); + return parseFileListLocation(stdout()); + } + + private String stdout() { + return outContent.toString(StandardCharsets.UTF_8); + } + + private String stderr() { + return errContent.toString(StandardCharsets.UTF_8); + } + + private static String parseFileListLocation(String output) { + for (String line : output.split("\n")) { + if (line.contains("File list location:")) { + return line.substring(line.indexOf("File list location:") + "File list location:".length()) + .trim(); + } + } + throw new IllegalStateException("File list location not found in command output: " + output); + } + /** * For every staged file in the CSV copy plan, asserts that internal paths are rewritten * to the target prefix: @@ -570,7 +624,7 @@ private static Set> readCsvPairs(Table tbl, String fileList } private Table createTable(String location) { - HadoopTables tables = new HadoopTables(new Configuration()); + HadoopTables tables = new HadoopTables(new OzoneConfiguration()); Table tbl = tables.create(SCHEMA, PartitionSpec.unpartitioned(), new HashMap<>(), location); for (int i = 0; i < COMMITS; i++) { String dataPath = location + "/data/batch-" + i + ".parquet"; From a867dfe468ef35eb0fa9e630272a2c73b9dc21e2 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Fri, 5 Jun 2026 10:46:58 +0530 Subject: [PATCH 2/5] Addressed minor review comments --- hadoop-ozone/iceberg/pom.xml | 5 +++++ .../iceberg/RewriteTablePathCommand.java | 19 +++++++++------- .../iceberg/RewriteTablePathOzoneAction.java | 22 +++++++++++-------- .../TestRewriteTablePathOzoneAction.java | 2 +- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/hadoop-ozone/iceberg/pom.xml b/hadoop-ozone/iceberg/pom.xml index fbb6d047fb98..56304a19d806 100644 --- a/hadoop-ozone/iceberg/pom.xml +++ b/hadoop-ozone/iceberg/pom.xml @@ -191,6 +191,11 @@ ozone-filesystem runtime + + org.slf4j + slf4j-reload4j + runtime + diff --git a/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathCommand.java b/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathCommand.java index d034761ade2f..c318251936c2 100644 --- a/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathCommand.java +++ b/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathCommand.java @@ -77,10 +77,11 @@ public class RewriteTablePathCommand implements Runnable { private String endVersion; @Option( - names = {"--parallelism"}, - description = "Number of threads to use" + names = {"--threads"}, + description = "Number of threads to use (positive integer). " + + "If omitted or zero, the default thread count is used." ) - private int parallelism; + private int threads; @Override public void run() { @@ -90,7 +91,6 @@ public void run() { System.out.println("Target prefix: " + targetPrefix); OzoneConfiguration conf = new OzoneConfiguration(); - conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem"); Table table = null; if (tableLocation != null && !tableLocation.isBlank()) { @@ -100,11 +100,12 @@ public void run() { } RewriteTablePathOzoneAction action; - if (parallelism > 0) { - action = new RewriteTablePathOzoneAction(table, parallelism); + if (threads > 0) { + action = new RewriteTablePathOzoneAction(table, threads); } else { action = new RewriteTablePathOzoneAction(table); } + System.out.println("Threads: " + action.getThreads()); RewriteTablePath rewriteAction = action.rewriteLocationPrefix(sourcePrefix, targetPrefix); @@ -125,10 +126,12 @@ public void run() { RewriteTablePath.Result result = rewriteAction.execute(); - System.out.println("\nRewrite completed successfully"); + System.out.println(); + System.out.println("Rewrite completed successfully"); System.out.println(" Latest version: " + result.latestVersion()); System.out.println(" Staging location: " + result.stagingLocation()); - System.out.println("\nNext step: Copy files from source to target using the file list"); + System.out.println(); + System.out.println("Next step: Copy files from source to target using the file list"); System.out.println(" File list location: " + result.fileListLocation()); } diff --git a/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathOzoneAction.java b/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathOzoneAction.java index 4dc50434a3c1..2ecb5e4e340a 100644 --- a/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathOzoneAction.java +++ b/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathOzoneAction.java @@ -96,7 +96,7 @@ public class RewriteTablePathOzoneAction implements RewriteTablePath { private String startVersionName; private String endVersionName; private String stagingDir; - private int parallelism; + private int threads; private ExecutorService executorService; private static final int MAX_INFLIGHT_MULTIPLIER = 4; @@ -106,12 +106,16 @@ public class RewriteTablePathOzoneAction implements RewriteTablePath { public RewriteTablePathOzoneAction(Table table) { this.table = table; - this.parallelism = DEFAULT_THREAD_COUNT; + this.threads = DEFAULT_THREAD_COUNT; } - public RewriteTablePathOzoneAction(Table table, int parallelism) { + public RewriteTablePathOzoneAction(Table table, int threads) { this.table = table; - this.parallelism = parallelism; + this.threads = threads; + } + + int getThreads() { + return threads; } @Override @@ -147,7 +151,7 @@ public RewriteTablePath stagingLocation(String stagingLocation) { @Override public Result execute() { validateInputs(); - executorService = Executors.newFixedThreadPool(parallelism); + executorService = Executors.newFixedThreadPool(threads); try { return doExecute(); } finally { @@ -326,7 +330,7 @@ private Set> rewriteVersionFile(TableMetadata metadata, Str private Set manifestsToRewrite(Set validSnapshots, Set deltaSnapshotIds) { Set manifestPaths = ConcurrentHashMap.newKeySet(); - int maxInFlight = parallelism * MAX_INFLIGHT_MULTIPLIER; + int maxInFlight = threads * MAX_INFLIGHT_MULTIPLIER; Semaphore semaphore = new Semaphore(maxInFlight); ExecutorCompletionService completionService = new ExecutorCompletionService<>(executorService); @@ -441,7 +445,7 @@ private RewriteResult rewriteManifestLists(Set validSnap return new RewriteResult<>(); } - int maxInFlight = parallelism * MAX_INFLIGHT_MULTIPLIER; + int maxInFlight = threads * MAX_INFLIGHT_MULTIPLIER; Semaphore semaphore = new Semaphore(maxInFlight); ExecutorCompletionService> completionService = new ExecutorCompletionService<>(executorService); @@ -535,7 +539,7 @@ private RewriteContentFileResult rewriteManifests( return new RewriteContentFileResult(); } - int maxInFlight = parallelism * MAX_INFLIGHT_MULTIPLIER; + int maxInFlight = threads * MAX_INFLIGHT_MULTIPLIER; Semaphore semaphore = new Semaphore(maxInFlight); ExecutorCompletionService completionService = new ExecutorCompletionService<>(executorService); @@ -735,7 +739,7 @@ private void rewritePositionDeletes(Set toRewrite) { } RewriteTablePathUtil.PositionDeleteReaderWriter posDeleteReaderWriter = new OzonePositionDeleteReaderWriter(); - int maxInFlight = parallelism * MAX_INFLIGHT_MULTIPLIER; + int maxInFlight = threads * MAX_INFLIGHT_MULTIPLIER; Semaphore semaphore = new Semaphore(maxInFlight); ExecutorCompletionService completionService = new ExecutorCompletionService<>(executorService); int submittedTasks = 0; diff --git a/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java b/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java index 120f0ad6d537..bd5bee9eae1e 100644 --- a/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java +++ b/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java @@ -142,7 +142,7 @@ public void restoreStreams() { @Test void fullTablePathRewrite() throws Exception { - String fileListLocation = executeRewriteCommand("--parallelism", "2"); + String fileListLocation = executeRewriteCommand("--threads", "2"); List metadataPaths = metadataLogEntryPaths(table); Set expectedTargets = new HashSet<>(); From 7a7af02f40df6076144002e15f1c00241a4ce82a Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Fri, 5 Jun 2026 11:22:36 +0530 Subject: [PATCH 3/5] Added ozone iceberg subcommand and droped ozone-iceberg script --- .../dev-support/bin/dist-layout-stitching | 17 ++-- hadoop-ozone/dist/src/shell/ozone/ozone | 32 ++++++- .../dist/src/shell/ozone/ozone-iceberg | 89 ------------------- .../iceberg/RewriteTablePathCommand.java | 3 +- 4 files changed, 42 insertions(+), 99 deletions(-) delete mode 100644 hadoop-ozone/dist/src/shell/ozone/ozone-iceberg diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching index a08fcd126b12..20f29a8c5ec9 100755 --- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching +++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching @@ -100,12 +100,6 @@ run cp -r "${ROOT}/hadoop-ozone/dist/src/main/dockerlibexec/." "libexec/" run cp "${ROOT}/hadoop-ozone/dist/src/shell/ozone/ozone" "bin/" -if [[ "${INCLUDE_ICEBERG}" == "true" ]]; then - run cp "${ROOT}/hadoop-ozone/dist/src/shell/ozone/ozone-iceberg" "bin/" - run chmod 755 "bin/ozone-iceberg" -fi - - run cp "${ROOT}/hadoop-ozone/dist/src/shell/ozone/ozone-config.sh" "libexec/" run cp "${ROOT}/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh" "libexec/" run cp -r "${ROOT}/hadoop-ozone/dist/src/shell/shellprofile.d" "libexec/" @@ -140,12 +134,23 @@ run mkdir compose/_keytabs for file in $(find "${ROOT}" -path '*/target/classes/*.classpath' | sort); do # We need to add the artifact manually as it's not part the generated classpath desciptor module=$(basename "${file%.classpath}") + if [[ "${INCLUDE_ICEBERG}" != "true" && "${module}" == "ozone-iceberg" ]]; then + continue + fi sed -i -e "s;$;:\$HDDS_LIB_JARS_DIR/${module}-${HDDS_VERSION}.jar;" "$file" run cp -p "$file" share/ozone/classpath/ done for file in $(find "${ROOT}" -path '*/share/ozone/lib/*jar' | sort); do + if [[ "${INCLUDE_ICEBERG}" != "true" ]]; then + jar_name=$(basename "${file}") + if [[ "${file}" == *"/hadoop-ozone/iceberg/target/"* ]] \ + || [[ "${jar_name}" == ozone-iceberg-*.jar ]] \ + || [[ "${jar_name}" == iceberg-*.jar ]]; then + continue + fi + fi # copy without printing to output due to large number of files cp -p "$file" share/ozone/lib/ done diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone b/hadoop-ozone/dist/src/shell/ozone/ozone index 432ed1d173d3..9e663e7e77d3 100755 --- a/hadoop-ozone/dist/src/shell/ozone/ozone +++ b/hadoop-ozone/dist/src/shell/ozone/ozone @@ -23,6 +23,16 @@ MYNAME="${BASH_SOURCE-$0}" bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) JVM_PID="$$" +## @description true when the ozone-iceberg artifact is present in this distribution +## @audience private +function ozone_iceberg_available +{ + local lib_dir="${HDDS_LIB_JARS_DIR:-${OZONE_HOME}/share/ozone/lib}" + + [[ -f "${OZONE_HOME}/share/ozone/classpath/ozone-iceberg.classpath" ]] \ + && compgen -G "${lib_dir}/ozone-iceberg-*.jar" > /dev/null +} + ## @description build up the ozone command's usage text. ## @audience public ## @stability stable @@ -65,7 +75,9 @@ function ozone_usage ozone_add_subcommand "repair" client "Ozone repair tool" ozone_add_subcommand "ratis" client "Ozone ratis tool" ozone_add_subcommand "vapor" client "Ozone server simulator" - + if ozone_iceberg_available; then + ozone_add_subcommand "iceberg" client "rewrite Iceberg table paths for table migration (see ozone iceberg rewrite-path --help for tool options)" + fi ozone_generate_usage "${OZONE_SHELL_EXECNAME}" false } @@ -252,6 +264,21 @@ function ozonecmd_case OZONE_VAPOR_OPTS="${OZONE_VAPOR_OPTS} ${RATIS_OPTS} ${OZONE_MODULE_ACCESS_ARGS}" OZONE_RUN_ARTIFACT_NAME="ozone-vapor" ;; + iceberg) + if ! ozone_iceberg_available; then + ozone_error "ERROR: ozone iceberg is not available in this distribution (requires JDK 11+ build)." + exit 1 + fi + if [[ $# -eq 0 ]] || [[ "$1" != "rewrite-path" ]]; then + echo "Usage: ozone iceberg rewrite-path [OPTIONS]" >&2 + exit 1 + fi + shift + OZONE_CLASSNAME="org.apache.hadoop.ozone.iceberg.RewriteTablePathCommand" + OZONE_RUN_ARTIFACT_NAME="ozone-iceberg" + OZONE_SUBCMD_SUPPORTDAEMONIZATION=false + OZONE_SUBCMD_ARGS=("$@") + ;; *) OZONE_CLASSNAME="${subcmd}" if ! ozone_validate_classname "${OZONE_CLASSNAME}"; then @@ -288,7 +315,8 @@ function ozone_suppress_shell_log { if [[ "${OZONE_RUN_ARTIFACT_NAME}" =~ ozone-cli-.* ]] \ || [[ "${OZONE_RUN_ARTIFACT_NAME}" == "ozone-dist" ]] \ - || [[ "${OZONE_RUN_ARTIFACT_NAME}" == "ozone-tools" ]]; then + || [[ "${OZONE_RUN_ARTIFACT_NAME}" == "ozone-tools" ]] \ + || [[ "${OZONE_RUN_ARTIFACT_NAME}" == "ozone-iceberg" ]]; then if [[ -z "${OZONE_ORIGINAL_LOGLEVEL}" ]] \ && [[ -z "${OZONE_ORIGINAL_ROOT_LOGGER}" ]]; then OZONE_LOGLEVEL=OFF diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone-iceberg b/hadoop-ozone/dist/src/shell/ozone/ozone-iceberg deleted file mode 100644 index 3f9988d738fe..000000000000 --- a/hadoop-ozone/dist/src/shell/ozone/ozone-iceberg +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Entry point for the Iceberg CLI -#it’s the standard Ozone way to turn ozone-iceberg JAR + main class into a runnable command. -OZONE_SHELL_EXECNAME="ozone-iceberg" -MYNAME="${BASH_SOURCE-$0}" -bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) -JVM_PID="$$" - -# load functions -for dir in "${OZONE_LIBEXEC_DIR}" "${OZONE_HOME}/libexec" "${HADOOP_LIBEXEC_DIR}" "${HADOOP_HOME}/libexec" "${bin}/../libexec"; do - if [[ -e "${dir}/ozone-functions.sh" ]]; then - # shellcheck source=ozone-functions.sh - . "${dir}/ozone-functions.sh" - if declare -F ozone_bootstrap >& /dev/null; then - break - fi - fi -done - -if ! declare -F ozone_bootstrap >& /dev/null; then - echo "ERROR: Cannot find ozone-functions.sh." 2>&1 - exit 1 -fi - -## @description Usage for ozone-iceberg (used by ozone-config.sh / ozone_exit_with_usage). -## @audience public -function ozone_usage -{ - ozone_reset_usage - - ozone_add_subcommand "rewrite-path" client "rewrite Iceberg table paths for table migration (see ${OZONE_SHELL_EXECNAME} rewrite-path --help for tool options)" - - ozone_generate_usage "${OZONE_SHELL_EXECNAME}" false -} - -ozone_bootstrap -# shellcheck source=ozone-config.sh -. "${OZONE_LIBEXEC_DIR}/ozone-config.sh" - -MYNAME=$(ozone_abs "${MYNAME}") - -if [[ $# -eq 0 ]] || [[ "$1" != "rewrite-path" ]]; then - echo "Usage: ozone-iceberg rewrite-path [OPTIONS]" >&2 - exit 1 -fi -shift - -OZONE_CLASSNAME="org.apache.hadoop.ozone.iceberg.RewriteTablePathCommand" -OZONE_RUN_ARTIFACT_NAME="ozone-iceberg" -OZONE_SUBCMD="iceberg" -OZONE_SUBCMD_SUPPORTDAEMONIZATION=false -OZONE_SUBCMD_ARGS=("$@") - -ozone_validate_classpath - -if [[ -z "${OZONE_ORIGINAL_LOGLEVEL}" ]] && [[ -z "${OZONE_ORIGINAL_ROOT_LOGGER}" ]]; then - OZONE_LOGLEVEL=OFF - OZONE_ROOT_LOGGER="${OZONE_LOGLEVEL},console" - OZONE_OPTS="${OZONE_OPTS} -Dslf4j.internal.verbosity=ERROR" -fi - -ozone_assemble_classpath - -ozone_add_client_opts -ozone_add_server_opts - -ozone_subcommand_opts "ozone" "iceberg" - -ozone_add_default_gc_opts - -ozone_generic_java_subcmd_handler diff --git a/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathCommand.java b/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathCommand.java index c318251936c2..de6f81ea83d2 100644 --- a/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathCommand.java +++ b/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathCommand.java @@ -26,8 +26,7 @@ import picocli.CommandLine.Option; /** - * CLI to rewrite Iceberg table paths. The {@code ozone-iceberg} shell script strips the - * {@code rewrite-path} token before invoking this class (same pattern as {@code ozone debug}). + * CLI to rewrite Iceberg table paths. */ @Command( name = "rewrite-path", From 580fd49a7ed01e14850057700931ed44e6a3df11 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Mon, 8 Jun 2026 18:40:25 +0530 Subject: [PATCH 4/5] Added parent command, removed INCLUDE_ICEBERG in dist-layout-stitching and made minor updates --- .../dev-support/bin/dist-layout-stitching | 14 ---- hadoop-ozone/dist/pom.xml | 6 -- hadoop-ozone/dist/src/shell/ozone/ozone | 10 +-- hadoop-ozone/iceberg/pom.xml | 4 ++ .../hadoop/ozone/iceberg/IcebergCommand.java | 42 +++++++++++ .../iceberg/RewriteTablePathCommand.java | 70 ++++++++----------- .../iceberg/RewriteTablePathOzoneAction.java | 10 --- .../TestRewriteTablePathOzoneAction.java | 30 ++++---- 8 files changed, 89 insertions(+), 97 deletions(-) create mode 100644 hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/IcebergCommand.java diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching index 20f29a8c5ec9..a15dea768ec8 100755 --- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching +++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching @@ -20,9 +20,6 @@ BASEDIR=$1 #hdds.version HDDS_VERSION=$2 -# true when ozone-iceberg is built (JDK 11+), false for JDK 8 builds -INCLUDE_ICEBERG=${3:-false} - ## @audience private ## @stability evolving function run() @@ -134,23 +131,12 @@ run mkdir compose/_keytabs for file in $(find "${ROOT}" -path '*/target/classes/*.classpath' | sort); do # We need to add the artifact manually as it's not part the generated classpath desciptor module=$(basename "${file%.classpath}") - if [[ "${INCLUDE_ICEBERG}" != "true" && "${module}" == "ozone-iceberg" ]]; then - continue - fi sed -i -e "s;$;:\$HDDS_LIB_JARS_DIR/${module}-${HDDS_VERSION}.jar;" "$file" run cp -p "$file" share/ozone/classpath/ done for file in $(find "${ROOT}" -path '*/share/ozone/lib/*jar' | sort); do - if [[ "${INCLUDE_ICEBERG}" != "true" ]]; then - jar_name=$(basename "${file}") - if [[ "${file}" == *"/hadoop-ozone/iceberg/target/"* ]] \ - || [[ "${jar_name}" == ozone-iceberg-*.jar ]] \ - || [[ "${jar_name}" == iceberg-*.jar ]]; then - continue - fi - fi # copy without printing to output due to large number of files cp -p "$file" share/ozone/lib/ done diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml index 176a57a51db8..108ca1deb69d 100644 --- a/hadoop-ozone/dist/pom.xml +++ b/hadoop-ozone/dist/pom.xml @@ -37,8 +37,6 @@ -rocky true UTF-8 - - false true @@ -197,7 +195,6 @@ ${basedir}/dev-support/bin/dist-layout-stitching ${project.build.directory} ${hdds.version} - ${include.iceberg} @@ -282,9 +279,6 @@ [11,) - - true - org.apache.ozone diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone b/hadoop-ozone/dist/src/shell/ozone/ozone index 9e663e7e77d3..a6c373976549 100755 --- a/hadoop-ozone/dist/src/shell/ozone/ozone +++ b/hadoop-ozone/dist/src/shell/ozone/ozone @@ -76,7 +76,7 @@ function ozone_usage ozone_add_subcommand "ratis" client "Ozone ratis tool" ozone_add_subcommand "vapor" client "Ozone server simulator" if ozone_iceberg_available; then - ozone_add_subcommand "iceberg" client "rewrite Iceberg table paths for table migration (see ozone iceberg rewrite-path --help for tool options)" + ozone_add_subcommand "iceberg" client "commands for Iceberg tables on Ozone (see ozone iceberg --help for subcommands)" fi ozone_generate_usage "${OZONE_SHELL_EXECNAME}" false } @@ -269,15 +269,9 @@ function ozonecmd_case ozone_error "ERROR: ozone iceberg is not available in this distribution (requires JDK 11+ build)." exit 1 fi - if [[ $# -eq 0 ]] || [[ "$1" != "rewrite-path" ]]; then - echo "Usage: ozone iceberg rewrite-path [OPTIONS]" >&2 - exit 1 - fi - shift - OZONE_CLASSNAME="org.apache.hadoop.ozone.iceberg.RewriteTablePathCommand" + OZONE_CLASSNAME="org.apache.hadoop.ozone.iceberg.IcebergCommand" OZONE_RUN_ARTIFACT_NAME="ozone-iceberg" OZONE_SUBCMD_SUPPORTDAEMONIZATION=false - OZONE_SUBCMD_ARGS=("$@") ;; *) OZONE_CLASSNAME="${subcmd}" diff --git a/hadoop-ozone/iceberg/pom.xml b/hadoop-ozone/iceberg/pom.xml index 56304a19d806..eea22a04d33b 100644 --- a/hadoop-ozone/iceberg/pom.xml +++ b/hadoop-ozone/iceberg/pom.xml @@ -110,6 +110,10 @@ + + org.apache.ozone + hdds-cli-common + org.apache.ozone hdds-common diff --git a/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/IcebergCommand.java b/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/IcebergCommand.java new file mode 100644 index 000000000000..af498c07ed97 --- /dev/null +++ b/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/IcebergCommand.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.iceberg; + +import org.apache.hadoop.hdds.cli.GenericCli; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import picocli.CommandLine.Command; + +/** + * Parent command for Iceberg tables on Ozone. + */ +@Command( + name = "ozone iceberg", + aliases = "iceberg", + description = "commands for Iceberg tables on Ozone", + subcommands = { + RewriteTablePathCommand.class + }, + versionProvider = HddsVersionProvider.class, + mixinStandardHelpOptions = true +) +public class IcebergCommand extends GenericCli { + + public static void main(String[] args) { + new IcebergCommand().run(args); + } +} diff --git a/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathCommand.java b/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathCommand.java index de6f81ea83d2..c8e972750f78 100644 --- a/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathCommand.java +++ b/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathCommand.java @@ -17,11 +17,11 @@ package org.apache.hadoop.ozone.iceberg; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import java.util.concurrent.Callable; +import org.apache.hadoop.hdds.cli.AbstractSubcommand; import org.apache.iceberg.Table; import org.apache.iceberg.actions.RewriteTablePath; import org.apache.iceberg.hadoop.HadoopTables; -import picocli.CommandLine; import picocli.CommandLine.Command; import picocli.CommandLine.Option; @@ -30,10 +30,9 @@ */ @Command( name = "rewrite-path", - description = "Rewrite Iceberg table paths for table migration", - mixinStandardHelpOptions = true + description = "Rewrite Iceberg table paths for table migration" ) -public class RewriteTablePathCommand implements Runnable { +public class RewriteTablePathCommand extends AbstractSubcommand implements Callable { @Option( names = {"-l", "--table-location"}, @@ -77,65 +76,52 @@ public class RewriteTablePathCommand implements Runnable { @Option( names = {"--threads"}, + defaultValue = "10", description = "Number of threads to use (positive integer). " - + "If omitted or zero, the default thread count is used." + + "If omitted or zero, the default thread count 10 is used." ) private int threads; @Override - public void run() { - System.out.println("Starting Iceberg table path rewrite"); - System.out.println("Table location: " + tableLocation); - System.out.println("Source prefix: " + sourcePrefix); - System.out.println("Target prefix: " + targetPrefix); - - OzoneConfiguration conf = new OzoneConfiguration(); - - Table table = null; - if (tableLocation != null && !tableLocation.isBlank()) { - HadoopTables tables = new HadoopTables(conf); - table = tables.load(tableLocation.trim()); - System.out.println("Table loaded: " + table.location()); - } + public Void call() { + out().println("Starting Iceberg table path rewrite"); + out().println("Table location: " + tableLocation); + out().println("Source prefix: " + sourcePrefix); + out().println("Target prefix: " + targetPrefix); - RewriteTablePathOzoneAction action; - if (threads > 0) { - action = new RewriteTablePathOzoneAction(table, threads); - } else { - action = new RewriteTablePathOzoneAction(table); - } - System.out.println("Threads: " + action.getThreads()); + HadoopTables tables = new HadoopTables(getOzoneConf()); + Table table = tables.load(tableLocation.trim()); + out().println("Table loaded: " + table.location()); + + RewriteTablePathOzoneAction action = new RewriteTablePathOzoneAction(table, threads); + out().println("Threads: " + threads); RewriteTablePath rewriteAction = action.rewriteLocationPrefix(sourcePrefix, targetPrefix); if (stagingLocation != null && !stagingLocation.isBlank()) { - System.out.println("Staging location: " + stagingLocation); + out().println("Staging location: " + stagingLocation); rewriteAction.stagingLocation(stagingLocation); } if (startVersion != null && !startVersion.isBlank()) { - System.out.println("Start version: " + startVersion); + out().println("Start version: " + startVersion); rewriteAction.startVersion(startVersion); } if (endVersion != null && !endVersion.isBlank()) { - System.out.println("End version: " + endVersion); + out().println("End version: " + endVersion); rewriteAction.endVersion(endVersion); } RewriteTablePath.Result result = rewriteAction.execute(); - System.out.println(); - System.out.println("Rewrite completed successfully"); - System.out.println(" Latest version: " + result.latestVersion()); - System.out.println(" Staging location: " + result.stagingLocation()); - System.out.println(); - System.out.println("Next step: Copy files from source to target using the file list"); - System.out.println(" File list location: " + result.fileListLocation()); - } - - public static void main(String[] args) { - int exitCode = new CommandLine(new RewriteTablePathCommand()).execute(args); - System.exit(exitCode); + out().println(); + out().println("Rewrite completed successfully"); + out().println(" Latest version: " + result.latestVersion()); + out().println(" Staging location: " + result.stagingLocation()); + out().println(); + out().println("Next step: Copy files from source to target using the file list"); + out().println(" File list location: " + result.fileListLocation()); + return null; } } diff --git a/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathOzoneAction.java b/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathOzoneAction.java index 2ecb5e4e340a..4a025b6e935e 100644 --- a/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathOzoneAction.java +++ b/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathOzoneAction.java @@ -100,24 +100,14 @@ public class RewriteTablePathOzoneAction implements RewriteTablePath { private ExecutorService executorService; private static final int MAX_INFLIGHT_MULTIPLIER = 4; - private static final int DEFAULT_THREAD_COUNT = 10; private final Table table; - public RewriteTablePathOzoneAction(Table table) { - this.table = table; - this.threads = DEFAULT_THREAD_COUNT; - } - public RewriteTablePathOzoneAction(Table table, int threads) { this.table = table; this.threads = threads; } - int getThreads() { - return threads; - } - @Override public RewriteTablePath rewriteLocationPrefix(String sPrefix, String tPrefix) { RewriteTablePathOzoneUtils.checkNonNullNonEmpty(sPrefix, "Source prefix"); diff --git a/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java b/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java index bd5bee9eae1e..514ec338ec8c 100644 --- a/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java +++ b/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java @@ -88,7 +88,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.mockito.Mockito; -import picocli.CommandLine; /** * Testing path rewrite of iceberg table metadata files. @@ -251,7 +250,7 @@ void tablePathRewriteForStartAndEndVersionProvided() throws Exception { @Test void executeRejectsMissingLocationPrefix() { NullPointerException exception = assertThrows(NullPointerException.class, - () -> new RewriteTablePathOzoneAction(table) + () -> new RewriteTablePathOzoneAction(table, 2) .stagingLocation(stagingDir.toString() + "/") .execute()); @@ -261,7 +260,7 @@ void executeRejectsMissingLocationPrefix() { @Test void executeRejectsMissingTargetPrefix() { NullPointerException exception = assertThrows(NullPointerException.class, - () -> new RewriteTablePathOzoneAction(table) + () -> new RewriteTablePathOzoneAction(table, 2) .rewriteLocationPrefix(sourcePrefix, null)); assertEquals("Target prefix is null", exception.getMessage()); @@ -270,7 +269,7 @@ void executeRejectsMissingTargetPrefix() { @Test void rewriteLocationPrefixRejectsSameSourceAndTarget() { IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, - () -> new RewriteTablePathOzoneAction(table) + () -> new RewriteTablePathOzoneAction(table, 2) .rewriteLocationPrefix(sourcePrefix, sourcePrefix) .execute()); @@ -281,7 +280,7 @@ void rewriteLocationPrefixRejectsSameSourceAndTarget() { @Test void startVersionRejectsUnknownVersion() { IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, - () -> new RewriteTablePathOzoneAction(table) + () -> new RewriteTablePathOzoneAction(table, 2) .rewriteLocationPrefix(sourcePrefix, targetPrefix) .startVersion("missing.metadata.json") .execute()); @@ -297,7 +296,7 @@ void startVersionRejectsDeletedVersionFile() { table.io().deleteFile(metadataPaths.get(0)); IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, - () -> new RewriteTablePathOzoneAction(table) + () -> new RewriteTablePathOzoneAction(table, 2) .rewriteLocationPrefix(sourcePrefix, targetPrefix) .startVersion(existingName) .execute()); @@ -308,7 +307,7 @@ void startVersionRejectsDeletedVersionFile() { @Test void endVersionRejectsUnknownVersion() { IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, - () -> new RewriteTablePathOzoneAction(table) + () -> new RewriteTablePathOzoneAction(table, 2) .rewriteLocationPrefix(sourcePrefix, targetPrefix) .endVersion("missing.metadata.json") .execute()); @@ -324,7 +323,7 @@ void endVersionRejectsDeletedVersionFile() { table.io().deleteFile(metadataPaths.get(0)); IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, - () -> new RewriteTablePathOzoneAction(table) + () -> new RewriteTablePathOzoneAction(table, 2) .rewriteLocationPrefix(sourcePrefix, targetPrefix) .endVersion(existingName) .execute()); @@ -335,7 +334,7 @@ void endVersionRejectsDeletedVersionFile() { @Test void usesCurrentMetadataIfEndVersionNotProvided() { String currentMetadata = ((HasTableOperations) table).operations().current().metadataFileLocation(); - RewriteTablePathOzoneAction action = new RewriteTablePathOzoneAction(table); + RewriteTablePathOzoneAction action = new RewriteTablePathOzoneAction(table, 2); action.rewriteLocationPrefix(sourcePrefix, targetPrefix).stagingLocation(stagingDir + "/"); RewriteTablePath.Result result = action.execute(); assertThat(result.latestVersion()).isEqualTo(RewriteTablePathUtil.fileName(currentMetadata)); @@ -344,7 +343,7 @@ void usesCurrentMetadataIfEndVersionNotProvided() { @Test void defaultStagingDirIsUnderTableMetadataLocation() { String metadataLocation = RewriteTablePathOzoneUtils.getMetadataLocation(table); - RewriteTablePath.Result result = new RewriteTablePathOzoneAction(table) + RewriteTablePath.Result result = new RewriteTablePathOzoneAction(table, 2) .rewriteLocationPrefix(sourcePrefix, targetPrefix) .execute(); @@ -418,7 +417,7 @@ void rejectsTablesWithPartitionStatistics() { TableOperations ops = ((HasTableOperations) table).operations(); ops.commit(baseMetadata, metadataWithStats); - RewriteTablePath action = new RewriteTablePathOzoneAction(table) + RewriteTablePath action = new RewriteTablePathOzoneAction(table, 2) .rewriteLocationPrefix(sourcePrefix, targetPrefix) .stagingLocation(stagingDir + "/"); @@ -497,7 +496,7 @@ void manifestsToRewriteRejectsMissingManifestList() { String manifestListLocation = snapshot.manifestListLocation(); table.io().deleteFile(manifestListLocation); - RewriteTablePath action = new RewriteTablePathOzoneAction(table) + RewriteTablePath action = new RewriteTablePathOzoneAction(table, 2) .rewriteLocationPrefix(sourcePrefix, targetPrefix) .stagingLocation(stagingDir + "/"); @@ -506,13 +505,10 @@ void manifestsToRewriteRejectsMissingManifestList() { assertThat(exception.getCause()).hasMessageContaining("Failed to read manifests for snapshot " + snapshot.snapshotId()); } - - private CommandLine newRewritePathCommand() { - return new CommandLine(new RewriteTablePathCommand()); - } private String executeRewriteCommand(String... optionalArgs) { List args = new ArrayList<>(); + args.add("rewrite-path"); args.add("-l"); args.add(table.location()); args.add("-s"); @@ -523,7 +519,7 @@ private String executeRewriteCommand(String... optionalArgs) { args.add(stagingDir + "/"); args.addAll(Arrays.asList(optionalArgs)); - int exitCode = newRewritePathCommand().execute(args.toArray(new String[0])); + int exitCode = new IcebergCommand().getCmd().execute(args.toArray(new String[0])); assertEquals(0, exitCode, "Command failed.\nstdout:\n" + stdout() + "\nstderr:\n" + stderr()); assertThat(stdout()) From cf04824dd5f0d174b3d0b9497b9d303a7795519b Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Tue, 9 Jun 2026 10:44:23 +0530 Subject: [PATCH 5/5] Removed unwanted edit --- hadoop-ozone/dist/dev-support/bin/dist-layout-stitching | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching index a15dea768ec8..17723b208cf6 100755 --- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching +++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching @@ -97,6 +97,7 @@ run cp -r "${ROOT}/hadoop-ozone/dist/src/main/dockerlibexec/." "libexec/" run cp "${ROOT}/hadoop-ozone/dist/src/shell/ozone/ozone" "bin/" + run cp "${ROOT}/hadoop-ozone/dist/src/shell/ozone/ozone-config.sh" "libexec/" run cp "${ROOT}/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh" "libexec/" run cp -r "${ROOT}/hadoop-ozone/dist/src/shell/shellprofile.d" "libexec/"