From a2721f73812ab4d0042b3ddf11bfd8ab2afdaf0c Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Thu, 30 Apr 2026 18:50:30 -0700 Subject: [PATCH 1/2] HDDS-15155. Add `ozone debug ldb drop_column_family` subcommand Generated-by: GPT-5.5 --- hadoop-hdds/docs/content/tools/debug/Ldb.md | 12 + .../ozone/debug/ldb/DropColumnFamily.java | 147 +++++++++++ .../hadoop/ozone/debug/ldb/RDBParser.java | 1 + .../ozone/debug/ldb/TestDropColumnFamily.java | 230 ++++++++++++++++++ .../TestLDBDropColumnFamilyWithSnapshots.java | 157 ++++++++++++ 5 files changed, 547 insertions(+) create mode 100644 hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/ldb/DropColumnFamily.java create mode 100644 hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/ldb/TestDropColumnFamily.java create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBDropColumnFamilyWithSnapshots.java diff --git a/hadoop-hdds/docs/content/tools/debug/Ldb.md b/hadoop-hdds/docs/content/tools/debug/Ldb.md index a6190b6c6d7a..fbcbf35e3d93 100644 --- a/hadoop-hdds/docs/content/tools/debug/Ldb.md +++ b/hadoop-hdds/docs/content/tools/debug/Ldb.md @@ -35,6 +35,7 @@ Commands: list_column_families, ls list all column families in db. value-schema Schema of value in metadataTable checkpoint Create checkpoint for specified db + drop_column_family Drop a column family from the DB. ``` ### list_column_families command @@ -60,6 +61,17 @@ revokedCerts move ``` +### drop_column_family command + +`drop_column_family` command drops a column family from the db provided. The service that owns the DB should be stopped +before running this command. The command prompts for confirmation by default; use `-y` or `--yes` to skip the prompt. + +```bash +$ ozone debug ldb --db=/path/to/om.db drop_column_family --cf=snapshotInfoTable +WARNING: Dropping a column family mutates RocksDB and can be unsafe if the DB is already open by an Ozone process. Ensure the service that owns this DB is stopped before running this command. Do you want to continue (y/N)? y +Dropped column family snapshotInfoTable +``` + ### scan command `scan` command parses a particular column family of a rocksdb provided and prints the records. diff --git a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/ldb/DropColumnFamily.java b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/ldb/DropColumnFamily.java new file mode 100644 index 000000000000..b6ff027ac548 --- /dev/null +++ b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/ldb/DropColumnFamily.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.debug.ldb; + +import static org.apache.hadoop.hdds.utils.db.DBStoreBuilder.DEFAULT_COLUMN_FAMILY_NAME; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Scanner; +import java.util.concurrent.Callable; +import org.apache.hadoop.hdds.cli.AbstractSubcommand; +import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.hdds.utils.db.RocksDatabase; +import org.apache.hadoop.hdds.utils.db.managed.ManagedConfigOptions; +import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; +import org.apache.hadoop.ozone.debug.RocksDBUtils; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.RocksDBException; +import picocli.CommandLine; + +/** + * Drop a column family from a RocksDB. + */ +@CommandLine.Command( + name = "drop_column_family", + aliases = "drop-column-family", + description = "Drop a column family from the DB.") +public class DropColumnFamily extends AbstractSubcommand + implements Callable { + + private static final String WARNING_TO_STOP_SERVICE = + "WARNING: Dropping a column family mutates RocksDB and can be unsafe " + + "if the DB is already open by an Ozone process. Ensure the service " + + "that owns this DB is stopped before running this command. " + + "Do you want to continue (y/N)? "; + + @CommandLine.ParentCommand + private RDBParser parent; + + @CommandLine.Option(names = {"--column_family", "--column-family", "--cf"}, + required = true, + description = "Column family name") + private String columnFamilyName; + + @CommandLine.Option(names = {"-y", "--yes"}, + description = "Continue without interactive user confirmation") + private boolean yes; + + @Override + public Void call() throws Exception { + if (isDefaultColumnFamily(columnFamilyName)) { + throw new IllegalArgumentException( + "Default column family cannot be dropped."); + } + if (!columnFamilyExists()) { + throw new IllegalArgumentException(columnFamilyName + + " is not a column family in DB for the given path."); + } + + confirm(); + + ManagedConfigOptions configOptions = new ManagedConfigOptions(); + ManagedDBOptions dbOptions = new ManagedDBOptions(); + List cfHandleList = new ArrayList<>(); + List cfDescList = new ArrayList<>(); + + try (ManagedRocksDB db = ManagedRocksDB.openWithLatestOptions( + configOptions, dbOptions, parent.getDbPath(), cfDescList, + cfHandleList)) { + ColumnFamilyHandle columnFamilyHandle = + RocksDBUtils.getColumnFamilyHandle(columnFamilyName, cfHandleList); + if (columnFamilyHandle == null) { + throw new IllegalArgumentException(columnFamilyName + + " is not a column family in DB for the given path."); + } + + db.get().dropColumnFamily(columnFamilyHandle); + out().println("Dropped column family " + columnFamilyName); + } catch (RocksDBException exception) { + String errorMessage = "Failed to drop column family " + columnFamilyName + + " from RocksDB for the given path: " + parent.getDbPath(); + throw new IOException(errorMessage, exception); + } finally { + IOUtils.closeQuietly(cfHandleList); + closeDescriptors(cfDescList); + IOUtils.closeQuietly(configOptions, dbOptions); + } + + return null; + } + + private static boolean isDefaultColumnFamily(String name) { + return DEFAULT_COLUMN_FAMILY_NAME.equals(name); + } + + private void confirm() { + if (yes) { + return; + } + + err().print(WARNING_TO_STOP_SERVICE); + err().flush(); + Scanner scanner = new Scanner(new InputStreamReader( + System.in, StandardCharsets.UTF_8)); + String confirmation = scanner.hasNextLine() + ? scanner.nextLine().trim() : ""; + if (!"y".equalsIgnoreCase(confirmation)) { + throw new IllegalStateException("Aborting command."); + } + } + + private boolean columnFamilyExists() throws RocksDBException { + for (byte[] columnFamily : RocksDatabase.listColumnFamiliesEmptyOptions( + parent.getDbPath())) { + if (columnFamilyName.equals(new String(columnFamily, + StandardCharsets.UTF_8))) { + return true; + } + } + return false; + } + + private static void closeDescriptors( + List descriptors) { + descriptors.forEach(descriptor -> descriptor.getOptions().close()); + } +} diff --git a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/ldb/RDBParser.java b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/ldb/RDBParser.java index e2f1722ea01e..528a2e41f465 100644 --- a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/ldb/RDBParser.java +++ b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/ldb/RDBParser.java @@ -31,6 +31,7 @@ ListTables.class, ValueSchema.class, Checkpoint.class, + DropColumnFamily.class, }, description = "Parse rocksdb file content") @MetaInfServices(DebugSubcommand.class) diff --git a/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/ldb/TestDropColumnFamily.java b/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/ldb/TestDropColumnFamily.java new file mode 100644 index 000000000000..e02bca42191b --- /dev/null +++ b/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/ldb/TestDropColumnFamily.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.debug.ldb; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.ozone.test.IntLambda.withTextFromSystemIn; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; +import org.apache.hadoop.hdds.utils.db.RocksDatabase; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import picocli.CommandLine; + +/** + * Tests for the ozone debug ldb drop_column_family command. + */ +public class TestDropColumnFamily { + + private static final String KEY_TABLE = "keyTable"; + private static final String PIPELINES = "pipelines"; + + @TempDir + private File tempDir; + + private DBStore dbStore; + private StringWriter stdout; + private StringWriter stderr; + private PrintWriter pstdout; + private PrintWriter pstderr; + private CommandLine cmd; + + @BeforeEach + public void setup() { + stdout = new StringWriter(); + stderr = new StringWriter(); + pstdout = new PrintWriter(stdout); + pstderr = new PrintWriter(stderr); + cmd = new CommandLine(new RDBParser()) + .setOut(pstdout) + .setErr(pstderr); + } + + @AfterEach + public void teardown() throws IOException { + if (dbStore != null) { + dbStore.close(); + } + pstderr.close(); + stderr.close(); + pstdout.close(); + stdout.close(); + } + + @Test + public void dropsColumnFamily() throws Exception { + File dbLocation = createDb(KEY_TABLE, PIPELINES); + + int exitCode = cmd.execute( + "--db", dbLocation.getAbsolutePath(), + "drop_column_family", + "--column-family", KEY_TABLE, + "-y"); + + assertEquals(0, exitCode, stderr.toString()); + assertThat(stderr.toString()).doesNotContain("WARNING:"); + assertThat(stdout.toString()) + .contains("Dropped column family " + KEY_TABLE); + assertThat(listColumnFamilies(dbLocation)) + .contains("default", PIPELINES) + .doesNotContain(KEY_TABLE); + } + + @Test + public void failsForMissingColumnFamily() throws Exception { + File dbLocation = createDb(KEY_TABLE); + + int exitCode = cmd.execute( + "--db", dbLocation.getAbsolutePath(), + "drop_column_family", + "--column-family", PIPELINES, + "-y"); + + assertEquals(1, exitCode); + assertThat(stderr.toString()) + .contains(PIPELINES + + " is not a column family in DB for the given path."); + assertThat(listColumnFamilies(dbLocation)) + .contains("default", KEY_TABLE); + } + + @Test + public void failsForDefaultColumnFamily() throws Exception { + File dbLocation = createDb(KEY_TABLE); + + int exitCode = cmd.execute( + "--db", dbLocation.getAbsolutePath(), + "drop_column_family", + "--column-family", "default"); + + assertEquals(1, exitCode); + assertThat(stderr.toString()) + .contains("Default column family cannot be dropped."); + assertThat(listColumnFamilies(dbLocation)) + .contains("default", KEY_TABLE); + } + + @Test + public void dropsColumnFamilyAfterConfirmation() throws Exception { + File dbLocation = createDb(KEY_TABLE, PIPELINES); + + int exitCode = withTextFromSystemIn("y") + .execute(() -> cmd.execute( + "--db", dbLocation.getAbsolutePath(), + "drop_column_family", + "--column-family", KEY_TABLE)); + + assertEquals(0, exitCode, stderr.toString()); + assertThat(stderr.toString()) + .contains("WARNING: Dropping a column family mutates RocksDB"); + assertThat(listColumnFamilies(dbLocation)) + .contains("default", PIPELINES) + .doesNotContain(KEY_TABLE); + } + + @Test + public void abortsDropColumnFamilyByDefault() throws Exception { + File dbLocation = createDb(KEY_TABLE, PIPELINES); + + int exitCode = withTextFromSystemIn("n") + .execute(() -> cmd.execute( + "--db", dbLocation.getAbsolutePath(), + "drop_column_family", + "--column-family", KEY_TABLE)); + + assertEquals(1, exitCode); + assertThat(stderr.toString()) + .contains("WARNING: Dropping a column family mutates RocksDB") + .contains("Aborting command."); + assertThat(listColumnFamilies(dbLocation)) + .contains("default", KEY_TABLE, PIPELINES); + } + + @Test + public void scanCountWorksWhenDbIsOpen() throws Exception { + File dbLocation = createOpenDb(KEY_TABLE); + + int exitCode = cmd.execute( + "--db", dbLocation.getAbsolutePath(), + "scan", + "--column-family", KEY_TABLE, + "--count"); + + assertEquals(0, exitCode, stderr.toString()); + assertThat(stdout.toString().trim()).isNotEmpty(); + } + + @Test + public void dropColumnFamilyFailsWhenDbIsOpen() throws Exception { + File dbLocation = createOpenDb(KEY_TABLE, PIPELINES); + + int exitCode = cmd.execute( + "--db", dbLocation.getAbsolutePath(), + "drop_column_family", + "--column-family", KEY_TABLE, + "-y"); + + assertEquals(1, exitCode); + assertThat(stderr.toString()) + .contains("Failed to drop column family " + KEY_TABLE); + assertThat(listColumnFamilies(dbLocation)) + .contains("default", KEY_TABLE, PIPELINES); + } + + private File createDb(String... columnFamilies) throws IOException { + File dbLocation = createOpenDb(columnFamilies); + dbStore.close(); + dbStore = null; + return dbLocation; + } + + private File createOpenDb(String... columnFamilies) throws IOException { + DBStoreBuilder builder = DBStoreBuilder.newBuilder(new OzoneConfiguration()) + .setName("om.db") + .setPath(tempDir.toPath()); + for (String columnFamily : columnFamilies) { + builder.addTable(columnFamily); + } + + dbStore = builder.build(); + return dbStore.getDbLocation(); + } + + private static List listColumnFamilies(File dbLocation) + throws Exception { + List columnFamilies = new ArrayList<>(); + for (byte[] columnFamily + : RocksDatabase.listColumnFamiliesEmptyOptions( + dbLocation.getAbsolutePath())) { + columnFamilies.add(new String(columnFamily, UTF_8)); + } + return columnFamilies; + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBDropColumnFamilyWithSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBDropColumnFamilyWithSnapshots.java new file mode 100644 index 000000000000..096d9fe2310e --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBDropColumnFamilyWithSnapshots.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.debug; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE; +import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE; +import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.SNAPSHOT_INFO_TABLE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.io.File; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.db.DBProfile; +import org.apache.hadoop.hdds.utils.db.RocksDatabase; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.TestDataUtil; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.debug.ldb.RDBParser; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.ozone.test.GenericTestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import picocli.CommandLine; + +/** + * Integration tests for dropping Ozone column families with ozone debug ldb. + */ +public class TestLDBDropColumnFamilyWithSnapshots { + + private MiniOzoneCluster cluster; + private OzoneClient client; + + @AfterEach + public void teardown() { + IOUtils.closeQuietly(client); + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + public void dropsOrganicallyPopulatedSnapshotInfoTable() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.setEnum(HDDS_DB_PROFILE, DBProfile.TEST); + conf.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true); + + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(1) + .build(); + cluster.waitForClusterToBeReady(); + client = cluster.newClient(); + + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String snapshotName1 = "snap1"; + String snapshotName2 = "snap2"; + ObjectStore objectStore = client.getObjectStore(); + TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName, + BucketLayout.DEFAULT); + + objectStore.createSnapshot(volumeName, bucketName, snapshotName1); + objectStore.createSnapshot(volumeName, bucketName, snapshotName2); + + assertSnapshotInfoExists(volumeName, bucketName, snapshotName1); + assertSnapshotInfoExists(volumeName, bucketName, snapshotName2); + GenericTestUtils.waitFor(() -> hasSnapshotInfoRowCount(2), + 100, 10_000); + assertEquals(2, countSnapshotInfoRows()); + + File omDbPath = cluster.getOzoneManager().getMetadataManager() + .getStore().getDbLocation(); + assertThat(listColumnFamilies(omDbPath)).contains(SNAPSHOT_INFO_TABLE); + + IOUtils.closeQuietly(client); + client = null; + cluster.stop(); + + StringWriter stdout = new StringWriter(); + StringWriter stderr = new StringWriter(); + CommandLine cmd = new CommandLine(new RDBParser()) + .setOut(new PrintWriter(stdout)) + .setErr(new PrintWriter(stderr)); + + int exitCode = cmd.execute( + "--db", omDbPath.getAbsolutePath(), + "drop_column_family", + "--cf", SNAPSHOT_INFO_TABLE, + "-y"); + + assertEquals(0, exitCode, stderr.toString()); + assertThat(stdout.toString()) + .contains("Dropped column family " + SNAPSHOT_INFO_TABLE); + assertThat(listColumnFamilies(omDbPath)) + .contains(KEY_TABLE) + .doesNotContain(SNAPSHOT_INFO_TABLE); + } + + private void assertSnapshotInfoExists(String volumeName, String bucketName, + String snapshotName) throws Exception { + SnapshotInfo snapshotInfo = cluster.getOzoneManager() + .getMetadataManager() + .getSnapshotInfoTable() + .get(SnapshotInfo.getTableKey(volumeName, bucketName, snapshotName)); + assertNotNull(snapshotInfo); + } + + private boolean hasSnapshotInfoRowCount(long expected) { + try { + return countSnapshotInfoRows() == expected; + } catch (Exception ex) { + throw new AssertionError(ex); + } + } + + private long countSnapshotInfoRows() throws Exception { + return cluster.getOzoneManager().getMetadataManager() + .countRowsInTable(cluster.getOzoneManager().getMetadataManager() + .getSnapshotInfoTable()); + } + + private static List listColumnFamilies(File dbLocation) + throws Exception { + List columnFamilies = new ArrayList<>(); + for (byte[] columnFamily + : RocksDatabase.listColumnFamiliesEmptyOptions( + dbLocation.getAbsolutePath())) { + columnFamilies.add(new String(columnFamily, UTF_8)); + } + return columnFamilies; + } +} From 7a05757befa5746d15f51d313a7db91c30d8fedd Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Fri, 1 May 2026 03:35:15 -0700 Subject: [PATCH 2/2] Address comments --- hadoop-hdds/docs/content/tools/debug/Ldb.md | 8 +- .../ozone/debug/ldb/DropColumnFamily.java | 5 +- .../ozone/debug/ldb/TestDropColumnFamily.java | 14 +- .../TestLDBDropColumnFamilyWithSnapshots.java | 157 ------------------ .../ozone/shell/TestOzoneDebugShell.java | 85 ++++++++++ 5 files changed, 98 insertions(+), 171 deletions(-) delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBDropColumnFamilyWithSnapshots.java diff --git a/hadoop-hdds/docs/content/tools/debug/Ldb.md b/hadoop-hdds/docs/content/tools/debug/Ldb.md index fbcbf35e3d93..a993821396fb 100644 --- a/hadoop-hdds/docs/content/tools/debug/Ldb.md +++ b/hadoop-hdds/docs/content/tools/debug/Ldb.md @@ -35,7 +35,7 @@ Commands: list_column_families, ls list all column families in db. value-schema Schema of value in metadataTable checkpoint Create checkpoint for specified db - drop_column_family Drop a column family from the DB. + drop-column-family Drop a column family from the DB. ``` ### list_column_families command @@ -61,13 +61,13 @@ revokedCerts move ``` -### drop_column_family command +### drop-column-family command -`drop_column_family` command drops a column family from the db provided. The service that owns the DB should be stopped +`drop-column-family` command drops a column family from the db provided. The service that owns the DB should be stopped before running this command. The command prompts for confirmation by default; use `-y` or `--yes` to skip the prompt. ```bash -$ ozone debug ldb --db=/path/to/om.db drop_column_family --cf=snapshotInfoTable +$ ozone debug ldb --db=/path/to/om.db drop-column-family --cf=snapshotInfoTable WARNING: Dropping a column family mutates RocksDB and can be unsafe if the DB is already open by an Ozone process. Ensure the service that owns this DB is stopped before running this command. Do you want to continue (y/N)? y Dropped column family snapshotInfoTable ``` diff --git a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/ldb/DropColumnFamily.java b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/ldb/DropColumnFamily.java index b6ff027ac548..5f37df5e85f8 100644 --- a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/ldb/DropColumnFamily.java +++ b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/ldb/DropColumnFamily.java @@ -42,8 +42,7 @@ * Drop a column family from a RocksDB. */ @CommandLine.Command( - name = "drop_column_family", - aliases = "drop-column-family", + name = "drop-column-family", description = "Drop a column family from the DB.") public class DropColumnFamily extends AbstractSubcommand implements Callable { @@ -57,7 +56,7 @@ public class DropColumnFamily extends AbstractSubcommand @CommandLine.ParentCommand private RDBParser parent; - @CommandLine.Option(names = {"--column_family", "--column-family", "--cf"}, + @CommandLine.Option(names = {"--column-family", "--cf"}, required = true, description = "Column family name") private String columnFamilyName; diff --git a/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/ldb/TestDropColumnFamily.java b/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/ldb/TestDropColumnFamily.java index e02bca42191b..fde1ca9dbab5 100644 --- a/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/ldb/TestDropColumnFamily.java +++ b/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/ldb/TestDropColumnFamily.java @@ -39,7 +39,7 @@ import picocli.CommandLine; /** - * Tests for the ozone debug ldb drop_column_family command. + * Tests for the ozone debug ldb drop-column-family command. */ public class TestDropColumnFamily { @@ -84,7 +84,7 @@ public void dropsColumnFamily() throws Exception { int exitCode = cmd.execute( "--db", dbLocation.getAbsolutePath(), - "drop_column_family", + "drop-column-family", "--column-family", KEY_TABLE, "-y"); @@ -103,7 +103,7 @@ public void failsForMissingColumnFamily() throws Exception { int exitCode = cmd.execute( "--db", dbLocation.getAbsolutePath(), - "drop_column_family", + "drop-column-family", "--column-family", PIPELINES, "-y"); @@ -121,7 +121,7 @@ public void failsForDefaultColumnFamily() throws Exception { int exitCode = cmd.execute( "--db", dbLocation.getAbsolutePath(), - "drop_column_family", + "drop-column-family", "--column-family", "default"); assertEquals(1, exitCode); @@ -138,7 +138,7 @@ public void dropsColumnFamilyAfterConfirmation() throws Exception { int exitCode = withTextFromSystemIn("y") .execute(() -> cmd.execute( "--db", dbLocation.getAbsolutePath(), - "drop_column_family", + "drop-column-family", "--column-family", KEY_TABLE)); assertEquals(0, exitCode, stderr.toString()); @@ -156,7 +156,7 @@ public void abortsDropColumnFamilyByDefault() throws Exception { int exitCode = withTextFromSystemIn("n") .execute(() -> cmd.execute( "--db", dbLocation.getAbsolutePath(), - "drop_column_family", + "drop-column-family", "--column-family", KEY_TABLE)); assertEquals(1, exitCode); @@ -187,7 +187,7 @@ public void dropColumnFamilyFailsWhenDbIsOpen() throws Exception { int exitCode = cmd.execute( "--db", dbLocation.getAbsolutePath(), - "drop_column_family", + "drop-column-family", "--column-family", KEY_TABLE, "-y"); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBDropColumnFamilyWithSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBDropColumnFamilyWithSnapshots.java deleted file mode 100644 index 096d9fe2310e..000000000000 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBDropColumnFamilyWithSnapshots.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.debug; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE; -import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE; -import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.SNAPSHOT_INFO_TABLE; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - -import java.io.File; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.utils.db.DBProfile; -import org.apache.hadoop.hdds.utils.db.RocksDatabase; -import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.TestDataUtil; -import org.apache.hadoop.ozone.client.ObjectStore; -import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.debug.ldb.RDBParser; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; -import org.apache.ozone.test.GenericTestUtils; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; -import picocli.CommandLine; - -/** - * Integration tests for dropping Ozone column families with ozone debug ldb. - */ -public class TestLDBDropColumnFamilyWithSnapshots { - - private MiniOzoneCluster cluster; - private OzoneClient client; - - @AfterEach - public void teardown() { - IOUtils.closeQuietly(client); - if (cluster != null) { - cluster.shutdown(); - } - } - - @Test - public void dropsOrganicallyPopulatedSnapshotInfoTable() throws Exception { - OzoneConfiguration conf = new OzoneConfiguration(); - conf.setEnum(HDDS_DB_PROFILE, DBProfile.TEST); - conf.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true); - - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(1) - .build(); - cluster.waitForClusterToBeReady(); - client = cluster.newClient(); - - String volumeName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); - String snapshotName1 = "snap1"; - String snapshotName2 = "snap2"; - ObjectStore objectStore = client.getObjectStore(); - TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName, - BucketLayout.DEFAULT); - - objectStore.createSnapshot(volumeName, bucketName, snapshotName1); - objectStore.createSnapshot(volumeName, bucketName, snapshotName2); - - assertSnapshotInfoExists(volumeName, bucketName, snapshotName1); - assertSnapshotInfoExists(volumeName, bucketName, snapshotName2); - GenericTestUtils.waitFor(() -> hasSnapshotInfoRowCount(2), - 100, 10_000); - assertEquals(2, countSnapshotInfoRows()); - - File omDbPath = cluster.getOzoneManager().getMetadataManager() - .getStore().getDbLocation(); - assertThat(listColumnFamilies(omDbPath)).contains(SNAPSHOT_INFO_TABLE); - - IOUtils.closeQuietly(client); - client = null; - cluster.stop(); - - StringWriter stdout = new StringWriter(); - StringWriter stderr = new StringWriter(); - CommandLine cmd = new CommandLine(new RDBParser()) - .setOut(new PrintWriter(stdout)) - .setErr(new PrintWriter(stderr)); - - int exitCode = cmd.execute( - "--db", omDbPath.getAbsolutePath(), - "drop_column_family", - "--cf", SNAPSHOT_INFO_TABLE, - "-y"); - - assertEquals(0, exitCode, stderr.toString()); - assertThat(stdout.toString()) - .contains("Dropped column family " + SNAPSHOT_INFO_TABLE); - assertThat(listColumnFamilies(omDbPath)) - .contains(KEY_TABLE) - .doesNotContain(SNAPSHOT_INFO_TABLE); - } - - private void assertSnapshotInfoExists(String volumeName, String bucketName, - String snapshotName) throws Exception { - SnapshotInfo snapshotInfo = cluster.getOzoneManager() - .getMetadataManager() - .getSnapshotInfoTable() - .get(SnapshotInfo.getTableKey(volumeName, bucketName, snapshotName)); - assertNotNull(snapshotInfo); - } - - private boolean hasSnapshotInfoRowCount(long expected) { - try { - return countSnapshotInfoRows() == expected; - } catch (Exception ex) { - throw new AssertionError(ex); - } - } - - private long countSnapshotInfoRows() throws Exception { - return cluster.getOzoneManager().getMetadataManager() - .countRowsInTable(cluster.getOzoneManager().getMetadataManager() - .getSnapshotInfoTable()); - } - - private static List listColumnFamilies(File dbLocation) - throws Exception { - List columnFamilies = new ArrayList<>(); - for (byte[] columnFamily - : RocksDatabase.listColumnFamiliesEmptyOptions( - dbLocation.getAbsolutePath())) { - columnFamilies.add(new String(columnFamily, UTF_8)); - } - return columnFamilies; - } -} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java index 14753394cfe3..7edee4c472cf 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java @@ -20,8 +20,11 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; +import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE; +import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.SNAPSHOT_INFO_TABLE; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -30,7 +33,9 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeoutException; @@ -43,6 +48,8 @@ import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.hdds.utils.db.DBCheckpoint; +import org.apache.hadoop.hdds.utils.db.RocksDatabase; import org.apache.hadoop.ozone.OzoneTestUtils; import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneClient; @@ -55,10 +62,12 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.NonHATests; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; @@ -172,6 +181,63 @@ public void testLdbCliForOzoneSnapshot(BucketLayout layout) throws Exception { assertThat(cmdOut).contains(keyName); } + @Test + public void testLdbDropColumnFamilyForSnapshotInfoTable() throws Exception { + StringWriter stdout = new StringWriter(); + StringWriter stderr = new StringWriter(); + CommandLine cmd = new CommandLine(new RDBParser()) + .setOut(new PrintWriter(stdout)) + .setErr(new PrintWriter(stderr)); + final String volumeName = UUID.randomUUID().toString(); + final String bucketName = UUID.randomUUID().toString(); + final String snapshotName1 = "snap1"; + final String snapshotName2 = "snap2"; + long initialSnapshotInfoRows = omMetadataManager.countRowsInTable( + omMetadataManager.getSnapshotInfoTable()); + + TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName, + BucketLayout.DEFAULT); + client.getObjectStore().createSnapshot(volumeName, bucketName, + snapshotName1); + client.getObjectStore().createSnapshot(volumeName, bucketName, + snapshotName2); + + assertSnapshotInfoExists(volumeName, bucketName, snapshotName1); + assertSnapshotInfoExists(volumeName, bucketName, snapshotName2); + // Wait until both organically created snapshots are visible in the table. + GenericTestUtils.waitFor(() -> { + try { + return omMetadataManager.countRowsInTable( + omMetadataManager.getSnapshotInfoTable()) + >= initialSnapshotInfoRows + 2; + } catch (Exception ex) { + throw new AssertionError(ex); + } + }, 100, 10_000); + + DBCheckpoint checkpoint = omMetadataManager.getStore() + .getCheckpoint(false); + try { + File dbPath = checkpoint.getCheckpointLocation().toFile(); + assertThat(listColumnFamilies(dbPath)).contains(SNAPSHOT_INFO_TABLE); + + int exitCode = cmd.execute( + "--db", dbPath.getAbsolutePath(), + "drop-column-family", + "--cf", SNAPSHOT_INFO_TABLE, + "-y"); + + assertEquals(0, exitCode, stderr.toString()); + assertThat(stdout.toString()) + .contains("Dropped column family " + SNAPSHOT_INFO_TABLE); + assertThat(listColumnFamilies(dbPath)) + .contains(KEY_TABLE) + .doesNotContain(SNAPSHOT_INFO_TABLE); + } finally { + checkpoint.cleanupCheckpoint(); + } + } + private String getSnapshotDBPath(String checkPointDir) { return OMStorage.getOmDbDir(cluster().getConf()) + OM_KEY_PREFIX + OM_SNAPSHOT_CHECKPOINT_DIR + OM_KEY_PREFIX + @@ -231,6 +297,25 @@ private int runChunkInfoAndVerifyPaths(String volumeName, String bucketName, return exitCode; } + private void assertSnapshotInfoExists(String volumeName, String bucketName, + String snapshotName) throws Exception { + SnapshotInfo snapshotInfo = omMetadataManager + .getSnapshotInfoTable() + .get(SnapshotInfo.getTableKey(volumeName, bucketName, snapshotName)); + assertNotNull(snapshotInfo); + } + + private static List listColumnFamilies(File dbLocation) + throws Exception { + List columnFamilies = new ArrayList<>(); + for (byte[] columnFamily + : RocksDatabase.listColumnFamiliesEmptyOptions( + dbLocation.getAbsolutePath())) { + columnFamilies.add(new String(columnFamily, StandardCharsets.UTF_8)); + } + return columnFamilies; + } + /** * Generate string to pass as extra arguments to the * ozone debug command line, This is necessary for client to