From 1a83e0a0db5d8e9220102c7d64d283db44739993 Mon Sep 17 00:00:00 2001 From: tom-vt4 Date: Thu, 25 Jun 2026 05:07:29 -0400 Subject: [PATCH 01/11] Add tool to identify tablets that need compaction Add a utility tool that is able to identify tablets that need compaction. This tool is hosted in the project check-tablet-extents and can be built as an executable JAR. Closes #3581 --- .gitignore | 2 +- .../check-tablet-extents/README.md | 69 +++ .../check-tablet-extents/pom.xml | 194 ++++++ .../java/datawave/TabletExtentChecker.java | 302 +++++++++ .../datawave/TabletExtentCheckerTest.java | 571 ++++++++++++++++++ 5 files changed, 1137 insertions(+), 1 deletion(-) create mode 100644 contrib/datawave-utils/check-tablet-extents/README.md create mode 100644 contrib/datawave-utils/check-tablet-extents/pom.xml create mode 100644 contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java create mode 100644 contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java diff --git a/.gitignore b/.gitignore index eaa9fd08edc..1fbbd622c05 100644 --- a/.gitignore +++ b/.gitignore @@ -53,4 +53,4 @@ services/sample_configuration/*-dev.yml /.jkube-dockerignore .claude - +.sdkmanrc diff --git a/contrib/datawave-utils/check-tablet-extents/README.md b/contrib/datawave-utils/check-tablet-extents/README.md new file mode 100644 index 00000000000..85a5a3f2305 --- /dev/null +++ b/contrib/datawave-utils/check-tablet-extents/README.md @@ -0,0 +1,69 @@ +# Tablet Extent Checker + +This project contains a tool for identifying tablets in a table that are candidates for compaction. + +## Building the tool + +Building the tool is done with the following command. To run unit tests, omit `-DskipTests`. + +```bash +$ mvn clean install -DskipTests -P create-shade-jar +``` + +This will create the executable jar `target/create-tablet-extents-shaded.jar`. + +## Running the tool + +The tool can be executed with the following command: + +```bash +$ java -cp path/to/check-tablet-extents-shaded.jar datawave.TabletExtentChecker +``` + +Use the `--help` option to list all available options: + +```bash +$ java -cp path/to/check-tablet-extents-shaded.jar datawave.TabletExtentChecker -h +Usage: TabletExtentChecker [options] + Options: + -b, --begin + The starting row (exclusive) of the range of tablets to scan + -e, --end + The ending row (inclusive) of the range of tablets to scan + -h, -?, --help, -help + + -m, --merge, --merge-extents + Merges suggested compaction ranges for neighboring compactable tablets + Default: false + * -p, --properties + The path to accumulo-client.properties + * -t, --table + The table name +``` + +The tool will list a set of recommended compaction commands if it finds any tablets that require compaction. For example, given a table `test_table` that has tablets with data outside the tablet extents for the following extents: +- Start: "2000", End: "3000" +- Start: "3000", End: "5000" +- Start: "8000", End: "9000" + +You can expect the following recommended compaction commands. + +```bash +$ java -cp target/check-tablet-extents-shaded.jar datawave.TabletExtentChecker -p $ACCUMULO_HOME/conf/accumulo-client.properties -t test_table +SLF4J(W): No SLF4J providers were found. +SLF4J(W): Defaulting to no-operation (NOP) logger implementation +SLF4J(W): See https://www.slf4j.org/codes.html#noProviders for further details. +compact -t chop_test -b 2000 -e 3000 +compact -t chop_test -b 3000 -e 5000 +compact -t chop_test -b 8000 -e 9000 +``` + +If the option `--merge-extents` was specified above, neighboring compactable tablet extents will be merged within the recommended compaction commands: +```bash +$ java -cp target/check-tablet-extents-shaded.jar datawave.TabletExtentChecker -p $ACCUMULO_HOME/conf/accumulo-client.properties -t test_table +SLF4J(W): No SLF4J providers were found. +SLF4J(W): Defaulting to no-operation (NOP) logger implementation +SLF4J(W): See https://www.slf4j.org/codes.html#noProviders for further details. +compact -t chop_test -b 2000 -e 5000 +compact -t chop_test -b 8000 -e 9000 +``` diff --git a/contrib/datawave-utils/check-tablet-extents/pom.xml b/contrib/datawave-utils/check-tablet-extents/pom.xml new file mode 100644 index 00000000000..ed7f6d84fc0 --- /dev/null +++ b/contrib/datawave-utils/check-tablet-extents/pom.xml @@ -0,0 +1,194 @@ + + + 4.0.0 + + + gov.nsa.datawave + datawave-parent + 7.42.0-SNAPSHOT + + + check-tablet-extents + 1.0.0-SNAPSHOT + Datawave Tablet Extent Checker + A utility tool for identifying tablets with data outside the tablets' extents. + + + + + org.apache.accumulo + accumulo-project + ${version.accumulo} + pom + import + + + + + + + com.beust + jcommander + + + com.google.guava + guava + + + io.opentelemetry + opentelemetry-api + + + io.opentelemetry + opentelemetry-context + + + org.apache.accumulo + accumulo-core + + + org.apache.logging.log4j + log4j-1.2-api + + + + + org.apache.accumulo + accumulo-hadoop-mapreduce + + + org.apache.accumulo + accumulo-tserver + + + org.apache.commons + commons-lang3 + 3.20.0 + + + org.apache.hadoop + hadoop-client-api + + + org.apache.zookeeper + zookeeper + + + io.dropwizard.metrics + metrics-core + 4.1.12.1 + test + + + org.apache.accumulo + accumulo-minicluster + test + + + org.apache.logging.log4j + log4j-1.2-api + + + org.apache.logging.log4j + log4j-api + + + org.apache.logging.log4j + log4j-core + + + org.apache.logging.log4j + log4j-web + + + + + + ${project.artifactId} + + + + org.apache.maven.plugins + maven-compiler-plugin + + true + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.3.0 + + + + + + + create-shade-jar + + + + org.apache.maven.plugins + maven-shade-plugin + + + check-tablet-extents-shade-jar + + shade + + package + + false + ${project.artifactId}-shaded + true + shaded + + + org.apache.accumulo:accumulo-native + jakarta.activation:jakarta.activation-api:jar: + org.apache.hadoop:hadoop-annotations:jar: + org.apache.hadoop:hadoop-auth:jar: + org.apache.hadoop:hadoop-mapreduce-client-common:jar: + org.apache.hadoop:hadoop-yarn-api:jar: + org.apache.hadoop:hadoop-yarn-common:jar: + org.apache.hadoop:hadoop-mapreduce-client-core:jar: + org.apache.hadoop:hadoop-common:jar: + org.apache.hadoop:hadoop-mapreduce-client-jobclient:jar: + org.apache.hadoop:hadoop-yarn-client:jar: + org.apache.hadoop:hadoop-hdfs-client:jar: + org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:jar: + + + + + *:* + + module-info.class + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + *:* + + **/Log4j2Plugins.dat + + + + + + + + datawave.TabletExtentChecker + + + + + + + + + + + diff --git a/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java new file mode 100644 index 00000000000..8c6e5c91e53 --- /dev/null +++ b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java @@ -0,0 +1,302 @@ +package datawave; + +import java.io.IOException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.accumulo.core.client.Accumulo; +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.admin.TableOperations; +import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.conf.ConfigurationCopy; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.file.FileOperations; +import org.apache.accumulo.core.file.FileSKVIterator; +import org.apache.accumulo.core.metadata.StoredTabletFile; +import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import org.apache.accumulo.core.metadata.schema.TabletsMetadata; +import org.apache.accumulo.core.spi.crypto.NoCryptoServiceFactory; +import org.apache.accumulo.tserver.tablet.Tablet; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.io.Text; + +import com.beust.jcommander.IStringConverter; +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; + +/** + * A utility that will identify tablets that require compaction within a given table for a given range and lists recommended compaction commands. + */ +public final class TabletExtentChecker { + + /** + * Converts a string argument to a {@link Text} instance. + */ + private static class TextConverter implements IStringConverter { + @Override + public Text convert(String s) { + return s == null ? null : new Text(s); + } + } + + /** + * Represents a set of options that can parsed and used via {@link TabletExtentChecker#main(String[])}. + */ + private static class Opts { + + @Parameter(names = {"-t", "--table"}, description = "The table name", required = true) + public String tableName = null; + + @Parameter(names = {"-p", "--properties"}, description = "The path to accumulo-client.properties", required = true) + public String propsPath = null; + + @Parameter(names = {"-h", "-?", "--help", "-help"}, help = true) + public boolean help = false; + + @Parameter(names = {"-b", "--begin"}, description = "The starting row (exclusive) of the range of tablets to scan", converter = TextConverter.class) + public Text beginRow = null; + + @Parameter(names = {"-e", "--end"}, description = "The ending row (inclusive) of the range of tablets to scan", converter = TextConverter.class) + public Text endRow = null; + + @Parameter(names = {"-m", "--merge", "--merge-extents"}, description = "Merges suggested compaction ranges for neighboring compactable tablets") + public boolean mergeExtents = false; + + /** + * Validate the arguments in this {@link Opts}. + */ + public void validateArgs() { + if (propsPath == null) { + throw new IllegalArgumentException("accumulo-client.properties must be set!"); + } + + if (!Paths.get(propsPath).toFile().exists()) { + throw new IllegalArgumentException(propsPath + " does not exist!"); + } + } + + /** + * Parse the given arguments array and populate this {@link Opts}. + * + * @param args + * the arguments to parse. + */ + public void parseArgs(String[] args) { + JCommander commander = JCommander.newBuilder().addObject(this).programName(TabletExtentChecker.class.getSimpleName()).build(); + + try { + commander.parse(args); + } catch (ParameterException ex) { + commander.usage(); + System.err.println(ex.getMessage()); + System.exit(1); + } + + if (help) { + commander.usage(); + System.exit(0); + } + } + } + + /** + * Parses the user-provided arguments and prints out recommended ranges for compaction. + * + * @param args + * the arguments + * @throws AccumuloException + * if an error occurs while connecting to Accumulo + * @throws TableNotFoundException + * if the specified table does not exist + * @throws IOException + * if an error occurs while reading the client properties file + */ + public static void main(String[] args) throws AccumuloException, TableNotFoundException, IOException { + // Parse the arguments. + Opts opts = new Opts(); + opts.parseArgs(args); + opts.validateArgs(); + + // Fetch the recommended tablet ranges to compact. + List> compactionExtents = TabletExtentChecker.checkTablets(opts.propsPath, opts.tableName, opts.beginRow, opts.endRow, + opts.mergeExtents); + + // Print out the recommended compaction commands. + for (Pair pair : compactionExtents) { + Text startRow = pair.getLeft(); + Text endRow = pair.getRight(); + System.out.println("compact -t " + opts.tableName + formatArg("-b", startRow) + formatArg("-e", endRow)); + + } + } + + /** + * Return the given value formatted as an argument snippet if the value is not null. + * + * @param arg + * the argument option + * @param value + * the value + * @return the formatted arg to value command snippet if the value is not null, otherwise returns an empty string + */ + private static String formatArg(String arg, Text value) { + if (value == null) { + return ""; + } else { + return " " + arg + " " + value; + } + } + + /** + * Returns a list of pairs consisting of the extents of tablets that require compaction. + * + * @param clientProperties + * the fully qualified path to the accumulo-client.properties file to use when connecting to Accumulo + * @param tableName + * the name of the table to evaluate for tablet compaction + * @param begin + * the starting row (exclusive) of the range to search for tablets within. A null value implies no starting boundary. + * @param end + * the ending row (inclusive) of the range to search for tablets within. A null value implies no ending boundary + * @param mergeExtents + * will merge recommended ranges of neighboring tablets requiring compaction + * @return the list of tablet boundaries recommended for compaction + * @throws AccumuloException + * if an error occurs while connecting to Accumulo + * @throws TableNotFoundException + * if the specified table does not exist + * @throws IOException + * if an error occurs while reading the client properties file + */ + static List> checkTablets(String clientProperties, String tableName, Text begin, Text end, boolean mergeExtents) + throws AccumuloException, TableNotFoundException, IOException { + List> compactionExtents = new ArrayList<>(); + try (AccumuloClient client = Accumulo.newClient().from(clientProperties).build()) { + TableId tableId = getTableId(client, tableName); + ClientContext context = (ClientContext) client; + // Fetch the metadata for all tablets in the given table whose extents overlap with the user provided range of tablets to scan. + try (TabletsMetadata tablets = context.getAmple().readTablets().forTable(tableId).overlapping(begin, false, end) + .fetch(TabletMetadata.ColumnType.PREV_ROW, TabletMetadata.ColumnType.FILES).build()) { + + // Tracks compaction ranges if we are merging extents. + boolean foundCompactableTablet = false; + Text compactionStart = null; + Text compactionEnd = null; + + // Iterate over each tablet. + for (TabletMetadata tablet : tablets) { + // Determine whether the tablet needs compaction. + boolean tabletRequiresCompaction = tabletRequiresCompaction(context, tableName, tablet); + KeyExtent extent = tablet.getExtent(); + + // The current tablet requires compaction. + if (tabletRequiresCompaction) { + // If we are merging extents, update the current compaction start and end based on whether the previous tablet also required compaction. + if (mergeExtents) { + // The previous tablet did not require compaction. Update the compaction range to reflect the current tablet. + if (!foundCompactableTablet) { + compactionStart = extent.prevEndRow(); + compactionEnd = extent.endRow(); + foundCompactableTablet = true; + } else { + // The previous tablet needs compaction, along with the current tablet. Update the end row. + compactionEnd = extent.endRow(); + } + } else { + compactionExtents.add(Pair.of(extent.prevEndRow(), extent.endRow())); + } + // The current tablet does not require compaction. + } else { + // We are merging tablet extents, and the previous tablet requires compaction. + if (mergeExtents && foundCompactableTablet) { + // Add a new recommended compaction range and reset our compaction boundaries. + compactionExtents.add(Pair.of(compactionStart, compactionEnd)); + compactionStart = null; + compactionEnd = null; + foundCompactableTablet = false; + } + } + } + + // Handle case where last tablet needs compaction when we are merging extents. + if (mergeExtents && foundCompactableTablet) { + compactionExtents.add(Pair.of(compactionStart, compactionEnd)); + } + + } + } + return compactionExtents; + } + + /** + * Return the table ID for the given table + * + * @param client + * the client to use when connecting to Accumulo + * @param tableName + * the table name + * @return the table ID + */ + private static TableId getTableId(AccumuloClient client, String tableName) { + TableOperations tableOperations = client.tableOperations(); + if (tableOperations.exists(tableName)) { + return TableId.of(tableOperations.tableIdMap().get(tableName)); + } else { + throw new IllegalArgumentException("Table " + tableName + " does not exist"); + } + } + + /** + * Return whether the given tablet requires compaction. This method represents a merging of the functions + * {@link org.apache.accumulo.tserver.tablet.CompactableUtils#getFirstAndLastKeys(Tablet, Set)} and + * {@link org.apache.accumulo.tserver.tablet.CompactableUtils#findChopFiles(KeyExtent, Map, Collection)} that is designed to return true as soon as we find + * an RFile that is empty, or whose first or last keys fall outside the tablet's extent. + * + * @param context + * the context to use when connecting to Accumulo + * @param tableName + * the tablet name + * @param tablet + * the tablet metadata + * @return true if the tablet requires compaction, or false + */ + private static boolean tabletRequiresCompaction(ClientContext context, String tableName, TabletMetadata tablet) + throws AccumuloException, TableNotFoundException, IOException { + // Fetch the list of RFiles for the tablet. + ConfigurationCopy tableConf = new ConfigurationCopy(context.tableOperations().getConfiguration(tableName)); + KeyExtent extent = tablet.getExtent(); + Set allFiles = new HashSet<>(tablet.getFiles()); + final FileOperations fileFactory = FileOperations.getInstance(); + + // Examine each file and determine whether any of them would be cleaned up/optimized by a compaction. + for (StoredTabletFile file : allFiles) { + FileSystem ns = FileSystem.get(file.getPath().toUri(), context.getHadoopConf()); + try (FileSKVIterator openReader = fileFactory.newReaderBuilder().forFile(file.getPathStr(), ns, ns.getConf(), NoCryptoServiceFactory.NONE) + .withTableConfiguration(tableConf).seekToBeginning().build()) { + Key first = openReader.getFirstKey(); + Key last = openReader.getLastKey(); + + // A tablet requires a compaction if any of the following are true: + // - The file is empty (first and last are null) + // - The first key is outside the tablet's extent. + // - The last key is outside the tablet's extent. + if ((first == null && last == null) || (first != null && !extent.contains(first.getRow())) + || (last != null && !extent.contains(last.getRow()))) { + return true; + } + } + } + return false; + } +} diff --git a/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java b/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java new file mode 100644 index 00000000000..828a095c47f --- /dev/null +++ b/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java @@ -0,0 +1,571 @@ +package datawave; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.minicluster.MiniAccumuloCluster; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.io.Text; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TabletExtentCheckerTest { + private static final String TABLE_NAME = "testTable"; + + @TempDir + private static Path tempDir; + private static String clientPropertiesPath; + private static MiniAccumuloCluster mac; + + private AccumuloClient client; + + // Tracks the range of tablet extents to compact on + private final List> expectedExtents = new ArrayList<>(); + String begin; + String end; + boolean mergeExtents; + + @BeforeAll + static void beforeAll() throws IOException, InterruptedException { + mac = new MiniAccumuloCluster(tempDir.resolve("mac").toFile(), "secret"); + mac.start(); + Path propertiesFile = tempDir.resolve("accumulo-client.properties"); + try (BufferedWriter out = Files.newBufferedWriter(propertiesFile)) { + mac.getClientProperties().store(out, null); + } + clientPropertiesPath = propertiesFile.toAbsolutePath().toString(); + } + + @AfterAll + static void afterAll() throws IOException { + mac.close(); + } + + @BeforeEach + void setUp() { + begin = null; + end = null; + mergeExtents = false; + client = mac.createAccumuloClient("root", new PasswordToken("secret")); + } + + @AfterEach + void tearDown() throws AccumuloException, TableNotFoundException, AccumuloSecurityException { + expectedExtents.clear(); + if (client.tableOperations().exists(TABLE_NAME)) { + client.tableOperations().delete(TABLE_NAME); + } + client.close(); + } + + /** + * Contains test methods that verify {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} tracks individual unmerged tablet + * extents given a table, tablet range, and mergeExtents == false. + */ + @Nested + class MergeExtentsDisabledTests { + + @BeforeEach + void setUp() { + mergeExtents = false; + } + + /** + * Verify that given a table where all tablets require compaction, and given a tablet range, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns multiple ranges to compact each tablet individually. + */ + @Test + void testAllSingleTabletsNeedCompaction() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) + addSplits("2500", "4000", "7000"); + + expectExtent(null, "2500"); + expectExtent("2500", "4000"); + expectExtent("4000", "7000"); + expectExtent("7000", null); + assertResult(); + } + + /** + * Verify that given a table where only the first tablet requires compaction, and given a tablet range, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns a single range that would compact only the first tablet. + */ + @Test + void testOnlyFirstTabletNeedsCompaction() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) + addSplits("2500", "4000", "7000"); + + // Compact all tablets other than the first tablet. + compact("2500", null); + + // Add uncompacted tablet to expectedExtents + expectExtent(null, "2500"); + assertResult(); + } + + /** + * Verify that given a table where only the last tablet requires compaction, and given a tablet range, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns a single range that would compact only the last tablet. + */ + @Test + void testOnlyLastTabletNeedsCompaction() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) + addSplits("2500", "4000", "7000"); + + // Compact all tablets except the last + compact(null, "7000"); + + expectExtent("7000", null); + assertResult(); + } + + /** + * Verify that given a table where no tablets require compaction, and given a tablet range, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns an empty list. + */ + @Test + void testNoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) + addSplits("2500", "4000", "7000"); + + compact(null, null); + + assertResult(); + } + + /** + * Verify that given a table where only the first and last tablets require compaction, and given a tablet range, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns separate ranges for the first and last compactable tablets. + */ + @Test + void testOnlyFirstAndLastTabletsNeedCompaction() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + addSplits("2500", "4000", "7000"); + + // Compact all tablets other than first tablet and last tablet + // Range of first tablet: null - 2500 + // Range of last tablet: 7000 - null + compact("2500", "7000"); + + expectExtent(null, "2500"); + expectExtent("7000", null); + assertResult(); + } + + /** + * Verify that given a table where only the last two tablets require compaction, and given a tablet range, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns separate ranges for those tablets. + */ + @Test + void testLastTwoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + addSplits("2500", "4000", "5500", "7000", "8500"); + + // Compact all tablets except last two tablets + compact(null, "7000"); + + // Add last two tablets to expectedExtents + expectExtent("7000", "8500"); + expectExtent("8500", null); + assertResult(); + } + + /** + * Verify that given a table where only the middle tablet requires compaction, and given a tablet range, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns a range to compact that middle tablet. + */ + @Test + void testMiddleTabletNeedsCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000, null), (8000, null) + addSplits("2500", "4000", "7000", "8000"); + + compact(null, "4000"); + compact("7000", null); + + expectExtent("4000", "7000"); + assertResult(); + } + + /** + * Verify that given a table where multiple series of tablets at different locations require compaction, and given a tablet range, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns multiple compaction ranges that includes a series of + * contiguous compactable tablets by merging their extents. + */ + @Test + void testMultipleInnerCompactionRanges() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + addSplits("1000", "1500", "2000", "2500", "3000", "4000", "5500", "6500", "7000", "7500", "8500", "9000"); + // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), + // (5500, 6500), (6500,7000), (7000,7500), (7500,8500) (8500,9000) + + // Compact extents to make a series of compacted tablets then track the uncompacted tablets: + compact(null, "1000"); + compact("2500", "3000"); + compact("6500", "7000"); + compact("8500", null); + + // Track uncompacted tablets in the key extent series ("1000","2500") + // Tablets: (1000, 1500), (1500,2000), (2000,2500) + expectExtent("1000", "1500"); + expectExtent("1500", "2000"); + expectExtent("2000", "2500"); + + // Track uncompacted tablets in the key extent series ("3000","6500") + // Tablets: (3000,4000), (4000,5500), (5500, 6500) + expectExtent("3000", "4000"); + expectExtent("4000", "5500"); + expectExtent("5500", "6500"); + + // Track uncompacted tablets in the key extent series ("7000","8500") + // Tablets: (7000,7500), (7500,8500) + expectExtent("7000", "7500"); + expectExtent("7500", "8500"); + assertResult(); + } + + /** + * Verify that given a table where all tablets starting from row 2000 to null are compacted, and given a tablet range from row 2000 to null, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns an empty list. + */ + @Test + void testCompactableTabletOutsideInputRange() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), + // (5500, 6500), (6500,7000), (7000,7500), (7500,8500) (8500,9000) + addSplits("1000", "1500", "2000", "2500", "3000", "4000", "5500", "6500", "7000", "7500", "8500", "9000"); + + compact("2000", null); + + givenBegin("2000"); + + // expectedExtents should be empty, so just make the assertion + assertResult(); + } + + /** + * Verify that given a table with a compactable tablet, and given a key extent range that is contained within a tablet's range, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} will return the full extent of the tablet. + */ + @Test + void testInputInCompactableTabletRange() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), + // (5500, 6500), (6500,7000), (7000,7500), (7500,8500) (8500,9000) + addSplits("1000", "1500", "2000", "2500", "3000", "4000", "5500", "6500", "7000", "7500", "8500", "9000"); + + // Compact all tablets from null to row 1000 and from row 1500 to null + compact(null, "1000"); + compact("1500", null); + + // Compactable tablet: (1000,1500) + givenBegin("1000"); + givenEnd("1250"); + expectExtent("1000", "1500"); + assertResult(); + } + + } + + /** + * Contains test methods that verify {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} tracks individual unmerged tablet + * extents given client properties, the table name, a user provided tablet range, and mergeExtents == true. + */ + @Nested + class MergeExtentsEnabledTests { + + @BeforeEach + void setUp() { + mergeExtents = true; + } + + /** + * Verify that given a table where all tablets require compaction, and given a tablet range, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean)} returns a single compaction range that would compact the entire table. + */ + @Test + void testAllTabletsNeedCompaction() throws AccumuloException, TableExistsException, AccumuloSecurityException, TableNotFoundException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) + addSplits("2500", "4000", "7000"); + + expectExtent(null, null); + assertResult(); + } + + /** + * Verify that given a table where only the first tablet requires compaction, and given a tablet range, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns a single range that would compact only the first tablet. + */ + @Test + void testOnlyFirstTabletNeedsCompaction() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) + addSplits("2500", "4000", "7000"); + + // Compact all tablets other than the first tablet. + compact("2500", null); + + expectExtent(null, "2500"); + assertResult(); + } + + /** + * Verify that given a table where only the last tablet requires compaction, and given a tablet range, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns a single range that would compact only the last tablet. + */ + @Test + void testOnlyLastTabletNeedsCompaction() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) + addSplits("2500", "4000", "7000"); + + // Compact all tablets except the last + compact(null, "7000"); + + expectExtent("7000", null); + assertResult(); + } + + /** + * Verify that given a table where no tablets require compaction, and given a tablet range, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns an empty list. + */ + @Test + void testNoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) + addSplits("2500", "4000", "7000"); + + // Compact all tablets + compact(null, null); + + assertResult(); + } + + /** + * Verify that given a table where only the first and last tablets require compaction, and given a tablet range, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns separate ranges for the first and last compactable tablets. + */ + @Test + void testOnlyFirstAndLastTabletsNeedCompaction() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) + addSplits("2500", "4000", "7000"); + + // Compact all tablets other than first tablet and last tablet + compact("2500", "7000"); + + expectExtent(null, "2500"); + expectExtent("7000", null); + + assertResult(); + } + + /** + * Verify that given a table where only the last two tablets require compaction, and given a tablet range, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns a single range that compacts the last two tablets. + */ + @Test + void testLastTwoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,5500), (5500,7000), (7000,8500) (8500,null) + addSplits("2500", "4000", "5500", "7000", "8500"); + + // Compact all tablets except last two tablets + compact(null, "7000"); + + // Add last two tablets to expectedExtents + expectExtent("7000", null); + assertResult(); + } + + /** + * Verify that given a table where only the middle tablet requires compaction, and given a tablet range, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns a range to compact that middle tablet. + */ + @Test + void testMiddleTabletNeedsCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null), (8000,null) + addSplits("2500", "4000", "7000", "8000"); + + // Compact all tablets except the tablet with the extent of (4000,7000) + compact(null, "4000"); + compact("7000", null); + + expectExtent("4000", "7000"); + assertResult(); + } + + /** + * Verify that given a table where multiple series of tablets at different locations require compaction, and given a tablet range, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns multiple compaction ranges that includes a series of + * contiguous compactable tablets by merging their extents. + */ + @Test + void testMultipleInnerCompactionRanges() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), + // (5500, 6500), (6500,7000), (7000,7500), (7500,8500) (8500,9000) + addSplits("1000", "1500", "2000", "2500", "3000", "4000", "5500", "6500", "7000", "7500", "8500", "9000"); + + // Compact extents to make a series of compacted tablets then track the series of uncompacted tablets: + compact(null, "1000"); + compact("2500", "3000"); + compact("6500", "7000"); + compact("8500", null); + + // Track uncompacted extent series: + expectExtent("1000", "2500"); // Tablets: (1000, 1500), (1500,2000), (2000,2500) + expectExtent("3000", "6500"); // Tablets: (3000,4000), (4000,5500), (5500, 6500) + expectExtent("7000", "8500"); // Tablets: (7000,7500), (7500,8500) + assertResult(); + } + + /** + * Verify that given a table where all tablets starting from row 2000 to null are compacted, and given a tablet range from row 2000 to null, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns an empty list. + */ + @Test + void testCompactableTabletOutsideInputRange() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), + // (5500, 6500), (6500,7000), (7000,7500), (7500,8500) (8500,9000) + addSplits("1000", "1500", "2000", "2500", "3000", "4000", "5500", "6500", "7000", "7500", "8500", "9000"); + + // Compact all tablets starting from row 2000 + compact("2000", null); + + // Only look at the range (2000,null) so expectedExtents should be empty + givenBegin("2000"); + givenEnd(null); + assertResult(); + } + + /** + * Verify that given a table with a compactable tablet, and given a key extent range that is contained within a tablet's range, + * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} will return the full extent of the tablet. + */ + @Test + void testInputInCompactableTabletRange() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), + // (5500, 6500), (6500,7000), (7000,7500), (7500,8500) (8500,9000) + addSplits("1000", "1500", "2000", "2500", "3000", "4000", "5500", "6500", "7000", "7500", "8500", "9000"); + + // Compact all tablets from null to row 1000 and from row 1500 to null + compact(null, "1000"); + compact("1500", null); + + // Compactable tablet: (1000,1500) + givenBegin("1000"); + givenEnd("1250"); + expectExtent("1000", "1500"); + assertResult(); + } + } + + private void createTableAndWriteData() throws AccumuloException, TableExistsException, AccumuloSecurityException, TableNotFoundException { + client.tableOperations().create(TABLE_NAME); + + try (BatchWriter writer = client.createBatchWriter(TABLE_NAME)) { + for (int i = 0; i < 10000; i += 250) { + String row = String.format("%04d", i); + Mutation m = new Mutation(row); + m.put("cf", "cq", "v"); + writer.addMutation(m); + } + } + client.tableOperations().flush(TABLE_NAME); + } + + private void addSplits(String... splits) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { + SortedSet set = new TreeSet<>(); + Arrays.stream(splits).map(Text::new).forEach(set::add); + client.tableOperations().addSplits(TABLE_NAME, set); + } + + private void compact(String start, String end) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { + client.tableOperations().compact(TABLE_NAME, getText(start), getText(end), true, true); + } + + private Text getText(String arg) { + return (arg == null) ? null : new Text(arg); + } + + private void expectExtent(String start, String end) { + expectedExtents.add(Pair.of(getText(start), getText(end))); + } + + private void givenBegin(String begin) { + this.begin = begin; + } + + private void givenEnd(String end) { + this.end = end; + } + + private void assertResult() throws AccumuloException, TableNotFoundException, IOException { + List> result = TabletExtentChecker.checkTablets(clientPropertiesPath, TABLE_NAME, getText(begin), getText(end), mergeExtents); + assertEquals(expectedExtents, result); + } + +} From a7686adbe0dc4d605fadc8f42fc923866fe28fe5 Mon Sep 17 00:00:00 2001 From: tom-vt4 Date: Mon, 6 Jul 2026 11:47:15 -0400 Subject: [PATCH 02/11] Incorporate PR review --- .../java/datawave/TabletExtentChecker.java | 174 ++++++++++-------- .../datawave/TabletExtentCheckerTest.java | 57 +++--- 2 files changed, 130 insertions(+), 101 deletions(-) diff --git a/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java index 8c6e5c91e53..b14e00518a7 100644 --- a/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java +++ b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java @@ -1,7 +1,6 @@ package datawave; import java.io.IOException; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; @@ -9,9 +8,11 @@ import java.util.Map; import java.util.Set; +import org.apache.accumulo.core.cli.ClientOpts; import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.accumulo.core.clientImpl.ClientContext; @@ -54,15 +55,24 @@ public Text convert(String s) { * Represents a set of options that can parsed and used via {@link TabletExtentChecker#main(String[])}. */ private static class Opts { + @Parameter(names = {"-h", "-?", "--help", "-help"}, help = true) + public boolean help = false; - @Parameter(names = {"-t", "--table"}, description = "The table name", required = true) - public String tableName = null; + @Parameter(names = {"-a", "--accumulo-instance"}, description = "The Accumulo instance.", required = true) + public String accumuloInstance = null; - @Parameter(names = {"-p", "--properties"}, description = "The path to accumulo-client.properties", required = true) - public String propsPath = null; + @Parameter(names = {"-z", "--zookeper-instance"}, description = "The Zookeeper instance.", required = true) + public String zookeeperInstance = null; - @Parameter(names = {"-h", "-?", "--help", "-help"}, help = true) - public boolean help = false; + @Parameter(names = {"-u", "--username"}, description = "The Accumulo username.", required = true) + public String username = null; + + @Parameter(names = {"-p", "--password"}, description = "The Accumulo password. Can be supplied from an environment variable via env:ENV_VARIABLE", + converter = ClientOpts.PasswordConverter.class, required = true) + public String password = null; + + @Parameter(names = {"-t", "--table"}, description = "The table name", required = true) + public String tableName = null; @Parameter(names = {"-b", "--begin"}, description = "The starting row (exclusive) of the range of tablets to scan", converter = TextConverter.class) public Text beginRow = null; @@ -73,16 +83,24 @@ private static class Opts { @Parameter(names = {"-m", "--merge", "--merge-extents"}, description = "Merges suggested compaction ranges for neighboring compactable tablets") public boolean mergeExtents = false; + @Parameter(names = {"-c", "--compact"}, description = "Compact tablets that have been identified as candidates for needing compaction") + public boolean compactTablets = false; + /** * Validate the arguments in this {@link Opts}. */ public void validateArgs() { - if (propsPath == null) { - throw new IllegalArgumentException("accumulo-client.properties must be set!"); + if (accumuloInstance == null) { + throw new IllegalArgumentException("Accumulo instance name must be entered!"); } - - if (!Paths.get(propsPath).toFile().exists()) { - throw new IllegalArgumentException(propsPath + " does not exist!"); + if (zookeeperInstance == null) { + throw new IllegalArgumentException("Zookeeper instance must be entered!"); + } + if (username == null) { + throw new IllegalArgumentException("Username must be entered!"); + } + if (password == null) { + throw new IllegalArgumentException("Password must be entered!"); } } @@ -122,22 +140,30 @@ public void parseArgs(String[] args) { * @throws IOException * if an error occurs while reading the client properties file */ - public static void main(String[] args) throws AccumuloException, TableNotFoundException, IOException { + public static void main(String[] args) throws AccumuloException, TableNotFoundException, IOException, AccumuloSecurityException { // Parse the arguments. Opts opts = new Opts(); opts.parseArgs(args); opts.validateArgs(); - - // Fetch the recommended tablet ranges to compact. - List> compactionExtents = TabletExtentChecker.checkTablets(opts.propsPath, opts.tableName, opts.beginRow, opts.endRow, - opts.mergeExtents); - - // Print out the recommended compaction commands. - for (Pair pair : compactionExtents) { - Text startRow = pair.getLeft(); - Text endRow = pair.getRight(); - System.out.println("compact -t " + opts.tableName + formatArg("-b", startRow) + formatArg("-e", endRow)); - + try (AccumuloClient client = Accumulo.newClient().to(opts.accumuloInstance, opts.zookeeperInstance).as(opts.username, opts.password).build()) { + // Fetch the recommended tablet ranges to compact. + List> compactionExtents = TabletExtentChecker.checkTablets(client, opts.tableName, opts.beginRow, opts.endRow, opts.mergeExtents); + + // Print a message when compactionExtents is empty + if (compactionExtents.isEmpty()) { + System.out.println("No candidates suitable for compaction."); + } else { + for (Pair pair : compactionExtents) { + Text startRow = pair.getLeft(); + Text endRow = pair.getRight(); + if (opts.compactTablets) { + System.out.println("Compacting range from " + startRow + "-" + endRow); + client.tableOperations().compact(opts.tableName, startRow, endRow, true, true); + } else { + System.out.println("compact -t " + opts.tableName + formatArg("-b", startRow) + formatArg("-e", endRow)); + } + } + } } } @@ -161,8 +187,6 @@ private static String formatArg(String arg, Text value) { /** * Returns a list of pairs consisting of the extents of tablets that require compaction. * - * @param clientProperties - * the fully qualified path to the accumulo-client.properties file to use when connecting to Accumulo * @param tableName * the name of the table to evaluate for tablet compaction * @param begin @@ -179,63 +203,63 @@ private static String formatArg(String arg, Text value) { * @throws IOException * if an error occurs while reading the client properties file */ - static List> checkTablets(String clientProperties, String tableName, Text begin, Text end, boolean mergeExtents) + static List> checkTablets(AccumuloClient client, String tableName, Text begin, Text end, boolean mergeExtents) throws AccumuloException, TableNotFoundException, IOException { List> compactionExtents = new ArrayList<>(); - try (AccumuloClient client = Accumulo.newClient().from(clientProperties).build()) { - TableId tableId = getTableId(client, tableName); - ClientContext context = (ClientContext) client; - // Fetch the metadata for all tablets in the given table whose extents overlap with the user provided range of tablets to scan. - try (TabletsMetadata tablets = context.getAmple().readTablets().forTable(tableId).overlapping(begin, false, end) - .fetch(TabletMetadata.ColumnType.PREV_ROW, TabletMetadata.ColumnType.FILES).build()) { - - // Tracks compaction ranges if we are merging extents. - boolean foundCompactableTablet = false; - Text compactionStart = null; - Text compactionEnd = null; - - // Iterate over each tablet. - for (TabletMetadata tablet : tablets) { - // Determine whether the tablet needs compaction. - boolean tabletRequiresCompaction = tabletRequiresCompaction(context, tableName, tablet); - KeyExtent extent = tablet.getExtent(); - - // The current tablet requires compaction. - if (tabletRequiresCompaction) { - // If we are merging extents, update the current compaction start and end based on whether the previous tablet also required compaction. - if (mergeExtents) { - // The previous tablet did not require compaction. Update the compaction range to reflect the current tablet. - if (!foundCompactableTablet) { - compactionStart = extent.prevEndRow(); - compactionEnd = extent.endRow(); - foundCompactableTablet = true; - } else { - // The previous tablet needs compaction, along with the current tablet. Update the end row. - compactionEnd = extent.endRow(); - } + + TableId tableId = getTableId(client, tableName); + ClientContext context = (ClientContext) client; + // Fetch the metadata for all tablets in the given table whose extents overlap with the user provided range of tablets to scan. + try (TabletsMetadata tablets = context.getAmple().readTablets().forTable(tableId).overlapping(begin, false, end) + .fetch(TabletMetadata.ColumnType.PREV_ROW, TabletMetadata.ColumnType.FILES).build()) { + + // Tracks compaction ranges if we are merging extents. + boolean foundCompactableTablet = false; + Text compactionStart = null; + Text compactionEnd = null; + + // Iterate over each tablet. + for (TabletMetadata tablet : tablets) { + // Determine whether the tablet needs compaction. + boolean tabletRequiresCompaction = tabletRequiresCompaction(context, tableName, tablet); + KeyExtent extent = tablet.getExtent(); + + // The current tablet requires compaction. + if (tabletRequiresCompaction) { + // If we are merging extents, update the current compaction start and end based on whether the previous tablet also required compaction. + if (mergeExtents) { + // The previous tablet did not require compaction. Update the compaction range to reflect the current tablet. + if (!foundCompactableTablet) { + compactionStart = extent.prevEndRow(); + compactionEnd = extent.endRow(); + foundCompactableTablet = true; } else { - compactionExtents.add(Pair.of(extent.prevEndRow(), extent.endRow())); + // The previous tablet needs compaction, along with the current tablet. Update the end row. + compactionEnd = extent.endRow(); } - // The current tablet does not require compaction. } else { - // We are merging tablet extents, and the previous tablet requires compaction. - if (mergeExtents && foundCompactableTablet) { - // Add a new recommended compaction range and reset our compaction boundaries. - compactionExtents.add(Pair.of(compactionStart, compactionEnd)); - compactionStart = null; - compactionEnd = null; - foundCompactableTablet = false; - } + compactionExtents.add(Pair.of(extent.prevEndRow(), extent.endRow())); + } + // The current tablet does not require compaction. + } else { + // We are merging tablet extents, and the previous tablet requires compaction. + if (mergeExtents && foundCompactableTablet) { + // Add a new recommended compaction range and reset our compaction boundaries. + compactionExtents.add(Pair.of(compactionStart, compactionEnd)); + compactionStart = null; + compactionEnd = null; + foundCompactableTablet = false; } } + } - // Handle case where last tablet needs compaction when we are merging extents. - if (mergeExtents && foundCompactableTablet) { - compactionExtents.add(Pair.of(compactionStart, compactionEnd)); - } - + // Handle case where last tablet needs compaction when we are merging extents. + if (mergeExtents && foundCompactableTablet) { + compactionExtents.add(Pair.of(compactionStart, compactionEnd)); } + } + return compactionExtents; } @@ -288,11 +312,9 @@ private static boolean tabletRequiresCompaction(ClientContext context, String ta Key last = openReader.getLastKey(); // A tablet requires a compaction if any of the following are true: - // - The file is empty (first and last are null) // - The first key is outside the tablet's extent. // - The last key is outside the tablet's extent. - if ((first == null && last == null) || (first != null && !extent.contains(first.getRow())) - || (last != null && !extent.contains(last.getRow()))) { + if ((first != null && !extent.contains(first.getRow())) || (last != null && !extent.contains(last.getRow()))) { return true; } } diff --git a/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java b/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java index 828a095c47f..b0550f20132 100644 --- a/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java +++ b/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java @@ -81,8 +81,8 @@ void tearDown() throws AccumuloException, TableNotFoundException, AccumuloSecuri } /** - * Contains test methods that verify {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} tracks individual unmerged tablet - * extents given a table, tablet range, and mergeExtents == false. + * Contains test methods that verify {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} tracks individual unmerged + * tablet extents given a table, tablet range, and mergeExtents == false. */ @Nested class MergeExtentsDisabledTests { @@ -94,7 +94,7 @@ void setUp() { /** * Verify that given a table where all tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns multiple ranges to compact each tablet individually. + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns multiple ranges to compact each tablet individually. */ @Test void testAllSingleTabletsNeedCompaction() @@ -113,7 +113,8 @@ void testAllSingleTabletsNeedCompaction() /** * Verify that given a table where only the first tablet requires compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns a single range that would compact only the first tablet. + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns a single range that would compact only the first + * tablet. */ @Test void testOnlyFirstTabletNeedsCompaction() @@ -133,7 +134,8 @@ void testOnlyFirstTabletNeedsCompaction() /** * Verify that given a table where only the last tablet requires compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns a single range that would compact only the last tablet. + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns a single range that would compact only the last + * tablet. */ @Test void testOnlyLastTabletNeedsCompaction() @@ -151,7 +153,7 @@ void testOnlyLastTabletNeedsCompaction() /** * Verify that given a table where no tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns an empty list. + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns an empty list. */ @Test void testNoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { @@ -167,7 +169,8 @@ void testNoTabletsNeedCompaction() throws AccumuloException, TableNotFoundExcept /** * Verify that given a table where only the first and last tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns separate ranges for the first and last compactable tablets. + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns separate ranges for the first and last compactable + * tablets. */ @Test void testOnlyFirstAndLastTabletsNeedCompaction() @@ -188,7 +191,7 @@ void testOnlyFirstAndLastTabletsNeedCompaction() /** * Verify that given a table where only the last two tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns separate ranges for those tablets. + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns separate ranges for those tablets. */ @Test void testLastTwoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { @@ -207,7 +210,7 @@ void testLastTwoTabletsNeedCompaction() throws AccumuloException, TableNotFoundE /** * Verify that given a table where only the middle tablet requires compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns a range to compact that middle tablet. + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns a range to compact that middle tablet. */ @Test void testMiddleTabletNeedsCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { @@ -225,7 +228,7 @@ void testMiddleTabletNeedsCompaction() throws AccumuloException, TableNotFoundEx /** * Verify that given a table where multiple series of tablets at different locations require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns multiple compaction ranges that includes a series of + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns multiple compaction ranges that includes a series of * contiguous compactable tablets by merging their extents. */ @Test @@ -263,7 +266,7 @@ void testMultipleInnerCompactionRanges() /** * Verify that given a table where all tablets starting from row 2000 to null are compacted, and given a tablet range from row 2000 to null, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns an empty list. + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns an empty list. */ @Test void testCompactableTabletOutsideInputRange() @@ -284,7 +287,7 @@ void testCompactableTabletOutsideInputRange() /** * Verify that given a table with a compactable tablet, and given a key extent range that is contained within a tablet's range, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} will return the full extent of the tablet. + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} will return the full extent of the tablet. */ @Test void testInputInCompactableTabletRange() @@ -309,8 +312,8 @@ void testInputInCompactableTabletRange() } /** - * Contains test methods that verify {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} tracks individual unmerged tablet - * extents given client properties, the table name, a user provided tablet range, and mergeExtents == true. + * Contains test methods that verify {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} tracks individual unmerged + * tablet extents given client properties, the table name, a user provided tablet range, and mergeExtents == true. */ @Nested class MergeExtentsEnabledTests { @@ -322,7 +325,8 @@ void setUp() { /** * Verify that given a table where all tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean)} returns a single compaction range that would compact the entire table. + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns a single compaction range that would compact the + * entire table. */ @Test void testAllTabletsNeedCompaction() throws AccumuloException, TableExistsException, AccumuloSecurityException, TableNotFoundException, IOException { @@ -337,7 +341,8 @@ void testAllTabletsNeedCompaction() throws AccumuloException, TableExistsExcepti /** * Verify that given a table where only the first tablet requires compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns a single range that would compact only the first tablet. + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns a single range that would compact only the first + * tablet. */ @Test void testOnlyFirstTabletNeedsCompaction() @@ -356,7 +361,8 @@ void testOnlyFirstTabletNeedsCompaction() /** * Verify that given a table where only the last tablet requires compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns a single range that would compact only the last tablet. + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns a single range that would compact only the last + * tablet. */ @Test void testOnlyLastTabletNeedsCompaction() @@ -375,7 +381,7 @@ void testOnlyLastTabletNeedsCompaction() /** * Verify that given a table where no tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns an empty list. + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns an empty list. */ @Test void testNoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { @@ -392,7 +398,8 @@ void testNoTabletsNeedCompaction() throws AccumuloException, TableNotFoundExcept /** * Verify that given a table where only the first and last tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns separate ranges for the first and last compactable tablets. + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns separate ranges for the first and last compactable + * tablets. */ @Test void testOnlyFirstAndLastTabletsNeedCompaction() @@ -413,7 +420,7 @@ void testOnlyFirstAndLastTabletsNeedCompaction() /** * Verify that given a table where only the last two tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns a single range that compacts the last two tablets. + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns a single range that compacts the last two tablets. */ @Test void testLastTwoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { @@ -432,7 +439,7 @@ void testLastTwoTabletsNeedCompaction() throws AccumuloException, TableNotFoundE /** * Verify that given a table where only the middle tablet requires compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns a range to compact that middle tablet. + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns a range to compact that middle tablet. */ @Test void testMiddleTabletNeedsCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { @@ -451,7 +458,7 @@ void testMiddleTabletNeedsCompaction() throws AccumuloException, TableNotFoundEx /** * Verify that given a table where multiple series of tablets at different locations require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns multiple compaction ranges that includes a series of + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns multiple compaction ranges that includes a series of * contiguous compactable tablets by merging their extents. */ @Test @@ -478,7 +485,7 @@ void testMultipleInnerCompactionRanges() /** * Verify that given a table where all tablets starting from row 2000 to null are compacted, and given a tablet range from row 2000 to null, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} returns an empty list. + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns an empty list. */ @Test void testCompactableTabletOutsideInputRange() @@ -500,7 +507,7 @@ void testCompactableTabletOutsideInputRange() /** * Verify that given a table with a compactable tablet, and given a key extent range that is contained within a tablet's range, - * {@link TabletExtentChecker#checkTablets(String, String, Text, Text, boolean )} will return the full extent of the tablet. + * {@link TabletExtentChecker#checkTablets(String, String, String, String, String, Text, Text, boolean )} will return the full extent of the tablet. */ @Test void testInputInCompactableTabletRange() @@ -564,7 +571,7 @@ private void givenEnd(String end) { } private void assertResult() throws AccumuloException, TableNotFoundException, IOException { - List> result = TabletExtentChecker.checkTablets(clientPropertiesPath, TABLE_NAME, getText(begin), getText(end), mergeExtents); + List> result = TabletExtentChecker.checkTablets(client, TABLE_NAME, getText(begin), getText(end), mergeExtents); assertEquals(expectedExtents, result); } From a3c6e1716b87531e95ce8ce9a1248dd12ba22f7b Mon Sep 17 00:00:00 2001 From: tom-vt4 Date: Mon, 6 Jul 2026 12:26:26 -0400 Subject: [PATCH 03/11] Updated documentation and README --- .../check-tablet-extents/README.md | 16 ++++++++++++-- .../java/datawave/TabletExtentChecker.java | 21 +------------------ .../datawave/TabletExtentCheckerTest.java | 2 +- 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/contrib/datawave-utils/check-tablet-extents/README.md b/contrib/datawave-utils/check-tablet-extents/README.md index 85a5a3f2305..421a14d78f1 100644 --- a/contrib/datawave-utils/check-tablet-extents/README.md +++ b/contrib/datawave-utils/check-tablet-extents/README.md @@ -26,8 +26,13 @@ Use the `--help` option to list all available options: $ java -cp path/to/check-tablet-extents-shaded.jar datawave.TabletExtentChecker -h Usage: TabletExtentChecker [options] Options: + * -a, --accumulo-instance + The Accumulo instance. -b, --begin The starting row (exclusive) of the range of tablets to scan + -c, --compact + Compact the tablets + Default: false -e, --end The ending row (inclusive) of the range of tablets to scan -h, -?, --help, -help @@ -35,10 +40,15 @@ Usage: TabletExtentChecker [options] -m, --merge, --merge-extents Merges suggested compaction ranges for neighboring compactable tablets Default: false - * -p, --properties - The path to accumulo-client.properties + * -p, --password + The Accumulo password. Can be supplied from an environment variable via + env:ENV_VARIABLE * -t, --table The table name + * -u, --username + The Accumulo username. + * -z, --zookeper-instance + The Zookeeper instance. ``` The tool will list a set of recommended compaction commands if it finds any tablets that require compaction. For example, given a table `test_table` that has tablets with data outside the tablet extents for the following extents: @@ -67,3 +77,5 @@ SLF4J(W): See https://www.slf4j.org/codes.html#noProviders for further details. compact -t chop_test -b 2000 -e 5000 compact -t chop_test -b 8000 -e 9000 ``` + +If the option `--compact` was specified above, the tablets would be compacted. diff --git a/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java index b14e00518a7..726ebb663c3 100644 --- a/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java +++ b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java @@ -83,27 +83,9 @@ private static class Opts { @Parameter(names = {"-m", "--merge", "--merge-extents"}, description = "Merges suggested compaction ranges for neighboring compactable tablets") public boolean mergeExtents = false; - @Parameter(names = {"-c", "--compact"}, description = "Compact tablets that have been identified as candidates for needing compaction") + @Parameter(names = {"-c", "--compact"}, description = "Compact the tablets") public boolean compactTablets = false; - /** - * Validate the arguments in this {@link Opts}. - */ - public void validateArgs() { - if (accumuloInstance == null) { - throw new IllegalArgumentException("Accumulo instance name must be entered!"); - } - if (zookeeperInstance == null) { - throw new IllegalArgumentException("Zookeeper instance must be entered!"); - } - if (username == null) { - throw new IllegalArgumentException("Username must be entered!"); - } - if (password == null) { - throw new IllegalArgumentException("Password must be entered!"); - } - } - /** * Parse the given arguments array and populate this {@link Opts}. * @@ -144,7 +126,6 @@ public static void main(String[] args) throws AccumuloException, TableNotFoundEx // Parse the arguments. Opts opts = new Opts(); opts.parseArgs(args); - opts.validateArgs(); try (AccumuloClient client = Accumulo.newClient().to(opts.accumuloInstance, opts.zookeeperInstance).as(opts.username, opts.password).build()) { // Fetch the recommended tablet ranges to compact. List> compactionExtents = TabletExtentChecker.checkTablets(client, opts.tableName, opts.beginRow, opts.endRow, opts.mergeExtents); diff --git a/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java b/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java index b0550f20132..eedcea1ff8b 100644 --- a/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java +++ b/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java @@ -507,7 +507,7 @@ void testCompactableTabletOutsideInputRange() /** * Verify that given a table with a compactable tablet, and given a key extent range that is contained within a tablet's range, - * {@link TabletExtentChecker#checkTablets(String, String, String, String, String, Text, Text, boolean )} will return the full extent of the tablet. + * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} will return the full extent of the tablet. */ @Test void testInputInCompactableTabletRange() From 361245fe3321043b6501b5ec803111efd6c8a443 Mon Sep 17 00:00:00 2001 From: tom-vt4 Date: Mon, 6 Jul 2026 12:31:09 -0400 Subject: [PATCH 04/11] Remove unused variable --- .../src/test/java/datawave/TabletExtentCheckerTest.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java b/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java index eedcea1ff8b..7a2a2030e8e 100644 --- a/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java +++ b/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java @@ -2,9 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -import java.io.BufferedWriter; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; @@ -36,7 +34,6 @@ class TabletExtentCheckerTest { @TempDir private static Path tempDir; - private static String clientPropertiesPath; private static MiniAccumuloCluster mac; private AccumuloClient client; @@ -51,11 +48,6 @@ class TabletExtentCheckerTest { static void beforeAll() throws IOException, InterruptedException { mac = new MiniAccumuloCluster(tempDir.resolve("mac").toFile(), "secret"); mac.start(); - Path propertiesFile = tempDir.resolve("accumulo-client.properties"); - try (BufferedWriter out = Files.newBufferedWriter(propertiesFile)) { - mac.getClientProperties().store(out, null); - } - clientPropertiesPath = propertiesFile.toAbsolutePath().toString(); } @AfterAll From 99d8372ca30d59756d4b2b36e6cc6c81d152fb60 Mon Sep 17 00:00:00 2001 From: tom-vt4 Date: Mon, 6 Jul 2026 18:50:10 -0400 Subject: [PATCH 05/11] Try to implement AutoService class loading --- .../check-tablet-extents/pom.xml | 12 ++++----- .../java/datawave/TabletExtentChecker.java | 27 ++++++++++++++++--- warehouse/ingest-core/pom.xml | 17 ++++++++++++ 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/contrib/datawave-utils/check-tablet-extents/pom.xml b/contrib/datawave-utils/check-tablet-extents/pom.xml index ed7f6d84fc0..857ab4323d0 100644 --- a/contrib/datawave-utils/check-tablet-extents/pom.xml +++ b/contrib/datawave-utils/check-tablet-extents/pom.xml @@ -30,6 +30,12 @@ com.beust jcommander + + + com.google.auto.service + auto-service-annotations + 1.1.1 + com.google.guava guava @@ -73,12 +79,6 @@ org.apache.zookeeper zookeeper - - io.dropwizard.metrics - metrics-core - 4.1.12.1 - test - org.apache.accumulo accumulo-minicluster diff --git a/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java index 726ebb663c3..2e055c63fd7 100644 --- a/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java +++ b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java @@ -26,6 +26,7 @@ import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletsMetadata; import org.apache.accumulo.core.spi.crypto.NoCryptoServiceFactory; +import org.apache.accumulo.start.spi.KeywordExecutable; import org.apache.accumulo.tserver.tablet.Tablet; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.fs.FileSystem; @@ -35,11 +36,13 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; +import com.google.auto.service.AutoService; /** * A utility that will identify tablets that require compaction within a given table for a given range and lists recommended compaction commands. */ -public final class TabletExtentChecker { +@AutoService(KeywordExecutable.class) +public final class TabletExtentChecker implements KeywordExecutable { /** * Converts a string argument to a {@link Text} instance. @@ -52,7 +55,7 @@ public Text convert(String s) { } /** - * Represents a set of options that can parsed and used via {@link TabletExtentChecker#main(String[])}. + * Represents a set of options that can parsed and used via {@link TabletExtentChecker#execute(String[])}. */ private static class Opts { @Parameter(names = {"-h", "-?", "--help", "-help"}, help = true) @@ -110,6 +113,16 @@ public void parseArgs(String[] args) { } } + @Override + public String keyword() { + return "check-tablets"; + } + + @Override + public String description() { + return "A utility that will identify tablets that require compaction within a given table for a given range and lists recommended compaction commands."; + } + /** * Parses the user-provided arguments and prints out recommended ranges for compaction. * @@ -122,7 +135,8 @@ public void parseArgs(String[] args) { * @throws IOException * if an error occurs while reading the client properties file */ - public static void main(String[] args) throws AccumuloException, TableNotFoundException, IOException, AccumuloSecurityException { + @Override + public void execute(String[] args) throws AccumuloException, TableNotFoundException, IOException, AccumuloSecurityException { // Parse the arguments. Opts opts = new Opts(); opts.parseArgs(args); @@ -148,6 +162,13 @@ public static void main(String[] args) throws AccumuloException, TableNotFoundEx } } + // Redundant main usage should be avoided and may remove after testing. + /* + public static void main(String[] args) throws AccumuloException, TableNotFoundException, IOException, AccumuloSecurityException { + new TabletExtentChecker().execute(args); + } + */ + /** * Return the given value formatted as an argument snippet if the value is not null. * diff --git a/warehouse/ingest-core/pom.xml b/warehouse/ingest-core/pom.xml index 084753f6e1a..7156a882479 100644 --- a/warehouse/ingest-core/pom.xml +++ b/warehouse/ingest-core/pom.xml @@ -222,6 +222,11 @@ mockito-core test + + com.google.auto.service + auto-service-annotations + 1.1.1 + @@ -236,6 +241,18 @@ + + maven-compiler-plugin + + + + com.google.auto.service + auto-service + 1.1.1 + + + + From f41b514baac77167d258f38f5982d895ee1ec1f0 Mon Sep 17 00:00:00 2001 From: tom-vt4 Date: Mon, 6 Jul 2026 21:02:26 -0400 Subject: [PATCH 06/11] Refactor class loading and update documentation -Add main method back to allow usage of the tool with java -cp -Update README for usage of tool with accumulo command -Fix dependency issues --- .../check-tablet-extents/README.md | 59 ++++++++++++++++++- .../check-tablet-extents/pom.xml | 19 +++++- .../java/datawave/TabletExtentChecker.java | 6 -- warehouse/ingest-core/pom.xml | 17 ------ 4 files changed, 75 insertions(+), 26 deletions(-) diff --git a/contrib/datawave-utils/check-tablet-extents/README.md b/contrib/datawave-utils/check-tablet-extents/README.md index 421a14d78f1..b471c1e4f3c 100644 --- a/contrib/datawave-utils/check-tablet-extents/README.md +++ b/contrib/datawave-utils/check-tablet-extents/README.md @@ -20,10 +20,33 @@ The tool can be executed with the following command: $ java -cp path/to/check-tablet-extents-shaded.jar datawave.TabletExtentChecker ``` +To use the tool with an Accumulo command, copy the shaded jar to the Accumulo lib folder: + +```bash +$ cp path/to/check-tablet-extents-shaded.jar $ACCUMULO_HOME/lib +``` + +Then use the following command: + +```bash +$ accumulo check-tablets +``` + Use the `--help` option to list all available options: ```bash $ java -cp path/to/check-tablet-extents-shaded.jar datawave.TabletExtentChecker -h +``` + +or + +```bash +$ accumulo check-tablets -h +``` + +Output: + +```bash Usage: TabletExtentChecker [options] Options: * -a, --accumulo-instance @@ -51,6 +74,7 @@ Usage: TabletExtentChecker [options] The Zookeeper instance. ``` + The tool will list a set of recommended compaction commands if it finds any tablets that require compaction. For example, given a table `test_table` that has tablets with data outside the tablet extents for the following extents: - Start: "2000", End: "3000" - Start: "3000", End: "5000" @@ -59,7 +83,7 @@ The tool will list a set of recommended compaction commands if it finds any tabl You can expect the following recommended compaction commands. ```bash -$ java -cp target/check-tablet-extents-shaded.jar datawave.TabletExtentChecker -p $ACCUMULO_HOME/conf/accumulo-client.properties -t test_table +$ java -cp target/check-tablet-extents-shaded.jar datawave.TabletExtentChecker -a -z -u -p env: -t test_table SLF4J(W): No SLF4J providers were found. SLF4J(W): Defaulting to no-operation (NOP) logger implementation SLF4J(W): See https://www.slf4j.org/codes.html#noProviders for further details. @@ -68,6 +92,31 @@ compact -t chop_test -b 3000 -e 5000 compact -t chop_test -b 8000 -e 9000 ``` +or + +```bash +$ accumulo check-tablets -a -z -u -p env: -t test_table +SLF4J(W): Class path contains multiple SLF4J providers. +SLF4J(W): Found provider [org.apache.logging.slf4j.SLF4JServiceProvider@53bd815b] +SLF4J(W): Found provider [ch.qos.logback.classic.spi.LogbackServiceProvider@2401f4c3] +SLF4J(W): See https://www.slf4j.org/codes.html#multiple_bindings for an explanation. +SLF4J(I): Actual provider is of type [org.apache.logging.slf4j.SLF4JServiceProvider@53bd815b] +[bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.SnappyCodec +[bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.accumulo.core.file.rfile.bcfile.IdentityCodec +[bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.Lz4Codec +[bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.BZip2Codec +[bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.ZStandardCodec +[bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.LzoCodec +[bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +[zlib.ZlibFactory] INFO : Successfully loaded & initialized native-zlib library +[compress.CodecPool] INFO : Got brand-new decompressor [.deflate] +[bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +compact -t chop_test -b 2000 -e 3000 +compact -t chop_test -b 3000 -e 5000 +compact -t chop_test -b 8000 -e 9000 +``` + + If the option `--merge-extents` was specified above, neighboring compactable tablet extents will be merged within the recommended compaction commands: ```bash $ java -cp target/check-tablet-extents-shaded.jar datawave.TabletExtentChecker -p $ACCUMULO_HOME/conf/accumulo-client.properties -t test_table @@ -78,4 +127,10 @@ compact -t chop_test -b 2000 -e 5000 compact -t chop_test -b 8000 -e 9000 ``` -If the option `--compact` was specified above, the tablets would be compacted. +If the options `-b 3000` and `-e null` were specified above, only the suggestions for the rows ranging from 3000 to positive infinity would be displayed: +```bash +compact -t chop_test -b 3000 -e 5000 +compact -t chop_test -b 8000 -e 9000 +``` + +If the option `--compact` was specified above, the tablets would be compacted. \ No newline at end of file diff --git a/contrib/datawave-utils/check-tablet-extents/pom.xml b/contrib/datawave-utils/check-tablet-extents/pom.xml index 857ab4323d0..79b06182e6c 100644 --- a/contrib/datawave-utils/check-tablet-extents/pom.xml +++ b/contrib/datawave-utils/check-tablet-extents/pom.xml @@ -30,7 +30,6 @@ com.beust jcommander - com.google.auto.service auto-service-annotations @@ -79,6 +78,12 @@ org.apache.zookeeper zookeeper + + io.dropwizard.metrics + metrics-core + 4.1.12.1 + test + org.apache.accumulo accumulo-minicluster @@ -114,6 +119,18 @@ true + + maven-compiler-plugin + + + + com.google.auto.service + auto-service + 1.1.1 + + + + org.apache.maven.plugins maven-dependency-plugin diff --git a/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java index 2e055c63fd7..66365456c2b 100644 --- a/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java +++ b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java @@ -162,12 +162,9 @@ public void execute(String[] args) throws AccumuloException, TableNotFoundExcept } } - // Redundant main usage should be avoided and may remove after testing. - /* public static void main(String[] args) throws AccumuloException, TableNotFoundException, IOException, AccumuloSecurityException { new TabletExtentChecker().execute(args); } - */ /** * Return the given value formatted as an argument snippet if the value is not null. @@ -254,14 +251,11 @@ static List> checkTablets(AccumuloClient client, String tableNam } } } - // Handle case where last tablet needs compaction when we are merging extents. if (mergeExtents && foundCompactableTablet) { compactionExtents.add(Pair.of(compactionStart, compactionEnd)); } - } - return compactionExtents; } diff --git a/warehouse/ingest-core/pom.xml b/warehouse/ingest-core/pom.xml index 7156a882479..084753f6e1a 100644 --- a/warehouse/ingest-core/pom.xml +++ b/warehouse/ingest-core/pom.xml @@ -222,11 +222,6 @@ mockito-core test - - com.google.auto.service - auto-service-annotations - 1.1.1 - @@ -241,18 +236,6 @@ - - maven-compiler-plugin - - - - com.google.auto.service - auto-service - 1.1.1 - - - - From dc9474c1e278f904e9c5db9035bd1b04e23f1eca Mon Sep 17 00:00:00 2001 From: tom-vt4 Date: Wed, 8 Jul 2026 17:43:12 -0400 Subject: [PATCH 07/11] Use ServerContext and remove shaded-jar profile -Replace AccumuloClient usage with ServerContext -Test class requires refactoring for ServerContext -Removed create-shade-jar profile to use regular jar --- .../check-tablet-extents/pom.xml | 69 ------------------- .../java/datawave/TabletExtentChecker.java | 62 ++++++++--------- .../datawave/TabletExtentCheckerTest.java | 32 +++++---- 3 files changed, 45 insertions(+), 118 deletions(-) diff --git a/contrib/datawave-utils/check-tablet-extents/pom.xml b/contrib/datawave-utils/check-tablet-extents/pom.xml index 79b06182e6c..b9674d3d6ce 100644 --- a/contrib/datawave-utils/check-tablet-extents/pom.xml +++ b/contrib/datawave-utils/check-tablet-extents/pom.xml @@ -139,73 +139,4 @@ - - - create-shade-jar - - - - org.apache.maven.plugins - maven-shade-plugin - - - check-tablet-extents-shade-jar - - shade - - package - - false - ${project.artifactId}-shaded - true - shaded - - - org.apache.accumulo:accumulo-native - jakarta.activation:jakarta.activation-api:jar: - org.apache.hadoop:hadoop-annotations:jar: - org.apache.hadoop:hadoop-auth:jar: - org.apache.hadoop:hadoop-mapreduce-client-common:jar: - org.apache.hadoop:hadoop-yarn-api:jar: - org.apache.hadoop:hadoop-yarn-common:jar: - org.apache.hadoop:hadoop-mapreduce-client-core:jar: - org.apache.hadoop:hadoop-common:jar: - org.apache.hadoop:hadoop-mapreduce-client-jobclient:jar: - org.apache.hadoop:hadoop-yarn-client:jar: - org.apache.hadoop:hadoop-hdfs-client:jar: - org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:jar: - - - - - *:* - - module-info.class - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - *:* - - **/Log4j2Plugins.dat - - - - - - - - datawave.TabletExtentChecker - - - - - - - - - - diff --git a/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java index 66365456c2b..4776b72cc96 100644 --- a/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java +++ b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java @@ -8,14 +8,11 @@ import java.util.Map; import java.util.Set; -import org.apache.accumulo.core.cli.ClientOpts; -import org.apache.accumulo.core.client.Accumulo; -import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.cli.ConfigOpts; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.admin.TableOperations; -import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.TableId; @@ -26,6 +23,7 @@ import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletsMetadata; import org.apache.accumulo.core.spi.crypto.NoCryptoServiceFactory; +import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.start.spi.KeywordExecutable; import org.apache.accumulo.tserver.tablet.Tablet; import org.apache.commons.lang3.tuple.Pair; @@ -56,24 +54,9 @@ public Text convert(String s) { /** * Represents a set of options that can parsed and used via {@link TabletExtentChecker#execute(String[])}. + * ConfigOpts provides getSiteConfiguration() to be used to create ServerContext */ - private static class Opts { - @Parameter(names = {"-h", "-?", "--help", "-help"}, help = true) - public boolean help = false; - - @Parameter(names = {"-a", "--accumulo-instance"}, description = "The Accumulo instance.", required = true) - public String accumuloInstance = null; - - @Parameter(names = {"-z", "--zookeper-instance"}, description = "The Zookeeper instance.", required = true) - public String zookeeperInstance = null; - - @Parameter(names = {"-u", "--username"}, description = "The Accumulo username.", required = true) - public String username = null; - - @Parameter(names = {"-p", "--password"}, description = "The Accumulo password. Can be supplied from an environment variable via env:ENV_VARIABLE", - converter = ClientOpts.PasswordConverter.class, required = true) - public String password = null; - + private static class Opts extends ConfigOpts { @Parameter(names = {"-t", "--table"}, description = "The table name", required = true) public String tableName = null; @@ -95,7 +78,8 @@ private static class Opts { * @param args * the arguments to parse. */ - public void parseArgs(String[] args) { + @Override + public void parseArgs(String programName, String[] args, Object... others) { JCommander commander = JCommander.newBuilder().addObject(this).programName(TabletExtentChecker.class.getSimpleName()).build(); try { @@ -139,10 +123,17 @@ public String description() { public void execute(String[] args) throws AccumuloException, TableNotFoundException, IOException, AccumuloSecurityException { // Parse the arguments. Opts opts = new Opts(); - opts.parseArgs(args); - try (AccumuloClient client = Accumulo.newClient().to(opts.accumuloInstance, opts.zookeeperInstance).as(opts.username, opts.password).build()) { + + opts.parseArgs(TabletExtentChecker.class.getName(), args); + + if (opts.getPropertiesPath() == null) { + System.out.println("ERROR: null props path"); + System.exit(1); + } + System.out.println("Site Configuration: " + opts.getSiteConfiguration()); + try (ServerContext context = new ServerContext(opts.getSiteConfiguration())) { // Fetch the recommended tablet ranges to compact. - List> compactionExtents = TabletExtentChecker.checkTablets(client, opts.tableName, opts.beginRow, opts.endRow, opts.mergeExtents); + List> compactionExtents = TabletExtentChecker.checkTablets(context, opts.tableName, opts.beginRow, opts.endRow, opts.mergeExtents); // Print a message when compactionExtents is empty if (compactionExtents.isEmpty()) { @@ -153,7 +144,7 @@ public void execute(String[] args) throws AccumuloException, TableNotFoundExcept Text endRow = pair.getRight(); if (opts.compactTablets) { System.out.println("Compacting range from " + startRow + "-" + endRow); - client.tableOperations().compact(opts.tableName, startRow, endRow, true, true); + context.tableOperations().compact(opts.tableName, startRow, endRow, true, true); } else { System.out.println("compact -t " + opts.tableName + formatArg("-b", startRow) + formatArg("-e", endRow)); } @@ -202,12 +193,11 @@ private static String formatArg(String arg, Text value) { * @throws IOException * if an error occurs while reading the client properties file */ - static List> checkTablets(AccumuloClient client, String tableName, Text begin, Text end, boolean mergeExtents) + static List> checkTablets(ServerContext context, String tableName, Text begin, Text end, boolean mergeExtents) throws AccumuloException, TableNotFoundException, IOException { List> compactionExtents = new ArrayList<>(); - TableId tableId = getTableId(client, tableName); - ClientContext context = (ClientContext) client; + TableId tableId = getTableId(context, tableName); // Fetch the metadata for all tablets in the given table whose extents overlap with the user provided range of tablets to scan. try (TabletsMetadata tablets = context.getAmple().readTablets().forTable(tableId).overlapping(begin, false, end) .fetch(TabletMetadata.ColumnType.PREV_ROW, TabletMetadata.ColumnType.FILES).build()) { @@ -262,14 +252,14 @@ static List> checkTablets(AccumuloClient client, String tableNam /** * Return the table ID for the given table * - * @param client - * the client to use when connecting to Accumulo + * @param context + * the context to use when connecting to Accumulo * @param tableName * the table name * @return the table ID */ - private static TableId getTableId(AccumuloClient client, String tableName) { - TableOperations tableOperations = client.tableOperations(); + private static TableId getTableId(ServerContext context, String tableName) { + TableOperations tableOperations = context.tableOperations(); if (tableOperations.exists(tableName)) { return TableId.of(tableOperations.tableIdMap().get(tableName)); } else { @@ -291,7 +281,7 @@ private static TableId getTableId(AccumuloClient client, String tableName) { * the tablet metadata * @return true if the tablet requires compaction, or false */ - private static boolean tabletRequiresCompaction(ClientContext context, String tableName, TabletMetadata tablet) + private static boolean tabletRequiresCompaction(ServerContext context, String tableName, TabletMetadata tablet) throws AccumuloException, TableNotFoundException, IOException { // Fetch the list of RFiles for the tablet. ConfigurationCopy tableConf = new ConfigurationCopy(context.tableOperations().getConfiguration(tableName)); @@ -310,6 +300,10 @@ private static boolean tabletRequiresCompaction(ClientContext context, String ta // A tablet requires a compaction if any of the following are true: // - The first key is outside the tablet's extent. // - The last key is outside the tablet's extent. + System.out.println("Extent: " + extent); + System.out.println("First key: " + first); + System.out.println("Last key: " + last); + if ((first != null && !extent.contains(first.getRow())) || (last != null && !extent.contains(last.getRow()))) { return true; } diff --git a/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java b/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java index 7a2a2030e8e..42b0e64e0b4 100644 --- a/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java +++ b/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java @@ -19,6 +19,7 @@ import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.minicluster.MiniAccumuloCluster; +import org.apache.accumulo.server.ServerContext; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.AfterAll; @@ -90,7 +91,7 @@ void setUp() { */ @Test void testAllSingleTabletsNeedCompaction() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) @@ -110,7 +111,7 @@ void testAllSingleTabletsNeedCompaction() */ @Test void testOnlyFirstTabletNeedsCompaction() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) @@ -131,7 +132,7 @@ void testOnlyFirstTabletNeedsCompaction() */ @Test void testOnlyLastTabletNeedsCompaction() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) addSplits("2500", "4000", "7000"); @@ -166,7 +167,7 @@ void testNoTabletsNeedCompaction() throws AccumuloException, TableNotFoundExcept */ @Test void testOnlyFirstAndLastTabletsNeedCompaction() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); addSplits("2500", "4000", "7000"); @@ -225,7 +226,7 @@ void testMiddleTabletNeedsCompaction() throws AccumuloException, TableNotFoundEx */ @Test void testMultipleInnerCompactionRanges() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); addSplits("1000", "1500", "2000", "2500", "3000", "4000", "5500", "6500", "7000", "7500", "8500", "9000"); // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), @@ -262,7 +263,7 @@ void testMultipleInnerCompactionRanges() */ @Test void testCompactableTabletOutsideInputRange() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), @@ -283,7 +284,7 @@ void testCompactableTabletOutsideInputRange() */ @Test void testInputInCompactableTabletRange() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), @@ -338,7 +339,7 @@ void testAllTabletsNeedCompaction() throws AccumuloException, TableExistsExcepti */ @Test void testOnlyFirstTabletNeedsCompaction() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) @@ -358,7 +359,7 @@ void testOnlyFirstTabletNeedsCompaction() */ @Test void testOnlyLastTabletNeedsCompaction() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) @@ -395,7 +396,7 @@ void testNoTabletsNeedCompaction() throws AccumuloException, TableNotFoundExcept */ @Test void testOnlyFirstAndLastTabletsNeedCompaction() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) @@ -455,7 +456,7 @@ void testMiddleTabletNeedsCompaction() throws AccumuloException, TableNotFoundEx */ @Test void testMultipleInnerCompactionRanges() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), @@ -481,7 +482,7 @@ void testMultipleInnerCompactionRanges() */ @Test void testCompactableTabletOutsideInputRange() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), @@ -503,7 +504,7 @@ void testCompactableTabletOutsideInputRange() */ @Test void testInputInCompactableTabletRange() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), @@ -563,8 +564,9 @@ private void givenEnd(String end) { } private void assertResult() throws AccumuloException, TableNotFoundException, IOException { - List> result = TabletExtentChecker.checkTablets(client, TABLE_NAME, getText(begin), getText(end), mergeExtents); + //broken ServerContext cast to be changed + List> result = TabletExtentChecker.checkTablets((ServerContext)(client), TABLE_NAME, getText(begin), getText(end), mergeExtents); assertEquals(expectedExtents, result); } -} +} \ No newline at end of file From 83ececa66c94f752cce8eec1bf418fa3e60acc9c Mon Sep 17 00:00:00 2001 From: tom-vt4 Date: Fri, 10 Jul 2026 06:28:05 -0400 Subject: [PATCH 08/11] Refactor main and test class to use ClientContext -Use ClientContext instead of ServerContext to allow test suite to create MiniAccumuloCluster for testing -Refactor original checkTablets() to findCompactableTablets() to handle extent tracking -Add new checkTablets() method to handle tool output and parameter based compaction -Update documentation -Update error logging --- .../check-tablet-extents/README.md | 238 +++++++++++++----- .../java/datawave/TabletExtentChecker.java | 53 ++-- .../datawave/TabletExtentCheckerTest.java | 165 ++++++++---- 3 files changed, 319 insertions(+), 137 deletions(-) diff --git a/contrib/datawave-utils/check-tablet-extents/README.md b/contrib/datawave-utils/check-tablet-extents/README.md index b471c1e4f3c..68dfe089bf4 100644 --- a/contrib/datawave-utils/check-tablet-extents/README.md +++ b/contrib/datawave-utils/check-tablet-extents/README.md @@ -1,47 +1,48 @@ # Tablet Extent Checker -This project contains a tool for identifying tablets in a table that are candidates for compaction. +This project contains a server utility tool for identifying tablets in a table that are candidates for compaction. ## Building the tool Building the tool is done with the following command. To run unit tests, omit `-DskipTests`. ```bash -$ mvn clean install -DskipTests -P create-shade-jar +$ mvn clean install -DskipTests ``` -This will create the executable jar `target/create-tablet-extents-shaded.jar`. +This will create the executable jar `target/check-tablet-extents.jar`. ## Running the tool -The tool can be executed with the following command: +Copy the jar to the Accumulo lib folder: ```bash -$ java -cp path/to/check-tablet-extents-shaded.jar datawave.TabletExtentChecker +$ cp path/to/check-tablet-extents.jar $ACCUMULO_HOME/lib ``` -To use the tool with an Accumulo command, copy the shaded jar to the Accumulo lib folder: +Then use the following command: ```bash -$ cp path/to/check-tablet-extents-shaded.jar $ACCUMULO_HOME/lib +$ accumulo check-tablets ``` -Then use the following command: +or ```bash -$ accumulo check-tablets +$ accumulo datawave.TabletExtentChecker ``` + Use the `--help` option to list all available options: ```bash -$ java -cp path/to/check-tablet-extents-shaded.jar datawave.TabletExtentChecker -h +$ accumulo check-tablets -h ``` or ```bash -$ accumulo check-tablets -h +$ accumulo datawave.TabletExtentChecker -h ``` Output: @@ -49,8 +50,6 @@ Output: ```bash Usage: TabletExtentChecker [options] Options: - * -a, --accumulo-instance - The Accumulo instance. -b, --begin The starting row (exclusive) of the range of tablets to scan -c, --compact @@ -63,74 +62,187 @@ Usage: TabletExtentChecker [options] -m, --merge, --merge-extents Merges suggested compaction ranges for neighboring compactable tablets Default: false - * -p, --password - The Accumulo password. Can be supplied from an environment variable via - env:ENV_VARIABLE + -p, -props, --props + Sets path to accumulo.properties.The classpath will be searched if this + property is not set * -t, --table The table name - * -u, --username - The Accumulo username. - * -z, --zookeper-instance - The Zookeeper instance. + -o + Overrides configuration set in accumulo.properties (but NOT system-wide + config set in Zookeeper). Expected format: -o = + Default: [] ``` -The tool will list a set of recommended compaction commands if it finds any tablets that require compaction. For example, given a table `test_table` that has tablets with data outside the tablet extents for the following extents: -- Start: "2000", End: "3000" -- Start: "3000", End: "5000" -- Start: "8000", End: "9000" +The tool will list a set of recommended compaction commands if it finds any tablets that require compaction. +For example, given a table `chop_test` that has tablets with data outside the tablet extents for the following extents: + +- Start: null, End: "2500" +- Start: "2500", End: "5000" +- Start: "5000", End: "7500" +- Start: "7500", End: null -You can expect the following recommended compaction commands. +You can expect the following output: ```bash -$ java -cp target/check-tablet-extents-shaded.jar datawave.TabletExtentChecker -a -z -u -p env: -t test_table -SLF4J(W): No SLF4J providers were found. -SLF4J(W): Defaulting to no-operation (NOP) logger implementation -SLF4J(W): See https://www.slf4j.org/codes.html#noProviders for further details. -compact -t chop_test -b 2000 -e 3000 -compact -t chop_test -b 3000 -e 5000 -compact -t chop_test -b 8000 -e 9000 +$ accumulo check-tablets -p $ACCUMULO_HOME/conf/accumulo.properties -t chop_test +2026-07-10T09:17:10,431 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.ZStandardCodec +2026-07-10T09:17:10,433 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.LzoCodec +2026-07-10T09:17:10,434 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.BZip2Codec +2026-07-10T09:17:10,436 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.accumulo.core.file.rfile.bcfile.IdentityCodec +2026-07-10T09:17:10,437 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.Lz4Codec +2026-07-10T09:17:10,438 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.SnappyCodec +2026-07-10T09:17:10,439 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +2026-07-10T09:17:10,441 [compress.CodecPool] INFO : Got brand-new compressor [.identity] +2026-07-10T09:17:10,658 [zlib.ZlibFactory] INFO : Successfully loaded & initialized native-zlib library +2026-07-10T09:17:10,658 [compress.CodecPool] INFO : Got brand-new decompressor [.deflate] +2026-07-10T09:17:10,659 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +Extent: 1;2500< +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1;5000;2500 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1;7500;5000 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1<;7500 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +compact -t chop_test -e 2500 +compact -t chop_test -b 2500 -e 5000 +compact -t chop_test -b 5000 -e 7500 +compact -t chop_test -b 7500 +2026-07-10T09:17:10,709 [metrics.MetricsInfoImpl] INFO : micrometer metrics enabled: false +2026-07-10T09:17:10,709 [metrics.MetricsInfoImpl] INFO : Closing metrics registry ``` +The tool recommended multiple compaction ranges for the table. -or +If the option `--merge-extents` is specified, neighboring compactable tablet extents will be merged within the recommended compaction commands ```bash -$ accumulo check-tablets -a -z -u -p env: -t test_table -SLF4J(W): Class path contains multiple SLF4J providers. -SLF4J(W): Found provider [org.apache.logging.slf4j.SLF4JServiceProvider@53bd815b] -SLF4J(W): Found provider [ch.qos.logback.classic.spi.LogbackServiceProvider@2401f4c3] -SLF4J(W): See https://www.slf4j.org/codes.html#multiple_bindings for an explanation. -SLF4J(I): Actual provider is of type [org.apache.logging.slf4j.SLF4JServiceProvider@53bd815b] -[bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.SnappyCodec -[bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.accumulo.core.file.rfile.bcfile.IdentityCodec -[bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.Lz4Codec -[bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.BZip2Codec -[bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.ZStandardCodec -[bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.LzoCodec -[bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec -[zlib.ZlibFactory] INFO : Successfully loaded & initialized native-zlib library -[compress.CodecPool] INFO : Got brand-new decompressor [.deflate] -[bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec -compact -t chop_test -b 2000 -e 3000 -compact -t chop_test -b 3000 -e 5000 -compact -t chop_test -b 8000 -e 9000 +accumulo check-tablets -p $ACCUMULO_HOME/conf/accumulo.properties -t chop_test --merge-extents +2026-07-10T09:42:21,296 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.accumulo.core.file.rfile.bcfile.IdentityCodec +2026-07-10T09:42:21,307 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.BZip2Codec +2026-07-10T09:42:21,308 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.LzoCodec +2026-07-10T09:42:21,309 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +2026-07-10T09:42:21,311 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.ZStandardCodec +2026-07-10T09:42:21,312 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.SnappyCodec +2026-07-10T09:42:21,313 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.Lz4Codec +2026-07-10T09:42:21,316 [compress.CodecPool] INFO : Got brand-new compressor [.identity] +2026-07-10T09:42:21,527 [zlib.ZlibFactory] INFO : Successfully loaded & initialized native-zlib library +2026-07-10T09:42:21,527 [compress.CodecPool] INFO : Got brand-new decompressor [.deflate] +2026-07-10T09:42:21,528 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +Extent: 1;2500< +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1;5000;2500 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1;7500;5000 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1<;7500 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +compact -t chop_test +2026-07-10T09:42:21,578 [metrics.MetricsInfoImpl] INFO : micrometer metrics enabled: false +2026-07-10T09:42:21,578 [metrics.MetricsInfoImpl] INFO : Closing metrics registry ``` +The compaction ranges with contiguous tablets have been combined. The tool recommended one command, `compact -t chop_test` which will compact the entire chop_test table. -If the option `--merge-extents` was specified above, neighboring compactable tablet extents will be merged within the recommended compaction commands: +If the options `-b 2500` and `-e 7500` are specified, only the suggestions for the rows ranging from 2500 to 7500 will be displayed: ```bash -$ java -cp target/check-tablet-extents-shaded.jar datawave.TabletExtentChecker -p $ACCUMULO_HOME/conf/accumulo-client.properties -t test_table -SLF4J(W): No SLF4J providers were found. -SLF4J(W): Defaulting to no-operation (NOP) logger implementation -SLF4J(W): See https://www.slf4j.org/codes.html#noProviders for further details. -compact -t chop_test -b 2000 -e 5000 -compact -t chop_test -b 8000 -e 9000 +accumulo check-tablets -p $ACCUMULO_HOME/conf/accumulo.properties -t chop_test -b 2500 -e 7500 +2026-07-10T09:46:22,218 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.accumulo.core.file.rfile.bcfile.IdentityCodec +2026-07-10T09:46:22,231 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.BZip2Codec +2026-07-10T09:46:22,233 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +2026-07-10T09:46:22,234 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.LzoCodec +2026-07-10T09:46:22,238 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.Lz4Codec +2026-07-10T09:46:22,240 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.SnappyCodec +2026-07-10T09:46:22,243 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.ZStandardCodec +2026-07-10T09:46:22,248 [compress.CodecPool] INFO : Got brand-new compressor [.identity] +2026-07-10T09:46:22,461 [zlib.ZlibFactory] INFO : Successfully loaded & initialized native-zlib library +2026-07-10T09:46:22,461 [compress.CodecPool] INFO : Got brand-new decompressor [.deflate] +2026-07-10T09:46:22,461 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +Extent: 1;5000;2500 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1;7500;5000 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +compact -t chop_test -b 2500 -e 5000 +compact -t chop_test -b 5000 -e 7500 +2026-07-10T09:46:22,487 [metrics.MetricsInfoImpl] INFO : micrometer metrics enabled: false +2026-07-10T09:46:22,487 [metrics.MetricsInfoImpl] INFO : Closing metrics registry ``` +If the --merge-extent option was used as well, the tool would have combined the extents and suggested: `compact -t chop_test -b 2500 -e 7500`. + -If the options `-b 3000` and `-e null` were specified above, only the suggestions for the rows ranging from 3000 to positive infinity would be displayed: +If the option `--compact` is specified, the tablets would be compacted automatically: ```bash -compact -t chop_test -b 3000 -e 5000 -compact -t chop_test -b 8000 -e 9000 +accumulo check-tablets -p $ACCUMULO_HOME/conf/accumulo.properties -t chop_test -c +2026-07-10T10:03:56,040 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.LzoCodec +2026-07-10T10:03:56,042 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.BZip2Codec +2026-07-10T10:03:56,043 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.SnappyCodec +2026-07-10T10:03:56,045 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.Lz4Codec +2026-07-10T10:03:56,046 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.accumulo.core.file.rfile.bcfile.IdentityCodec +2026-07-10T10:03:56,047 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.ZStandardCodec +2026-07-10T10:03:56,048 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +2026-07-10T10:03:56,050 [compress.CodecPool] INFO : Got brand-new compressor [.identity] +2026-07-10T10:03:56,266 [zlib.ZlibFactory] INFO : Successfully loaded & initialized native-zlib library +2026-07-10T10:03:56,266 [compress.CodecPool] INFO : Got brand-new decompressor [.deflate] +2026-07-10T10:03:56,266 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +Extent: 1;2500< +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1;5000;2500 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1;7500;5000 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1<;7500 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Compacting range from null-2500 +Compacting range from 2500-5000 +Compacting range from 5000-7500 +Compacting range from 7500-null +2026-07-10T10:03:59,007 [metrics.MetricsInfoImpl] INFO : micrometer metrics enabled: false +2026-07-10T10:03:59,007 [metrics.MetricsInfoImpl] INFO : Closing metrics registry ``` -If the option `--compact` was specified above, the tablets would be compacted. \ No newline at end of file + +We can verify that the compaction was successful by running the tool again: +```bash +accumulo check-tablets -p $ACCUMULO_HOME/conf/accumulo.properties -t chop_test +2026-07-10T10:08:17,880 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +2026-07-10T10:08:17,892 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.LzoCodec +2026-07-10T10:08:17,893 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.SnappyCodec +2026-07-10T10:08:17,895 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.accumulo.core.file.rfile.bcfile.IdentityCodec +2026-07-10T10:08:17,895 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.ZStandardCodec +2026-07-10T10:08:17,896 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.BZip2Codec +2026-07-10T10:08:17,898 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.Lz4Codec +2026-07-10T10:08:17,900 [compress.CodecPool] INFO : Got brand-new compressor [.identity] +2026-07-10T10:08:18,123 [zlib.ZlibFactory] INFO : Successfully loaded & initialized native-zlib library +2026-07-10T10:08:18,123 [compress.CodecPool] INFO : Got brand-new decompressor [.deflate] +2026-07-10T10:08:18,124 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +Extent: 1;2500< +First key: 0000 cf:cq [] 1783674529729 false +Last key: 2500 cf:cq [] 1783674529729 false +Extent: 1;5000;2500 +First key: 2501 cf:cq [] 1783674529729 false +Last key: 5000 cf:cq [] 1783674529729 false +Extent: 1;7500;5000 +First key: 5001 cf:cq [] 1783674529729 false +Last key: 7500 cf:cq [] 1783674529729 false +Extent: 1<;7500 +First key: 7501 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +No candidates suitable for compaction. +2026-07-10T10:08:18,191 [metrics.MetricsInfoImpl] INFO : micrometer metrics enabled: false +2026-07-10T10:08:18,191 [metrics.MetricsInfoImpl] INFO : Closing metrics registry +``` +We can see that the tablets now only contain rows within their respective extents, and the tool outputs a message that there are no compactable tablets. \ No newline at end of file diff --git a/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java index 4776b72cc96..511bb2d368b 100644 --- a/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java +++ b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java @@ -13,6 +13,7 @@ import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.admin.TableOperations; +import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.TableId; @@ -53,8 +54,8 @@ public Text convert(String s) { } /** - * Represents a set of options that can parsed and used via {@link TabletExtentChecker#execute(String[])}. - * ConfigOpts provides getSiteConfiguration() to be used to create ServerContext + * Represents a set of options that can parsed and used via {@link TabletExtentChecker#execute(String[])}. ConfigOpts provides getSiteConfiguration() to be + * used to create ServerContext */ private static class Opts extends ConfigOpts { @Parameter(names = {"-t", "--table"}, description = "The table name", required = true) @@ -85,8 +86,8 @@ public void parseArgs(String programName, String[] args, Object... others) { try { commander.parse(args); } catch (ParameterException ex) { + System.err.println("ERROR :"+ex.getMessage()); commander.usage(); - System.err.println(ex.getMessage()); System.exit(1); } @@ -130,26 +131,9 @@ public void execute(String[] args) throws AccumuloException, TableNotFoundExcept System.out.println("ERROR: null props path"); System.exit(1); } - System.out.println("Site Configuration: " + opts.getSiteConfiguration()); try (ServerContext context = new ServerContext(opts.getSiteConfiguration())) { // Fetch the recommended tablet ranges to compact. - List> compactionExtents = TabletExtentChecker.checkTablets(context, opts.tableName, opts.beginRow, opts.endRow, opts.mergeExtents); - - // Print a message when compactionExtents is empty - if (compactionExtents.isEmpty()) { - System.out.println("No candidates suitable for compaction."); - } else { - for (Pair pair : compactionExtents) { - Text startRow = pair.getLeft(); - Text endRow = pair.getRight(); - if (opts.compactTablets) { - System.out.println("Compacting range from " + startRow + "-" + endRow); - context.tableOperations().compact(opts.tableName, startRow, endRow, true, true); - } else { - System.out.println("compact -t " + opts.tableName + formatArg("-b", startRow) + formatArg("-e", endRow)); - } - } - } + checkTablets(context, opts.tableName, opts.beginRow, opts.endRow, opts.mergeExtents, opts.compactTablets); } } @@ -193,7 +177,7 @@ private static String formatArg(String arg, Text value) { * @throws IOException * if an error occurs while reading the client properties file */ - static List> checkTablets(ServerContext context, String tableName, Text begin, Text end, boolean mergeExtents) + static List> findCompactableTablets(ClientContext context, String tableName, Text begin, Text end, boolean mergeExtents) throws AccumuloException, TableNotFoundException, IOException { List> compactionExtents = new ArrayList<>(); @@ -258,7 +242,7 @@ static List> checkTablets(ServerContext context, String tableNam * the table name * @return the table ID */ - private static TableId getTableId(ServerContext context, String tableName) { + private static TableId getTableId(ClientContext context, String tableName) { TableOperations tableOperations = context.tableOperations(); if (tableOperations.exists(tableName)) { return TableId.of(tableOperations.tableIdMap().get(tableName)); @@ -281,7 +265,7 @@ private static TableId getTableId(ServerContext context, String tableName) { * the tablet metadata * @return true if the tablet requires compaction, or false */ - private static boolean tabletRequiresCompaction(ServerContext context, String tableName, TabletMetadata tablet) + private static boolean tabletRequiresCompaction(ClientContext context, String tableName, TabletMetadata tablet) throws AccumuloException, TableNotFoundException, IOException { // Fetch the list of RFiles for the tablet. ConfigurationCopy tableConf = new ConfigurationCopy(context.tableOperations().getConfiguration(tableName)); @@ -311,4 +295,25 @@ private static boolean tabletRequiresCompaction(ServerContext context, String ta } return false; } + + public static void checkTablets(ClientContext context, String tableName, Text begin, Text end, boolean mergeExtents, boolean compactTablets) + throws AccumuloException, TableNotFoundException, IOException, AccumuloSecurityException { + List> compactionExtents = findCompactableTablets(context, tableName, begin, end, mergeExtents); + + // Print a message when compactionExtents is empty + if (compactionExtents.isEmpty()) { + System.out.println("No candidates suitable for compaction."); + } else { + for (Pair pair : compactionExtents) { + Text startRow = pair.getLeft(); + Text endRow = pair.getRight(); + if (compactTablets) { + System.out.println("Compacting range from " + startRow + "-" + endRow); + context.tableOperations().compact(tableName, startRow, endRow, true, true); + } else { + System.out.println("compact -t " + tableName + formatArg("-b", startRow) + formatArg("-e", endRow)); + } + } + } + } } diff --git a/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java b/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java index 42b0e64e0b4..cb236d180fe 100644 --- a/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java +++ b/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java @@ -17,9 +17,9 @@ import org.apache.accumulo.core.client.TableExistsException; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.minicluster.MiniAccumuloCluster; -import org.apache.accumulo.server.ServerContext; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.AfterAll; @@ -74,8 +74,8 @@ void tearDown() throws AccumuloException, TableNotFoundException, AccumuloSecuri } /** - * Contains test methods that verify {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} tracks individual unmerged - * tablet extents given a table, tablet range, and mergeExtents == false. + * Contains test methods that verify {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} tracks individual + * unmerged tablet extents given a table, tablet range, and mergeExtents == false. */ @Nested class MergeExtentsDisabledTests { @@ -87,11 +87,12 @@ void setUp() { /** * Verify that given a table where all tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns multiple ranges to compact each tablet individually. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns multiple ranges to compact each tablet + * individually. */ @Test void testAllSingleTabletsNeedCompaction() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) @@ -106,12 +107,12 @@ void testAllSingleTabletsNeedCompaction() /** * Verify that given a table where only the first tablet requires compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns a single range that would compact only the first - * tablet. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns a single range that would compact only the + * first tablet. */ @Test void testOnlyFirstTabletNeedsCompaction() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) @@ -127,12 +128,12 @@ void testOnlyFirstTabletNeedsCompaction() /** * Verify that given a table where only the last tablet requires compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns a single range that would compact only the last - * tablet. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns a single range that would compact only the + * last tablet. */ @Test void testOnlyLastTabletNeedsCompaction() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) addSplits("2500", "4000", "7000"); @@ -146,7 +147,7 @@ void testOnlyLastTabletNeedsCompaction() /** * Verify that given a table where no tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns an empty list. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns an empty list. */ @Test void testNoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { @@ -162,12 +163,12 @@ void testNoTabletsNeedCompaction() throws AccumuloException, TableNotFoundExcept /** * Verify that given a table where only the first and last tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns separate ranges for the first and last compactable - * tablets. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns separate ranges for the first and last + * compactable tablets. */ @Test void testOnlyFirstAndLastTabletsNeedCompaction() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); addSplits("2500", "4000", "7000"); @@ -184,7 +185,7 @@ void testOnlyFirstAndLastTabletsNeedCompaction() /** * Verify that given a table where only the last two tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns separate ranges for those tablets. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns separate ranges for those tablets. */ @Test void testLastTwoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { @@ -203,7 +204,7 @@ void testLastTwoTabletsNeedCompaction() throws AccumuloException, TableNotFoundE /** * Verify that given a table where only the middle tablet requires compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns a range to compact that middle tablet. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns a range to compact that middle tablet. */ @Test void testMiddleTabletNeedsCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { @@ -221,12 +222,12 @@ void testMiddleTabletNeedsCompaction() throws AccumuloException, TableNotFoundEx /** * Verify that given a table where multiple series of tablets at different locations require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns multiple compaction ranges that includes a series of - * contiguous compactable tablets by merging their extents. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns multiple compaction ranges that includes a + * series of contiguous compactable tablets by merging their extents. */ @Test void testMultipleInnerCompactionRanges() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); addSplits("1000", "1500", "2000", "2500", "3000", "4000", "5500", "6500", "7000", "7500", "8500", "9000"); // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), @@ -259,11 +260,11 @@ void testMultipleInnerCompactionRanges() /** * Verify that given a table where all tablets starting from row 2000 to null are compacted, and given a tablet range from row 2000 to null, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns an empty list. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns an empty list. */ @Test void testCompactableTabletOutsideInputRange() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), @@ -280,11 +281,11 @@ void testCompactableTabletOutsideInputRange() /** * Verify that given a table with a compactable tablet, and given a key extent range that is contained within a tablet's range, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} will return the full extent of the tablet. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} will return the full extent of the tablet. */ @Test void testInputInCompactableTabletRange() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), @@ -302,11 +303,33 @@ void testInputInCompactableTabletRange() assertResult(); } + /** + * Verify that given a table with compactable tablets, a key extent range, starting row and ending row, mergeExtents = false, and compactTablets = true, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} will return an empty list after compacting the + * tablets automatically in {@link TabletExtentChecker#checkTablets(ClientContext, String, Text, Text, boolean, boolean)}. + * + * @throws TableNotFoundException + * @throws TableExistsException + * @throws AccumuloSecurityException + * @throws IOException + */ + + @Test + void testCompactionParameter() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), + // (5500, 6500), (6500,7000), (7000,7500), (7500,8500) (8500,9000) + addSplits("1000", "1500", "2000", "2500", "3000", "4000", "5500", "6500", "7000", "7500", "8500", "9000"); + + assertCompactedResult(); + } + } /** - * Contains test methods that verify {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} tracks individual unmerged - * tablet extents given client properties, the table name, a user provided tablet range, and mergeExtents == true. + * Contains test methods that verify {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} tracks individual + * unmerged tablet extents given client properties, the table name, a user provided tablet range, and mergeExtents == true. */ @Nested class MergeExtentsEnabledTests { @@ -318,8 +341,8 @@ void setUp() { /** * Verify that given a table where all tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns a single compaction range that would compact the - * entire table. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns a single compaction range that would compact + * the entire table. */ @Test void testAllTabletsNeedCompaction() throws AccumuloException, TableExistsException, AccumuloSecurityException, TableNotFoundException, IOException { @@ -334,12 +357,12 @@ void testAllTabletsNeedCompaction() throws AccumuloException, TableExistsExcepti /** * Verify that given a table where only the first tablet requires compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns a single range that would compact only the first - * tablet. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns a single range that would compact only the + * first tablet. */ @Test void testOnlyFirstTabletNeedsCompaction() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) @@ -354,12 +377,12 @@ void testOnlyFirstTabletNeedsCompaction() /** * Verify that given a table where only the last tablet requires compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns a single range that would compact only the last - * tablet. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns a single range that would compact only the + * last tablet. */ @Test void testOnlyLastTabletNeedsCompaction() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) @@ -374,7 +397,7 @@ void testOnlyLastTabletNeedsCompaction() /** * Verify that given a table where no tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns an empty list. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns an empty list. */ @Test void testNoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { @@ -391,12 +414,12 @@ void testNoTabletsNeedCompaction() throws AccumuloException, TableNotFoundExcept /** * Verify that given a table where only the first and last tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns separate ranges for the first and last compactable - * tablets. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns separate ranges for the first and last + * compactable tablets. */ @Test void testOnlyFirstAndLastTabletsNeedCompaction() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) @@ -413,7 +436,8 @@ void testOnlyFirstAndLastTabletsNeedCompaction() /** * Verify that given a table where only the last two tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns a single range that compacts the last two tablets. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns a single range that compacts the last two + * tablets. */ @Test void testLastTwoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { @@ -432,7 +456,7 @@ void testLastTwoTabletsNeedCompaction() throws AccumuloException, TableNotFoundE /** * Verify that given a table where only the middle tablet requires compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns a range to compact that middle tablet. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns a range to compact that middle tablet. */ @Test void testMiddleTabletNeedsCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { @@ -451,12 +475,12 @@ void testMiddleTabletNeedsCompaction() throws AccumuloException, TableNotFoundEx /** * Verify that given a table where multiple series of tablets at different locations require compaction, and given a tablet range, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns multiple compaction ranges that includes a series of - * contiguous compactable tablets by merging their extents. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns multiple compaction ranges that includes a + * series of contiguous compactable tablets by merging their extents. */ @Test void testMultipleInnerCompactionRanges() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), @@ -478,11 +502,11 @@ void testMultipleInnerCompactionRanges() /** * Verify that given a table where all tablets starting from row 2000 to null are compacted, and given a tablet range from row 2000 to null, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} returns an empty list. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns an empty list. */ @Test void testCompactableTabletOutsideInputRange() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), @@ -500,11 +524,11 @@ void testCompactableTabletOutsideInputRange() /** * Verify that given a table with a compactable tablet, and given a key extent range that is contained within a tablet's range, - * {@link TabletExtentChecker#checkTablets(AccumuloClient, String, Text, Text, boolean )} will return the full extent of the tablet. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} will return the full extent of the tablet. */ @Test void testInputInCompactableTabletRange() - throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { createTableAndWriteData(); // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), @@ -521,6 +545,29 @@ void testInputInCompactableTabletRange() expectExtent("1000", "1500"); assertResult(); } + + /** + * Verify that given a table with compactable tablets, a key extent range, starting row and ending row, mergeExtents = true, and compactTablets = true, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} will return an empty list after compacting the + * tablets automatically in {@link TabletExtentChecker#checkTablets(ClientContext, String, Text, Text, boolean, boolean)}. + * + * @throws TableNotFoundException + * @throws TableExistsException + * @throws AccumuloSecurityException + * @throws IOException + */ + + @Test + void testCompactionParameter() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), + // (5500, 6500), (6500,7000), (7000,7500), (7500,8500) (8500,9000) + addSplits("1000", "1500", "2000", "2500", "3000", "4000", "5500", "6500", "7000", "7500", "8500", "9000"); + + assertCompactedResult(); + } + } private void createTableAndWriteData() throws AccumuloException, TableExistsException, AccumuloSecurityException, TableNotFoundException { @@ -564,9 +611,27 @@ private void givenEnd(String end) { } private void assertResult() throws AccumuloException, TableNotFoundException, IOException { - //broken ServerContext cast to be changed - List> result = TabletExtentChecker.checkTablets((ServerContext)(client), TABLE_NAME, getText(begin), getText(end), mergeExtents); + List> result = TabletExtentChecker.findCompactableTablets((ClientContext) client, TABLE_NAME, getText(begin), getText(end), + mergeExtents); assertEquals(expectedExtents, result); } -} \ No newline at end of file + private void assertCompactedResult() throws AccumuloException, TableNotFoundException, AccumuloSecurityException, IOException { + // Build the list of compactable tablets + List> result = TabletExtentChecker.findCompactableTablets((ClientContext) client, TABLE_NAME, getText(begin), getText(end), + mergeExtents); + + // Compact the tablets instead of suggesting compactions (compactTablets == true) + TabletExtentChecker.checkTablets((ClientContext) client, TABLE_NAME, getText(begin), getText(end), mergeExtents, true); + + List> result2 = TabletExtentChecker.findCompactableTablets((ClientContext) client, TABLE_NAME, getText(begin), getText(end), + mergeExtents); + + // Create empty list for comparison + List> lst = new ArrayList<>(); + + // Resulting list should be empty since all tablets have been compacted + assertEquals(lst, result2); + } + +} From adf3e225a0915c0c67297e5327c57bd36e9f8bbd Mon Sep 17 00:00:00 2001 From: tom-vt4 Date: Fri, 10 Jul 2026 13:50:58 -0400 Subject: [PATCH 09/11] Remove dependencies and -p parameter requirement -This parameter isn't required if accumulo.properties is on classpath --- .../check-tablet-extents/pom.xml | 20 ------------------- .../java/datawave/TabletExtentChecker.java | 8 ++------ 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/contrib/datawave-utils/check-tablet-extents/pom.xml b/contrib/datawave-utils/check-tablet-extents/pom.xml index b9674d3d6ce..2863031c8e5 100644 --- a/contrib/datawave-utils/check-tablet-extents/pom.xml +++ b/contrib/datawave-utils/check-tablet-extents/pom.xml @@ -26,27 +26,11 @@ - - com.beust - jcommander - com.google.auto.service auto-service-annotations 1.1.1 - - com.google.guava - guava - - - io.opentelemetry - opentelemetry-api - - - io.opentelemetry - opentelemetry-context - org.apache.accumulo accumulo-core @@ -57,10 +41,6 @@ - - org.apache.accumulo - accumulo-hadoop-mapreduce - org.apache.accumulo accumulo-tserver diff --git a/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java index 511bb2d368b..699af4ceb3b 100644 --- a/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java +++ b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java @@ -86,7 +86,7 @@ public void parseArgs(String programName, String[] args, Object... others) { try { commander.parse(args); } catch (ParameterException ex) { - System.err.println("ERROR :"+ex.getMessage()); + System.err.println("ERROR :" + ex.getMessage()); commander.usage(); System.exit(1); } @@ -105,7 +105,7 @@ public String keyword() { @Override public String description() { - return "A utility that will identify tablets that require compaction within a given table for a given range and lists recommended compaction commands."; + return "Identifies tablets with data outside their extents and if specified, compacts them."; } /** @@ -127,10 +127,6 @@ public void execute(String[] args) throws AccumuloException, TableNotFoundExcept opts.parseArgs(TabletExtentChecker.class.getName(), args); - if (opts.getPropertiesPath() == null) { - System.out.println("ERROR: null props path"); - System.exit(1); - } try (ServerContext context = new ServerContext(opts.getSiteConfiguration())) { // Fetch the recommended tablet ranges to compact. checkTablets(context, opts.tableName, opts.beginRow, opts.endRow, opts.mergeExtents, opts.compactTablets); From d2a6871d61d49755c861c4dd3a9e5a552cc26630 Mon Sep 17 00:00:00 2001 From: tom-vt4 Date: Sat, 11 Jul 2026 02:32:38 -0400 Subject: [PATCH 10/11] Add debug parameter and update test suite -Debug parameter controls tablet extent logging -Removed redundant dependencies in pom.xml --- .../check-tablet-extents/pom.xml | 4 - .../java/datawave/TabletExtentChecker.java | 24 ++++-- .../datawave/TabletExtentCheckerTest.java | 86 ++++++++++--------- 3 files changed, 59 insertions(+), 55 deletions(-) diff --git a/contrib/datawave-utils/check-tablet-extents/pom.xml b/contrib/datawave-utils/check-tablet-extents/pom.xml index 2863031c8e5..edfccfd32d8 100644 --- a/contrib/datawave-utils/check-tablet-extents/pom.xml +++ b/contrib/datawave-utils/check-tablet-extents/pom.xml @@ -50,10 +50,6 @@ commons-lang3 3.20.0 - - org.apache.hadoop - hadoop-client-api - org.apache.zookeeper zookeeper diff --git a/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java index 699af4ceb3b..4328a456e50 100644 --- a/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java +++ b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java @@ -73,6 +73,9 @@ private static class Opts extends ConfigOpts { @Parameter(names = {"-c", "--compact"}, description = "Compact the tablets") public boolean compactTablets = false; + @Parameter(names = {"-d", "--debug"}, description = "Display tool debug info") + public boolean debug = false; + /** * Parse the given arguments array and populate this {@link Opts}. * @@ -129,7 +132,7 @@ public void execute(String[] args) throws AccumuloException, TableNotFoundExcept try (ServerContext context = new ServerContext(opts.getSiteConfiguration())) { // Fetch the recommended tablet ranges to compact. - checkTablets(context, opts.tableName, opts.beginRow, opts.endRow, opts.mergeExtents, opts.compactTablets); + checkTablets(context, opts.tableName, opts.beginRow, opts.endRow, opts.mergeExtents, opts.compactTablets, opts.debug); } } @@ -173,7 +176,7 @@ private static String formatArg(String arg, Text value) { * @throws IOException * if an error occurs while reading the client properties file */ - static List> findCompactableTablets(ClientContext context, String tableName, Text begin, Text end, boolean mergeExtents) + static List> findCompactableTablets(ClientContext context, String tableName, Text begin, Text end, boolean mergeExtents, boolean debug) throws AccumuloException, TableNotFoundException, IOException { List> compactionExtents = new ArrayList<>(); @@ -190,7 +193,7 @@ static List> findCompactableTablets(ClientContext context, Strin // Iterate over each tablet. for (TabletMetadata tablet : tablets) { // Determine whether the tablet needs compaction. - boolean tabletRequiresCompaction = tabletRequiresCompaction(context, tableName, tablet); + boolean tabletRequiresCompaction = tabletRequiresCompaction(context, tableName, tablet, debug); KeyExtent extent = tablet.getExtent(); // The current tablet requires compaction. @@ -261,7 +264,7 @@ private static TableId getTableId(ClientContext context, String tableName) { * the tablet metadata * @return true if the tablet requires compaction, or false */ - private static boolean tabletRequiresCompaction(ClientContext context, String tableName, TabletMetadata tablet) + private static boolean tabletRequiresCompaction(ClientContext context, String tableName, TabletMetadata tablet, boolean debug) throws AccumuloException, TableNotFoundException, IOException { // Fetch the list of RFiles for the tablet. ConfigurationCopy tableConf = new ConfigurationCopy(context.tableOperations().getConfiguration(tableName)); @@ -277,12 +280,15 @@ private static boolean tabletRequiresCompaction(ClientContext context, String ta Key first = openReader.getFirstKey(); Key last = openReader.getLastKey(); + if (debug) { + System.out.println("Extent: " + extent); + System.out.println("First key: " + first); + System.out.println("Last key: " + last); + } + // A tablet requires a compaction if any of the following are true: // - The first key is outside the tablet's extent. // - The last key is outside the tablet's extent. - System.out.println("Extent: " + extent); - System.out.println("First key: " + first); - System.out.println("Last key: " + last); if ((first != null && !extent.contains(first.getRow())) || (last != null && !extent.contains(last.getRow()))) { return true; @@ -292,9 +298,9 @@ private static boolean tabletRequiresCompaction(ClientContext context, String ta return false; } - public static void checkTablets(ClientContext context, String tableName, Text begin, Text end, boolean mergeExtents, boolean compactTablets) + public static void checkTablets(ClientContext context, String tableName, Text begin, Text end, boolean mergeExtents, boolean compactTablets, boolean debug) throws AccumuloException, TableNotFoundException, IOException, AccumuloSecurityException { - List> compactionExtents = findCompactableTablets(context, tableName, begin, end, mergeExtents); + List> compactionExtents = findCompactableTablets(context, tableName, begin, end, mergeExtents, debug); // Print a message when compactionExtents is empty if (compactionExtents.isEmpty()) { diff --git a/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java b/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java index cb236d180fe..e138f846eca 100644 --- a/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java +++ b/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java @@ -74,8 +74,8 @@ void tearDown() throws AccumuloException, TableNotFoundException, AccumuloSecuri } /** - * Contains test methods that verify {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} tracks individual - * unmerged tablet extents given a table, tablet range, and mergeExtents == false. + * Contains test methods that verify {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} tracks + * individual unmerged tablet extents given a table, tablet range, and mergeExtents == false. */ @Nested class MergeExtentsDisabledTests { @@ -87,8 +87,8 @@ void setUp() { /** * Verify that given a table where all tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns multiple ranges to compact each tablet - * individually. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns multiple ranges to compact each + * tablet individually. */ @Test void testAllSingleTabletsNeedCompaction() @@ -107,8 +107,8 @@ void testAllSingleTabletsNeedCompaction() /** * Verify that given a table where only the first tablet requires compaction, and given a tablet range, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns a single range that would compact only the - * first tablet. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns a single range that would compact + * only the first tablet. */ @Test void testOnlyFirstTabletNeedsCompaction() @@ -128,7 +128,7 @@ void testOnlyFirstTabletNeedsCompaction() /** * Verify that given a table where only the last tablet requires compaction, and given a tablet range, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns a single range that would compact only the + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} a single range that would compact only the * last tablet. */ @Test @@ -147,7 +147,7 @@ void testOnlyLastTabletNeedsCompaction() /** * Verify that given a table where no tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns an empty list. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns an empty list. */ @Test void testNoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { @@ -163,8 +163,8 @@ void testNoTabletsNeedCompaction() throws AccumuloException, TableNotFoundExcept /** * Verify that given a table where only the first and last tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns separate ranges for the first and last - * compactable tablets. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns separate ranges for the first and + * last compactable tablets. */ @Test void testOnlyFirstAndLastTabletsNeedCompaction() @@ -185,7 +185,7 @@ void testOnlyFirstAndLastTabletsNeedCompaction() /** * Verify that given a table where only the last two tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns separate ranges for those tablets. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns separate ranges for those tablets. */ @Test void testLastTwoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { @@ -204,7 +204,8 @@ void testLastTwoTabletsNeedCompaction() throws AccumuloException, TableNotFoundE /** * Verify that given a table where only the middle tablet requires compaction, and given a tablet range, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns a range to compact that middle tablet. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns a range to compact that middle + * tablet. */ @Test void testMiddleTabletNeedsCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { @@ -222,8 +223,8 @@ void testMiddleTabletNeedsCompaction() throws AccumuloException, TableNotFoundEx /** * Verify that given a table where multiple series of tablets at different locations require compaction, and given a tablet range, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns multiple compaction ranges that includes a - * series of contiguous compactable tablets by merging their extents. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns multiple compaction ranges that + * includes a series of contiguous compactable tablets by merging their extents. */ @Test void testMultipleInnerCompactionRanges() @@ -260,7 +261,7 @@ void testMultipleInnerCompactionRanges() /** * Verify that given a table where all tablets starting from row 2000 to null are compacted, and given a tablet range from row 2000 to null, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns an empty list. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns an empty list. */ @Test void testCompactableTabletOutsideInputRange() @@ -281,7 +282,7 @@ void testCompactableTabletOutsideInputRange() /** * Verify that given a table with a compactable tablet, and given a key extent range that is contained within a tablet's range, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} will return the full extent of the tablet. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} will return the full extent of the tablet. */ @Test void testInputInCompactableTabletRange() @@ -305,8 +306,8 @@ void testInputInCompactableTabletRange() /** * Verify that given a table with compactable tablets, a key extent range, starting row and ending row, mergeExtents = false, and compactTablets = true, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} will return an empty list after compacting the - * tablets automatically in {@link TabletExtentChecker#checkTablets(ClientContext, String, Text, Text, boolean, boolean)}. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} will return an empty list after compacting + * the tablets automatically in {@link TabletExtentChecker#checkTablets(ClientContext, String, Text, Text, boolean, boolean, boolean)} . * * @throws TableNotFoundException * @throws TableExistsException @@ -328,8 +329,8 @@ void testCompactionParameter() throws AccumuloException, TableNotFoundException, } /** - * Contains test methods that verify {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} tracks individual - * unmerged tablet extents given client properties, the table name, a user provided tablet range, and mergeExtents == true. + * Contains test methods that verify {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} tracks + * individual unmerged tablet extents given client properties, the table name, a user provided tablet range, and mergeExtents == true. */ @Nested class MergeExtentsEnabledTests { @@ -341,8 +342,8 @@ void setUp() { /** * Verify that given a table where all tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns a single compaction range that would compact - * the entire table. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns a single compaction range that would + * compact the entire table. */ @Test void testAllTabletsNeedCompaction() throws AccumuloException, TableExistsException, AccumuloSecurityException, TableNotFoundException, IOException { @@ -357,8 +358,8 @@ void testAllTabletsNeedCompaction() throws AccumuloException, TableExistsExcepti /** * Verify that given a table where only the first tablet requires compaction, and given a tablet range, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns a single range that would compact only the - * first tablet. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns a single range that would compact + * only the first tablet. */ @Test void testOnlyFirstTabletNeedsCompaction() @@ -377,8 +378,8 @@ void testOnlyFirstTabletNeedsCompaction() /** * Verify that given a table where only the last tablet requires compaction, and given a tablet range, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns a single range that would compact only the - * last tablet. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns a single range that would compact + * only the last tablet. */ @Test void testOnlyLastTabletNeedsCompaction() @@ -397,7 +398,7 @@ void testOnlyLastTabletNeedsCompaction() /** * Verify that given a table where no tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns an empty list. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns an empty list. */ @Test void testNoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { @@ -414,8 +415,8 @@ void testNoTabletsNeedCompaction() throws AccumuloException, TableNotFoundExcept /** * Verify that given a table where only the first and last tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns separate ranges for the first and last - * compactable tablets. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns separate ranges for the first and + * last compactable tablets. */ @Test void testOnlyFirstAndLastTabletsNeedCompaction() @@ -436,8 +437,8 @@ void testOnlyFirstAndLastTabletsNeedCompaction() /** * Verify that given a table where only the last two tablets require compaction, and given a tablet range, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns a single range that compacts the last two - * tablets. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns a single range that compacts the last + * two tablets. */ @Test void testLastTwoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { @@ -456,7 +457,8 @@ void testLastTwoTabletsNeedCompaction() throws AccumuloException, TableNotFoundE /** * Verify that given a table where only the middle tablet requires compaction, and given a tablet range, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns a range to compact that middle tablet. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns a range to compact that middle + * tablet. */ @Test void testMiddleTabletNeedsCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { @@ -475,8 +477,8 @@ void testMiddleTabletNeedsCompaction() throws AccumuloException, TableNotFoundEx /** * Verify that given a table where multiple series of tablets at different locations require compaction, and given a tablet range, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns multiple compaction ranges that includes a - * series of contiguous compactable tablets by merging their extents. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns multiple compaction ranges that + * includes a series of contiguous compactable tablets by merging their extents. */ @Test void testMultipleInnerCompactionRanges() @@ -502,7 +504,7 @@ void testMultipleInnerCompactionRanges() /** * Verify that given a table where all tablets starting from row 2000 to null are compacted, and given a tablet range from row 2000 to null, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} returns an empty list. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns an empty list. */ @Test void testCompactableTabletOutsideInputRange() @@ -524,7 +526,7 @@ void testCompactableTabletOutsideInputRange() /** * Verify that given a table with a compactable tablet, and given a key extent range that is contained within a tablet's range, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} will return the full extent of the tablet. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} will return the full extent of the tablet. */ @Test void testInputInCompactableTabletRange() @@ -548,8 +550,8 @@ void testInputInCompactableTabletRange() /** * Verify that given a table with compactable tablets, a key extent range, starting row and ending row, mergeExtents = true, and compactTablets = true, - * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean )} will return an empty list after compacting the - * tablets automatically in {@link TabletExtentChecker#checkTablets(ClientContext, String, Text, Text, boolean, boolean)}. + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} will return an empty list after compacting + * the tablets automatically in {@link TabletExtentChecker#checkTablets(ClientContext, String, Text, Text, boolean, boolean, boolean)}. * * @throws TableNotFoundException * @throws TableExistsException @@ -612,20 +614,20 @@ private void givenEnd(String end) { private void assertResult() throws AccumuloException, TableNotFoundException, IOException { List> result = TabletExtentChecker.findCompactableTablets((ClientContext) client, TABLE_NAME, getText(begin), getText(end), - mergeExtents); + mergeExtents, true); assertEquals(expectedExtents, result); } private void assertCompactedResult() throws AccumuloException, TableNotFoundException, AccumuloSecurityException, IOException { // Build the list of compactable tablets List> result = TabletExtentChecker.findCompactableTablets((ClientContext) client, TABLE_NAME, getText(begin), getText(end), - mergeExtents); + mergeExtents, true); // Compact the tablets instead of suggesting compactions (compactTablets == true) - TabletExtentChecker.checkTablets((ClientContext) client, TABLE_NAME, getText(begin), getText(end), mergeExtents, true); + TabletExtentChecker.checkTablets((ClientContext) client, TABLE_NAME, getText(begin), getText(end), mergeExtents, true, true); List> result2 = TabletExtentChecker.findCompactableTablets((ClientContext) client, TABLE_NAME, getText(begin), getText(end), - mergeExtents); + mergeExtents, true); // Create empty list for comparison List> lst = new ArrayList<>(); From 883b3633f77c049832c02087c7c58f36c320ef67 Mon Sep 17 00:00:00 2001 From: tom-vt4 Date: Tue, 14 Jul 2026 11:06:57 -0400 Subject: [PATCH 11/11] Remove unused dependencies --- .../check-tablet-extents/pom.xml | 15 -------------- .../datawave/TabletExtentCheckerTest.java | 20 ++++++------------- 2 files changed, 6 insertions(+), 29 deletions(-) diff --git a/contrib/datawave-utils/check-tablet-extents/pom.xml b/contrib/datawave-utils/check-tablet-extents/pom.xml index edfccfd32d8..3f8ab6323ca 100644 --- a/contrib/datawave-utils/check-tablet-extents/pom.xml +++ b/contrib/datawave-utils/check-tablet-extents/pom.xml @@ -45,21 +45,6 @@ org.apache.accumulo accumulo-tserver - - org.apache.commons - commons-lang3 - 3.20.0 - - - org.apache.zookeeper - zookeeper - - - io.dropwizard.metrics - metrics-core - 4.1.12.1 - test - org.apache.accumulo accumulo-minicluster diff --git a/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java b/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java index e138f846eca..aecc661f0e1 100644 --- a/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java +++ b/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java @@ -1,6 +1,7 @@ package datawave; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.nio.file.Path; @@ -308,11 +309,6 @@ void testInputInCompactableTabletRange() * Verify that given a table with compactable tablets, a key extent range, starting row and ending row, mergeExtents = false, and compactTablets = true, * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} will return an empty list after compacting * the tablets automatically in {@link TabletExtentChecker#checkTablets(ClientContext, String, Text, Text, boolean, boolean, boolean)} . - * - * @throws TableNotFoundException - * @throws TableExistsException - * @throws AccumuloSecurityException - * @throws IOException */ @Test @@ -552,11 +548,6 @@ void testInputInCompactableTabletRange() * Verify that given a table with compactable tablets, a key extent range, starting row and ending row, mergeExtents = true, and compactTablets = true, * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} will return an empty list after compacting * the tablets automatically in {@link TabletExtentChecker#checkTablets(ClientContext, String, Text, Text, boolean, boolean, boolean)}. - * - * @throws TableNotFoundException - * @throws TableExistsException - * @throws AccumuloSecurityException - * @throws IOException */ @Test @@ -629,11 +620,12 @@ private void assertCompactedResult() throws AccumuloException, TableNotFoundExce List> result2 = TabletExtentChecker.findCompactableTablets((ClientContext) client, TABLE_NAME, getText(begin), getText(end), mergeExtents, true); - // Create empty list for comparison - List> lst = new ArrayList<>(); + // Fetch the list of compactable tablets within the same range. + List> extents = TabletExtentChecker.findCompactableTablets((ClientContext) client, TABLE_NAME, getText(begin), getText(end), + mergeExtents, false); - // Resulting list should be empty since all tablets have been compacted - assertEquals(lst, result2); + // Verify the list is empty, indicating success. + assertTrue(extents.isEmpty()); } }