diff --git a/.gitignore b/.gitignore index eaa9fd08edc..1fbbd622c05 100644 --- a/.gitignore +++ b/.gitignore @@ -53,4 +53,4 @@ services/sample_configuration/*-dev.yml /.jkube-dockerignore .claude - +.sdkmanrc diff --git a/contrib/datawave-utils/check-tablet-extents/README.md b/contrib/datawave-utils/check-tablet-extents/README.md new file mode 100644 index 00000000000..68dfe089bf4 --- /dev/null +++ b/contrib/datawave-utils/check-tablet-extents/README.md @@ -0,0 +1,248 @@ +# Tablet Extent Checker + +This project contains a server utility tool for identifying tablets in a table that are candidates for compaction. + +## Building the tool + +Building the tool is done with the following command. To run unit tests, omit `-DskipTests`. + +```bash +$ mvn clean install -DskipTests +``` + +This will create the executable jar `target/check-tablet-extents.jar`. + +## Running the tool + +Copy the jar to the Accumulo lib folder: + +```bash +$ cp path/to/check-tablet-extents.jar $ACCUMULO_HOME/lib +``` + +Then use the following command: + +```bash +$ accumulo check-tablets +``` + +or + +```bash +$ accumulo datawave.TabletExtentChecker +``` + + +Use the `--help` option to list all available options: + +```bash +$ accumulo check-tablets -h +``` + +or + +```bash +$ accumulo datawave.TabletExtentChecker -h +``` + +Output: + +```bash +Usage: TabletExtentChecker [options] + Options: + -b, --begin + The starting row (exclusive) of the range of tablets to scan + -c, --compact + Compact the tablets + Default: false + -e, --end + The ending row (inclusive) of the range of tablets to scan + -h, -?, --help, -help + + -m, --merge, --merge-extents + Merges suggested compaction ranges for neighboring compactable tablets + Default: false + -p, -props, --props + Sets path to accumulo.properties.The classpath will be searched if this + property is not set + * -t, --table + The table name + -o + Overrides configuration set in accumulo.properties (but NOT system-wide + config set in Zookeeper). Expected format: -o = + Default: [] +``` + + +The tool will list a set of recommended compaction commands if it finds any tablets that require compaction. +For example, given a table `chop_test` that has tablets with data outside the tablet extents for the following extents: + +- Start: null, End: "2500" +- Start: "2500", End: "5000" +- Start: "5000", End: "7500" +- Start: "7500", End: null + + +You can expect the following output: +```bash +$ accumulo check-tablets -p $ACCUMULO_HOME/conf/accumulo.properties -t chop_test +2026-07-10T09:17:10,431 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.ZStandardCodec +2026-07-10T09:17:10,433 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.LzoCodec +2026-07-10T09:17:10,434 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.BZip2Codec +2026-07-10T09:17:10,436 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.accumulo.core.file.rfile.bcfile.IdentityCodec +2026-07-10T09:17:10,437 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.Lz4Codec +2026-07-10T09:17:10,438 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.SnappyCodec +2026-07-10T09:17:10,439 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +2026-07-10T09:17:10,441 [compress.CodecPool] INFO : Got brand-new compressor [.identity] +2026-07-10T09:17:10,658 [zlib.ZlibFactory] INFO : Successfully loaded & initialized native-zlib library +2026-07-10T09:17:10,658 [compress.CodecPool] INFO : Got brand-new decompressor [.deflate] +2026-07-10T09:17:10,659 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +Extent: 1;2500< +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1;5000;2500 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1;7500;5000 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1<;7500 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +compact -t chop_test -e 2500 +compact -t chop_test -b 2500 -e 5000 +compact -t chop_test -b 5000 -e 7500 +compact -t chop_test -b 7500 +2026-07-10T09:17:10,709 [metrics.MetricsInfoImpl] INFO : micrometer metrics enabled: false +2026-07-10T09:17:10,709 [metrics.MetricsInfoImpl] INFO : Closing metrics registry +``` +The tool recommended multiple compaction ranges for the table. + + +If the option `--merge-extents` is specified, neighboring compactable tablet extents will be merged within the recommended compaction commands +```bash +accumulo check-tablets -p $ACCUMULO_HOME/conf/accumulo.properties -t chop_test --merge-extents +2026-07-10T09:42:21,296 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.accumulo.core.file.rfile.bcfile.IdentityCodec +2026-07-10T09:42:21,307 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.BZip2Codec +2026-07-10T09:42:21,308 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.LzoCodec +2026-07-10T09:42:21,309 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +2026-07-10T09:42:21,311 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.ZStandardCodec +2026-07-10T09:42:21,312 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.SnappyCodec +2026-07-10T09:42:21,313 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.Lz4Codec +2026-07-10T09:42:21,316 [compress.CodecPool] INFO : Got brand-new compressor [.identity] +2026-07-10T09:42:21,527 [zlib.ZlibFactory] INFO : Successfully loaded & initialized native-zlib library +2026-07-10T09:42:21,527 [compress.CodecPool] INFO : Got brand-new decompressor [.deflate] +2026-07-10T09:42:21,528 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +Extent: 1;2500< +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1;5000;2500 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1;7500;5000 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1<;7500 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +compact -t chop_test +2026-07-10T09:42:21,578 [metrics.MetricsInfoImpl] INFO : micrometer metrics enabled: false +2026-07-10T09:42:21,578 [metrics.MetricsInfoImpl] INFO : Closing metrics registry +``` +The compaction ranges with contiguous tablets have been combined. The tool recommended one command, `compact -t chop_test` which will compact the entire chop_test table. + + +If the options `-b 2500` and `-e 7500` are specified, only the suggestions for the rows ranging from 2500 to 7500 will be displayed: +```bash +accumulo check-tablets -p $ACCUMULO_HOME/conf/accumulo.properties -t chop_test -b 2500 -e 7500 +2026-07-10T09:46:22,218 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.accumulo.core.file.rfile.bcfile.IdentityCodec +2026-07-10T09:46:22,231 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.BZip2Codec +2026-07-10T09:46:22,233 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +2026-07-10T09:46:22,234 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.LzoCodec +2026-07-10T09:46:22,238 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.Lz4Codec +2026-07-10T09:46:22,240 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.SnappyCodec +2026-07-10T09:46:22,243 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.ZStandardCodec +2026-07-10T09:46:22,248 [compress.CodecPool] INFO : Got brand-new compressor [.identity] +2026-07-10T09:46:22,461 [zlib.ZlibFactory] INFO : Successfully loaded & initialized native-zlib library +2026-07-10T09:46:22,461 [compress.CodecPool] INFO : Got brand-new decompressor [.deflate] +2026-07-10T09:46:22,461 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +Extent: 1;5000;2500 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1;7500;5000 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +compact -t chop_test -b 2500 -e 5000 +compact -t chop_test -b 5000 -e 7500 +2026-07-10T09:46:22,487 [metrics.MetricsInfoImpl] INFO : micrometer metrics enabled: false +2026-07-10T09:46:22,487 [metrics.MetricsInfoImpl] INFO : Closing metrics registry +``` +If the --merge-extent option was used as well, the tool would have combined the extents and suggested: `compact -t chop_test -b 2500 -e 7500`. + + +If the option `--compact` is specified, the tablets would be compacted automatically: +```bash +accumulo check-tablets -p $ACCUMULO_HOME/conf/accumulo.properties -t chop_test -c +2026-07-10T10:03:56,040 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.LzoCodec +2026-07-10T10:03:56,042 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.BZip2Codec +2026-07-10T10:03:56,043 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.SnappyCodec +2026-07-10T10:03:56,045 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.Lz4Codec +2026-07-10T10:03:56,046 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.accumulo.core.file.rfile.bcfile.IdentityCodec +2026-07-10T10:03:56,047 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.ZStandardCodec +2026-07-10T10:03:56,048 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +2026-07-10T10:03:56,050 [compress.CodecPool] INFO : Got brand-new compressor [.identity] +2026-07-10T10:03:56,266 [zlib.ZlibFactory] INFO : Successfully loaded & initialized native-zlib library +2026-07-10T10:03:56,266 [compress.CodecPool] INFO : Got brand-new decompressor [.deflate] +2026-07-10T10:03:56,266 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +Extent: 1;2500< +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1;5000;2500 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1;7500;5000 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Extent: 1<;7500 +First key: 0000 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +Compacting range from null-2500 +Compacting range from 2500-5000 +Compacting range from 5000-7500 +Compacting range from 7500-null +2026-07-10T10:03:59,007 [metrics.MetricsInfoImpl] INFO : micrometer metrics enabled: false +2026-07-10T10:03:59,007 [metrics.MetricsInfoImpl] INFO : Closing metrics registry +``` + + +We can verify that the compaction was successful by running the tool again: +```bash +accumulo check-tablets -p $ACCUMULO_HOME/conf/accumulo.properties -t chop_test +2026-07-10T10:08:17,880 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +2026-07-10T10:08:17,892 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.LzoCodec +2026-07-10T10:08:17,893 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.SnappyCodec +2026-07-10T10:08:17,895 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.accumulo.core.file.rfile.bcfile.IdentityCodec +2026-07-10T10:08:17,895 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.ZStandardCodec +2026-07-10T10:08:17,896 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.BZip2Codec +2026-07-10T10:08:17,898 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.Lz4Codec +2026-07-10T10:08:17,900 [compress.CodecPool] INFO : Got brand-new compressor [.identity] +2026-07-10T10:08:18,123 [zlib.ZlibFactory] INFO : Successfully loaded & initialized native-zlib library +2026-07-10T10:08:18,123 [compress.CodecPool] INFO : Got brand-new decompressor [.deflate] +2026-07-10T10:08:18,124 [bcfile.CompressionAlgorithm] INFO : Trying to load codec class org.apache.hadoop.io.compress.DefaultCodec +Extent: 1;2500< +First key: 0000 cf:cq [] 1783674529729 false +Last key: 2500 cf:cq [] 1783674529729 false +Extent: 1;5000;2500 +First key: 2501 cf:cq [] 1783674529729 false +Last key: 5000 cf:cq [] 1783674529729 false +Extent: 1;7500;5000 +First key: 5001 cf:cq [] 1783674529729 false +Last key: 7500 cf:cq [] 1783674529729 false +Extent: 1<;7500 +First key: 7501 cf:cq [] 1783674529729 false +Last key: 9999 cf:cq [] 1783674529729 false +No candidates suitable for compaction. +2026-07-10T10:08:18,191 [metrics.MetricsInfoImpl] INFO : micrometer metrics enabled: false +2026-07-10T10:08:18,191 [metrics.MetricsInfoImpl] INFO : Closing metrics registry +``` +We can see that the tablets now only contain rows within their respective extents, and the tool outputs a message that there are no compactable tablets. \ No newline at end of file diff --git a/contrib/datawave-utils/check-tablet-extents/pom.xml b/contrib/datawave-utils/check-tablet-extents/pom.xml new file mode 100644 index 00000000000..3f8ab6323ca --- /dev/null +++ b/contrib/datawave-utils/check-tablet-extents/pom.xml @@ -0,0 +1,103 @@ + + + 4.0.0 + + + gov.nsa.datawave + datawave-parent + 7.42.0-SNAPSHOT + + + check-tablet-extents + 1.0.0-SNAPSHOT + Datawave Tablet Extent Checker + A utility tool for identifying tablets with data outside the tablets' extents. + + + + + org.apache.accumulo + accumulo-project + ${version.accumulo} + pom + import + + + + + + + com.google.auto.service + auto-service-annotations + 1.1.1 + + + org.apache.accumulo + accumulo-core + + + org.apache.logging.log4j + log4j-1.2-api + + + + + org.apache.accumulo + accumulo-tserver + + + org.apache.accumulo + accumulo-minicluster + test + + + org.apache.logging.log4j + log4j-1.2-api + + + org.apache.logging.log4j + log4j-api + + + org.apache.logging.log4j + log4j-core + + + org.apache.logging.log4j + log4j-web + + + + + + ${project.artifactId} + + + + org.apache.maven.plugins + maven-compiler-plugin + + true + + + + maven-compiler-plugin + + + + com.google.auto.service + auto-service + 1.1.1 + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.3.0 + + + + + diff --git a/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java new file mode 100644 index 00000000000..4328a456e50 --- /dev/null +++ b/contrib/datawave-utils/check-tablet-extents/src/main/java/datawave/TabletExtentChecker.java @@ -0,0 +1,321 @@ +package datawave; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.accumulo.core.cli.ConfigOpts; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.admin.TableOperations; +import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.conf.ConfigurationCopy; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.file.FileOperations; +import org.apache.accumulo.core.file.FileSKVIterator; +import org.apache.accumulo.core.metadata.StoredTabletFile; +import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import org.apache.accumulo.core.metadata.schema.TabletsMetadata; +import org.apache.accumulo.core.spi.crypto.NoCryptoServiceFactory; +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.start.spi.KeywordExecutable; +import org.apache.accumulo.tserver.tablet.Tablet; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.io.Text; + +import com.beust.jcommander.IStringConverter; +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; +import com.google.auto.service.AutoService; + +/** + * A utility that will identify tablets that require compaction within a given table for a given range and lists recommended compaction commands. + */ +@AutoService(KeywordExecutable.class) +public final class TabletExtentChecker implements KeywordExecutable { + + /** + * Converts a string argument to a {@link Text} instance. + */ + private static class TextConverter implements IStringConverter { + @Override + public Text convert(String s) { + return s == null ? null : new Text(s); + } + } + + /** + * Represents a set of options that can parsed and used via {@link TabletExtentChecker#execute(String[])}. ConfigOpts provides getSiteConfiguration() to be + * used to create ServerContext + */ + private static class Opts extends ConfigOpts { + @Parameter(names = {"-t", "--table"}, description = "The table name", required = true) + public String tableName = null; + + @Parameter(names = {"-b", "--begin"}, description = "The starting row (exclusive) of the range of tablets to scan", converter = TextConverter.class) + public Text beginRow = null; + + @Parameter(names = {"-e", "--end"}, description = "The ending row (inclusive) of the range of tablets to scan", converter = TextConverter.class) + public Text endRow = null; + + @Parameter(names = {"-m", "--merge", "--merge-extents"}, description = "Merges suggested compaction ranges for neighboring compactable tablets") + public boolean mergeExtents = false; + + @Parameter(names = {"-c", "--compact"}, description = "Compact the tablets") + public boolean compactTablets = false; + + @Parameter(names = {"-d", "--debug"}, description = "Display tool debug info") + public boolean debug = false; + + /** + * Parse the given arguments array and populate this {@link Opts}. + * + * @param args + * the arguments to parse. + */ + @Override + public void parseArgs(String programName, String[] args, Object... others) { + JCommander commander = JCommander.newBuilder().addObject(this).programName(TabletExtentChecker.class.getSimpleName()).build(); + + try { + commander.parse(args); + } catch (ParameterException ex) { + System.err.println("ERROR :" + ex.getMessage()); + commander.usage(); + System.exit(1); + } + + if (help) { + commander.usage(); + System.exit(0); + } + } + } + + @Override + public String keyword() { + return "check-tablets"; + } + + @Override + public String description() { + return "Identifies tablets with data outside their extents and if specified, compacts them."; + } + + /** + * Parses the user-provided arguments and prints out recommended ranges for compaction. + * + * @param args + * the arguments + * @throws AccumuloException + * if an error occurs while connecting to Accumulo + * @throws TableNotFoundException + * if the specified table does not exist + * @throws IOException + * if an error occurs while reading the client properties file + */ + @Override + public void execute(String[] args) throws AccumuloException, TableNotFoundException, IOException, AccumuloSecurityException { + // Parse the arguments. + Opts opts = new Opts(); + + opts.parseArgs(TabletExtentChecker.class.getName(), args); + + try (ServerContext context = new ServerContext(opts.getSiteConfiguration())) { + // Fetch the recommended tablet ranges to compact. + checkTablets(context, opts.tableName, opts.beginRow, opts.endRow, opts.mergeExtents, opts.compactTablets, opts.debug); + } + } + + public static void main(String[] args) throws AccumuloException, TableNotFoundException, IOException, AccumuloSecurityException { + new TabletExtentChecker().execute(args); + } + + /** + * Return the given value formatted as an argument snippet if the value is not null. + * + * @param arg + * the argument option + * @param value + * the value + * @return the formatted arg to value command snippet if the value is not null, otherwise returns an empty string + */ + private static String formatArg(String arg, Text value) { + if (value == null) { + return ""; + } else { + return " " + arg + " " + value; + } + } + + /** + * Returns a list of pairs consisting of the extents of tablets that require compaction. + * + * @param tableName + * the name of the table to evaluate for tablet compaction + * @param begin + * the starting row (exclusive) of the range to search for tablets within. A null value implies no starting boundary. + * @param end + * the ending row (inclusive) of the range to search for tablets within. A null value implies no ending boundary + * @param mergeExtents + * will merge recommended ranges of neighboring tablets requiring compaction + * @return the list of tablet boundaries recommended for compaction + * @throws AccumuloException + * if an error occurs while connecting to Accumulo + * @throws TableNotFoundException + * if the specified table does not exist + * @throws IOException + * if an error occurs while reading the client properties file + */ + static List> findCompactableTablets(ClientContext context, String tableName, Text begin, Text end, boolean mergeExtents, boolean debug) + throws AccumuloException, TableNotFoundException, IOException { + List> compactionExtents = new ArrayList<>(); + + TableId tableId = getTableId(context, tableName); + // Fetch the metadata for all tablets in the given table whose extents overlap with the user provided range of tablets to scan. + try (TabletsMetadata tablets = context.getAmple().readTablets().forTable(tableId).overlapping(begin, false, end) + .fetch(TabletMetadata.ColumnType.PREV_ROW, TabletMetadata.ColumnType.FILES).build()) { + + // Tracks compaction ranges if we are merging extents. + boolean foundCompactableTablet = false; + Text compactionStart = null; + Text compactionEnd = null; + + // Iterate over each tablet. + for (TabletMetadata tablet : tablets) { + // Determine whether the tablet needs compaction. + boolean tabletRequiresCompaction = tabletRequiresCompaction(context, tableName, tablet, debug); + KeyExtent extent = tablet.getExtent(); + + // The current tablet requires compaction. + if (tabletRequiresCompaction) { + // If we are merging extents, update the current compaction start and end based on whether the previous tablet also required compaction. + if (mergeExtents) { + // The previous tablet did not require compaction. Update the compaction range to reflect the current tablet. + if (!foundCompactableTablet) { + compactionStart = extent.prevEndRow(); + compactionEnd = extent.endRow(); + foundCompactableTablet = true; + } else { + // The previous tablet needs compaction, along with the current tablet. Update the end row. + compactionEnd = extent.endRow(); + } + } else { + compactionExtents.add(Pair.of(extent.prevEndRow(), extent.endRow())); + } + // The current tablet does not require compaction. + } else { + // We are merging tablet extents, and the previous tablet requires compaction. + if (mergeExtents && foundCompactableTablet) { + // Add a new recommended compaction range and reset our compaction boundaries. + compactionExtents.add(Pair.of(compactionStart, compactionEnd)); + compactionStart = null; + compactionEnd = null; + foundCompactableTablet = false; + } + } + } + // Handle case where last tablet needs compaction when we are merging extents. + if (mergeExtents && foundCompactableTablet) { + compactionExtents.add(Pair.of(compactionStart, compactionEnd)); + } + } + return compactionExtents; + } + + /** + * Return the table ID for the given table + * + * @param context + * the context to use when connecting to Accumulo + * @param tableName + * the table name + * @return the table ID + */ + private static TableId getTableId(ClientContext context, String tableName) { + TableOperations tableOperations = context.tableOperations(); + if (tableOperations.exists(tableName)) { + return TableId.of(tableOperations.tableIdMap().get(tableName)); + } else { + throw new IllegalArgumentException("Table " + tableName + " does not exist"); + } + } + + /** + * Return whether the given tablet requires compaction. This method represents a merging of the functions + * {@link org.apache.accumulo.tserver.tablet.CompactableUtils#getFirstAndLastKeys(Tablet, Set)} and + * {@link org.apache.accumulo.tserver.tablet.CompactableUtils#findChopFiles(KeyExtent, Map, Collection)} that is designed to return true as soon as we find + * an RFile that is empty, or whose first or last keys fall outside the tablet's extent. + * + * @param context + * the context to use when connecting to Accumulo + * @param tableName + * the tablet name + * @param tablet + * the tablet metadata + * @return true if the tablet requires compaction, or false + */ + private static boolean tabletRequiresCompaction(ClientContext context, String tableName, TabletMetadata tablet, boolean debug) + throws AccumuloException, TableNotFoundException, IOException { + // Fetch the list of RFiles for the tablet. + ConfigurationCopy tableConf = new ConfigurationCopy(context.tableOperations().getConfiguration(tableName)); + KeyExtent extent = tablet.getExtent(); + Set allFiles = new HashSet<>(tablet.getFiles()); + final FileOperations fileFactory = FileOperations.getInstance(); + + // Examine each file and determine whether any of them would be cleaned up/optimized by a compaction. + for (StoredTabletFile file : allFiles) { + FileSystem ns = FileSystem.get(file.getPath().toUri(), context.getHadoopConf()); + try (FileSKVIterator openReader = fileFactory.newReaderBuilder().forFile(file.getPathStr(), ns, ns.getConf(), NoCryptoServiceFactory.NONE) + .withTableConfiguration(tableConf).seekToBeginning().build()) { + Key first = openReader.getFirstKey(); + Key last = openReader.getLastKey(); + + if (debug) { + System.out.println("Extent: " + extent); + System.out.println("First key: " + first); + System.out.println("Last key: " + last); + } + + // A tablet requires a compaction if any of the following are true: + // - The first key is outside the tablet's extent. + // - The last key is outside the tablet's extent. + + if ((first != null && !extent.contains(first.getRow())) || (last != null && !extent.contains(last.getRow()))) { + return true; + } + } + } + return false; + } + + public static void checkTablets(ClientContext context, String tableName, Text begin, Text end, boolean mergeExtents, boolean compactTablets, boolean debug) + throws AccumuloException, TableNotFoundException, IOException, AccumuloSecurityException { + List> compactionExtents = findCompactableTablets(context, tableName, begin, end, mergeExtents, debug); + + // Print a message when compactionExtents is empty + if (compactionExtents.isEmpty()) { + System.out.println("No candidates suitable for compaction."); + } else { + for (Pair pair : compactionExtents) { + Text startRow = pair.getLeft(); + Text endRow = pair.getRight(); + if (compactTablets) { + System.out.println("Compacting range from " + startRow + "-" + endRow); + context.tableOperations().compact(tableName, startRow, endRow, true, true); + } else { + System.out.println("compact -t " + tableName + formatArg("-b", startRow) + formatArg("-e", endRow)); + } + } + } + } +} diff --git a/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java b/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java new file mode 100644 index 00000000000..aecc661f0e1 --- /dev/null +++ b/contrib/datawave-utils/check-tablet-extents/src/test/java/datawave/TabletExtentCheckerTest.java @@ -0,0 +1,631 @@ +package datawave; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.minicluster.MiniAccumuloCluster; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.io.Text; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TabletExtentCheckerTest { + private static final String TABLE_NAME = "testTable"; + + @TempDir + private static Path tempDir; + private static MiniAccumuloCluster mac; + + private AccumuloClient client; + + // Tracks the range of tablet extents to compact on + private final List> expectedExtents = new ArrayList<>(); + String begin; + String end; + boolean mergeExtents; + + @BeforeAll + static void beforeAll() throws IOException, InterruptedException { + mac = new MiniAccumuloCluster(tempDir.resolve("mac").toFile(), "secret"); + mac.start(); + } + + @AfterAll + static void afterAll() throws IOException { + mac.close(); + } + + @BeforeEach + void setUp() { + begin = null; + end = null; + mergeExtents = false; + client = mac.createAccumuloClient("root", new PasswordToken("secret")); + } + + @AfterEach + void tearDown() throws AccumuloException, TableNotFoundException, AccumuloSecurityException { + expectedExtents.clear(); + if (client.tableOperations().exists(TABLE_NAME)) { + client.tableOperations().delete(TABLE_NAME); + } + client.close(); + } + + /** + * Contains test methods that verify {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} tracks + * individual unmerged tablet extents given a table, tablet range, and mergeExtents == false. + */ + @Nested + class MergeExtentsDisabledTests { + + @BeforeEach + void setUp() { + mergeExtents = false; + } + + /** + * Verify that given a table where all tablets require compaction, and given a tablet range, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns multiple ranges to compact each + * tablet individually. + */ + @Test + void testAllSingleTabletsNeedCompaction() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) + addSplits("2500", "4000", "7000"); + + expectExtent(null, "2500"); + expectExtent("2500", "4000"); + expectExtent("4000", "7000"); + expectExtent("7000", null); + assertResult(); + } + + /** + * Verify that given a table where only the first tablet requires compaction, and given a tablet range, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns a single range that would compact + * only the first tablet. + */ + @Test + void testOnlyFirstTabletNeedsCompaction() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) + addSplits("2500", "4000", "7000"); + + // Compact all tablets other than the first tablet. + compact("2500", null); + + // Add uncompacted tablet to expectedExtents + expectExtent(null, "2500"); + assertResult(); + } + + /** + * Verify that given a table where only the last tablet requires compaction, and given a tablet range, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} a single range that would compact only the + * last tablet. + */ + @Test + void testOnlyLastTabletNeedsCompaction() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) + addSplits("2500", "4000", "7000"); + + // Compact all tablets except the last + compact(null, "7000"); + + expectExtent("7000", null); + assertResult(); + } + + /** + * Verify that given a table where no tablets require compaction, and given a tablet range, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns an empty list. + */ + @Test + void testNoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) + addSplits("2500", "4000", "7000"); + + compact(null, null); + + assertResult(); + } + + /** + * Verify that given a table where only the first and last tablets require compaction, and given a tablet range, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns separate ranges for the first and + * last compactable tablets. + */ + @Test + void testOnlyFirstAndLastTabletsNeedCompaction() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + addSplits("2500", "4000", "7000"); + + // Compact all tablets other than first tablet and last tablet + // Range of first tablet: null - 2500 + // Range of last tablet: 7000 - null + compact("2500", "7000"); + + expectExtent(null, "2500"); + expectExtent("7000", null); + assertResult(); + } + + /** + * Verify that given a table where only the last two tablets require compaction, and given a tablet range, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns separate ranges for those tablets. + */ + @Test + void testLastTwoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + addSplits("2500", "4000", "5500", "7000", "8500"); + + // Compact all tablets except last two tablets + compact(null, "7000"); + + // Add last two tablets to expectedExtents + expectExtent("7000", "8500"); + expectExtent("8500", null); + assertResult(); + } + + /** + * Verify that given a table where only the middle tablet requires compaction, and given a tablet range, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns a range to compact that middle + * tablet. + */ + @Test + void testMiddleTabletNeedsCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000, null), (8000, null) + addSplits("2500", "4000", "7000", "8000"); + + compact(null, "4000"); + compact("7000", null); + + expectExtent("4000", "7000"); + assertResult(); + } + + /** + * Verify that given a table where multiple series of tablets at different locations require compaction, and given a tablet range, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns multiple compaction ranges that + * includes a series of contiguous compactable tablets by merging their extents. + */ + @Test + void testMultipleInnerCompactionRanges() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + addSplits("1000", "1500", "2000", "2500", "3000", "4000", "5500", "6500", "7000", "7500", "8500", "9000"); + // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), + // (5500, 6500), (6500,7000), (7000,7500), (7500,8500) (8500,9000) + + // Compact extents to make a series of compacted tablets then track the uncompacted tablets: + compact(null, "1000"); + compact("2500", "3000"); + compact("6500", "7000"); + compact("8500", null); + + // Track uncompacted tablets in the key extent series ("1000","2500") + // Tablets: (1000, 1500), (1500,2000), (2000,2500) + expectExtent("1000", "1500"); + expectExtent("1500", "2000"); + expectExtent("2000", "2500"); + + // Track uncompacted tablets in the key extent series ("3000","6500") + // Tablets: (3000,4000), (4000,5500), (5500, 6500) + expectExtent("3000", "4000"); + expectExtent("4000", "5500"); + expectExtent("5500", "6500"); + + // Track uncompacted tablets in the key extent series ("7000","8500") + // Tablets: (7000,7500), (7500,8500) + expectExtent("7000", "7500"); + expectExtent("7500", "8500"); + assertResult(); + } + + /** + * Verify that given a table where all tablets starting from row 2000 to null are compacted, and given a tablet range from row 2000 to null, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns an empty list. + */ + @Test + void testCompactableTabletOutsideInputRange() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), + // (5500, 6500), (6500,7000), (7000,7500), (7500,8500) (8500,9000) + addSplits("1000", "1500", "2000", "2500", "3000", "4000", "5500", "6500", "7000", "7500", "8500", "9000"); + + compact("2000", null); + + givenBegin("2000"); + + // expectedExtents should be empty, so just make the assertion + assertResult(); + } + + /** + * Verify that given a table with a compactable tablet, and given a key extent range that is contained within a tablet's range, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} will return the full extent of the tablet. + */ + @Test + void testInputInCompactableTabletRange() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), + // (5500, 6500), (6500,7000), (7000,7500), (7500,8500) (8500,9000) + addSplits("1000", "1500", "2000", "2500", "3000", "4000", "5500", "6500", "7000", "7500", "8500", "9000"); + + // Compact all tablets from null to row 1000 and from row 1500 to null + compact(null, "1000"); + compact("1500", null); + + // Compactable tablet: (1000,1500) + givenBegin("1000"); + givenEnd("1250"); + expectExtent("1000", "1500"); + assertResult(); + } + + /** + * Verify that given a table with compactable tablets, a key extent range, starting row and ending row, mergeExtents = false, and compactTablets = true, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} will return an empty list after compacting + * the tablets automatically in {@link TabletExtentChecker#checkTablets(ClientContext, String, Text, Text, boolean, boolean, boolean)} . + */ + + @Test + void testCompactionParameter() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), + // (5500, 6500), (6500,7000), (7000,7500), (7500,8500) (8500,9000) + addSplits("1000", "1500", "2000", "2500", "3000", "4000", "5500", "6500", "7000", "7500", "8500", "9000"); + + assertCompactedResult(); + } + + } + + /** + * Contains test methods that verify {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} tracks + * individual unmerged tablet extents given client properties, the table name, a user provided tablet range, and mergeExtents == true. + */ + @Nested + class MergeExtentsEnabledTests { + + @BeforeEach + void setUp() { + mergeExtents = true; + } + + /** + * Verify that given a table where all tablets require compaction, and given a tablet range, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns a single compaction range that would + * compact the entire table. + */ + @Test + void testAllTabletsNeedCompaction() throws AccumuloException, TableExistsException, AccumuloSecurityException, TableNotFoundException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) + addSplits("2500", "4000", "7000"); + + expectExtent(null, null); + assertResult(); + } + + /** + * Verify that given a table where only the first tablet requires compaction, and given a tablet range, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns a single range that would compact + * only the first tablet. + */ + @Test + void testOnlyFirstTabletNeedsCompaction() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) + addSplits("2500", "4000", "7000"); + + // Compact all tablets other than the first tablet. + compact("2500", null); + + expectExtent(null, "2500"); + assertResult(); + } + + /** + * Verify that given a table where only the last tablet requires compaction, and given a tablet range, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns a single range that would compact + * only the last tablet. + */ + @Test + void testOnlyLastTabletNeedsCompaction() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) + addSplits("2500", "4000", "7000"); + + // Compact all tablets except the last + compact(null, "7000"); + + expectExtent("7000", null); + assertResult(); + } + + /** + * Verify that given a table where no tablets require compaction, and given a tablet range, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns an empty list. + */ + @Test + void testNoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) + addSplits("2500", "4000", "7000"); + + // Compact all tablets + compact(null, null); + + assertResult(); + } + + /** + * Verify that given a table where only the first and last tablets require compaction, and given a tablet range, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns separate ranges for the first and + * last compactable tablets. + */ + @Test + void testOnlyFirstAndLastTabletsNeedCompaction() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null) + addSplits("2500", "4000", "7000"); + + // Compact all tablets other than first tablet and last tablet + compact("2500", "7000"); + + expectExtent(null, "2500"); + expectExtent("7000", null); + + assertResult(); + } + + /** + * Verify that given a table where only the last two tablets require compaction, and given a tablet range, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns a single range that compacts the last + * two tablets. + */ + @Test + void testLastTwoTabletsNeedCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,5500), (5500,7000), (7000,8500) (8500,null) + addSplits("2500", "4000", "5500", "7000", "8500"); + + // Compact all tablets except last two tablets + compact(null, "7000"); + + // Add last two tablets to expectedExtents + expectExtent("7000", null); + assertResult(); + } + + /** + * Verify that given a table where only the middle tablet requires compaction, and given a tablet range, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns a range to compact that middle + * tablet. + */ + @Test + void testMiddleTabletNeedsCompaction() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null,2500) , (2500,4000) , (4000,7000) (7000,null), (8000,null) + addSplits("2500", "4000", "7000", "8000"); + + // Compact all tablets except the tablet with the extent of (4000,7000) + compact(null, "4000"); + compact("7000", null); + + expectExtent("4000", "7000"); + assertResult(); + } + + /** + * Verify that given a table where multiple series of tablets at different locations require compaction, and given a tablet range, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns multiple compaction ranges that + * includes a series of contiguous compactable tablets by merging their extents. + */ + @Test + void testMultipleInnerCompactionRanges() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), + // (5500, 6500), (6500,7000), (7000,7500), (7500,8500) (8500,9000) + addSplits("1000", "1500", "2000", "2500", "3000", "4000", "5500", "6500", "7000", "7500", "8500", "9000"); + + // Compact extents to make a series of compacted tablets then track the series of uncompacted tablets: + compact(null, "1000"); + compact("2500", "3000"); + compact("6500", "7000"); + compact("8500", null); + + // Track uncompacted extent series: + expectExtent("1000", "2500"); // Tablets: (1000, 1500), (1500,2000), (2000,2500) + expectExtent("3000", "6500"); // Tablets: (3000,4000), (4000,5500), (5500, 6500) + expectExtent("7000", "8500"); // Tablets: (7000,7500), (7500,8500) + assertResult(); + } + + /** + * Verify that given a table where all tablets starting from row 2000 to null are compacted, and given a tablet range from row 2000 to null, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} returns an empty list. + */ + @Test + void testCompactableTabletOutsideInputRange() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), + // (5500, 6500), (6500,7000), (7000,7500), (7500,8500) (8500,9000) + addSplits("1000", "1500", "2000", "2500", "3000", "4000", "5500", "6500", "7000", "7500", "8500", "9000"); + + // Compact all tablets starting from row 2000 + compact("2000", null); + + // Only look at the range (2000,null) so expectedExtents should be empty + givenBegin("2000"); + givenEnd(null); + assertResult(); + } + + /** + * Verify that given a table with a compactable tablet, and given a key extent range that is contained within a tablet's range, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} will return the full extent of the tablet. + */ + @Test + void testInputInCompactableTabletRange() + throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), + // (5500, 6500), (6500,7000), (7000,7500), (7500,8500) (8500,9000) + addSplits("1000", "1500", "2000", "2500", "3000", "4000", "5500", "6500", "7000", "7500", "8500", "9000"); + + // Compact all tablets from null to row 1000 and from row 1500 to null + compact(null, "1000"); + compact("1500", null); + + // Compactable tablet: (1000,1500) + givenBegin("1000"); + givenEnd("1250"); + expectExtent("1000", "1500"); + assertResult(); + } + + /** + * Verify that given a table with compactable tablets, a key extent range, starting row and ending row, mergeExtents = true, and compactTablets = true, + * {@link TabletExtentChecker#findCompactableTablets(ClientContext, String, Text, Text, boolean, boolean)} will return an empty list after compacting + * the tablets automatically in {@link TabletExtentChecker#checkTablets(ClientContext, String, Text, Text, boolean, boolean, boolean)}. + */ + + @Test + void testCompactionParameter() throws AccumuloException, TableNotFoundException, TableExistsException, AccumuloSecurityException, IOException { + createTableAndWriteData(); + + // Tablet Extents: (null, 1000), (1000, 1500), (1500,2000), (2000,2500), (2500,3000), (3000,4000), (4000,5500), + // (5500, 6500), (6500,7000), (7000,7500), (7500,8500) (8500,9000) + addSplits("1000", "1500", "2000", "2500", "3000", "4000", "5500", "6500", "7000", "7500", "8500", "9000"); + + assertCompactedResult(); + } + + } + + private void createTableAndWriteData() throws AccumuloException, TableExistsException, AccumuloSecurityException, TableNotFoundException { + client.tableOperations().create(TABLE_NAME); + + try (BatchWriter writer = client.createBatchWriter(TABLE_NAME)) { + for (int i = 0; i < 10000; i += 250) { + String row = String.format("%04d", i); + Mutation m = new Mutation(row); + m.put("cf", "cq", "v"); + writer.addMutation(m); + } + } + client.tableOperations().flush(TABLE_NAME); + } + + private void addSplits(String... splits) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { + SortedSet set = new TreeSet<>(); + Arrays.stream(splits).map(Text::new).forEach(set::add); + client.tableOperations().addSplits(TABLE_NAME, set); + } + + private void compact(String start, String end) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { + client.tableOperations().compact(TABLE_NAME, getText(start), getText(end), true, true); + } + + private Text getText(String arg) { + return (arg == null) ? null : new Text(arg); + } + + private void expectExtent(String start, String end) { + expectedExtents.add(Pair.of(getText(start), getText(end))); + } + + private void givenBegin(String begin) { + this.begin = begin; + } + + private void givenEnd(String end) { + this.end = end; + } + + private void assertResult() throws AccumuloException, TableNotFoundException, IOException { + List> result = TabletExtentChecker.findCompactableTablets((ClientContext) client, TABLE_NAME, getText(begin), getText(end), + mergeExtents, true); + assertEquals(expectedExtents, result); + } + + private void assertCompactedResult() throws AccumuloException, TableNotFoundException, AccumuloSecurityException, IOException { + // Build the list of compactable tablets + List> result = TabletExtentChecker.findCompactableTablets((ClientContext) client, TABLE_NAME, getText(begin), getText(end), + mergeExtents, true); + + // Compact the tablets instead of suggesting compactions (compactTablets == true) + TabletExtentChecker.checkTablets((ClientContext) client, TABLE_NAME, getText(begin), getText(end), mergeExtents, true, true); + + List> result2 = TabletExtentChecker.findCompactableTablets((ClientContext) client, TABLE_NAME, getText(begin), getText(end), + mergeExtents, true); + + // Fetch the list of compactable tablets within the same range. + List> extents = TabletExtentChecker.findCompactableTablets((ClientContext) client, TABLE_NAME, getText(begin), getText(end), + mergeExtents, false); + + // Verify the list is empty, indicating success. + assertTrue(extents.isEmpty()); + } + +}