diff --git a/warehouse/query-core/src/test/java/datawave/query/IfThisTestFailsThenHitTermsAreBroken.java b/warehouse/query-core/src/test/java/datawave/query/IfThisTestFailsThenHitTermsAreBroken.java index 9dbba1f1579..d488037a1a6 100644 --- a/warehouse/query-core/src/test/java/datawave/query/IfThisTestFailsThenHitTermsAreBroken.java +++ b/warehouse/query-core/src/test/java/datawave/query/IfThisTestFailsThenHitTermsAreBroken.java @@ -1,61 +1,40 @@ package datawave.query; -import java.io.File; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Collections; +import java.nio.file.Path; import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; import java.util.TimeZone; -import java.util.UUID; import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; -import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; -import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.hadoop.io.Text; import org.apache.log4j.Logger; -import org.jboss.arquillian.junit.Arquillian; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; - -import datawave.core.query.configuration.GenericQueryConfiguration; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + import datawave.data.ColumnFamilyConstants; import datawave.data.hash.UID; import datawave.data.type.LcNoDiacriticsType; import datawave.data.type.Type; -import datawave.helpers.PrintUtility; import datawave.ingest.data.TypeRegistry; import datawave.ingest.protobuf.Uid; import datawave.marking.MarkingFunctions; -import datawave.microservice.query.QueryImpl; -import datawave.query.attributes.Document; -import datawave.query.function.deserializer.KryoDocumentDeserializer; +import datawave.query.index.day.IndexIngestUtil; import datawave.query.planner.DefaultQueryPlanner; import datawave.query.tables.ShardQueryLogic; +import datawave.query.util.AbstractQueryTest; import datawave.query.util.DateIndexHelperFactory; import datawave.query.util.MetadataHelperFactory; -import datawave.security.util.ScannerHelper; import datawave.table.constants.TableName; -import datawave.test.HitTermAssertions; /** * @@ -63,117 +42,55 @@ * grouping context levels, and assures that the hits contain the fields with the correct grouping context. It also confirms that an 'or' query that hits * different fields in the returned documents will have the correct hit terms. *

- * The same tests are made against document ranges and shard ranges - *

* If this test fails, then hit terms are broken... maybe... probably... */ -public abstract class IfThisTestFailsThenHitTermsAreBroken { - - @ClassRule - // Temporary folders are not successfully deleted in this test with @Rule for some reason, but they are with @ClassRule. - public static TemporaryFolder temporaryFolder = new TemporaryFolder(); - - enum WhatKindaRange { - SHARD, DOCUMENT - } +public class IfThisTestFailsThenHitTermsAreBroken extends AbstractQueryTest { private static final Logger log = Logger.getLogger(IfThisTestFailsThenHitTermsAreBroken.class); + private static final Authorizations auths = new Authorizations("A"); - protected static AccumuloClient client = null; - - protected static final Authorizations auths = new Authorizations("A"); - - protected Set authSet = Collections.singleton(auths); + private static AccumuloClient clientForTest; - protected ShardQueryLogic logic = null; + protected ShardQueryLogic logic; - protected KryoDocumentDeserializer deserializer; - - private final DateFormat format = new SimpleDateFormat("yyyyMMdd"); - - private String query; - private int expectedResultCount = 0; - private final Set results = new HashSet<>(); - private final Map extraParameters = new HashMap<>(); - private final HitTermAssertions hitTermAssertions = new HitTermAssertions(); - - @RunWith(Arquillian.class) - public static class ShardRangeTest extends IfThisTestFailsThenHitTermsAreBroken { - - @BeforeClass - public static void beforeAll() throws Exception { - QueryTestTableHelper qtth = new QueryTestTableHelper(ShardRangeTest.class.toString(), log); - client = qtth.client; - - MoreTestData.writeItAll(client, WhatKindaRange.SHARD); - if (log.isDebugEnabled()) { - log.debug("testWithShardRange"); - PrintUtility.printTable(client, auths, TableName.SHARD); - PrintUtility.printTable(client, auths, TableName.SHARD_INDEX); - PrintUtility.printTable(client, auths, QueryTestTableHelper.MODEL_TABLE_NAME); - } - } - - @Before - public void setup() throws Exception { - super.setup(); - logic.setCollapseUids(true); - } + @Override + public ShardQueryLogic getLogic() { + return logic; } - @RunWith(Arquillian.class) - public static class DocumentRangeTest extends IfThisTestFailsThenHitTermsAreBroken { - - @BeforeClass - public static void beforeAll() throws Exception { - QueryTestTableHelper qtth = new QueryTestTableHelper(DocumentRangeTest.class.toString(), log); - client = qtth.client; - - MoreTestData.writeItAll(client, WhatKindaRange.DOCUMENT); - if (log.isDebugEnabled()) { - log.debug("testWithDocumentRange"); - PrintUtility.printTable(client, auths, TableName.SHARD); - PrintUtility.printTable(client, auths, TableName.SHARD_INDEX); - PrintUtility.printTable(client, auths, QueryTestTableHelper.MODEL_TABLE_NAME); - } - } + @Override + public Authorizations getAuths() { + return auths; + } - @Before - public void setup() throws Exception { - super.setup(); - logic.setCollapseUids(false); - } + @Override + protected void extraConfigurations() { + disableQueryPlanAssertion(); } - @Before - public void beforeEach() { - extraParameters.put("hit.list", "true"); - query = null; - expectedResultCount = 0; + @Override + protected void extraAssertions() { + // no-op; hit term validation is handled by AbstractQueryTest via expectHitTermsRequiredAllOf/AnyOf } - @After - public void after() { - TypeRegistry.reset(); - System.clearProperty("type.metadata.dir"); + @BeforeAll + public static void beforeAll() throws Exception { + TimeZone.setDefault(TimeZone.getTimeZone("GMT")); - extraParameters.clear(); - results.clear(); - hitTermAssertions.resetState(); + QueryTestTableHelper qtth = new QueryTestTableHelper(IfThisTestFailsThenHitTermsAreBroken.class.toString(), log); + clientForTest = qtth.client; + MoreTestData.writeItAll(clientForTest); } - @AfterClass - public static void teardown() { + @AfterAll + public static void afterAll() { TypeRegistry.reset(); } - @Before - public void setup() throws Exception { - TimeZone.setDefault(TimeZone.getTimeZone("GMT")); - File tempDir = temporaryFolder.newFolder(); - System.setProperty("type.metadata.dir", tempDir.getAbsolutePath()); + @BeforeEach + public void setup(@TempDir Path tempDir) throws Exception { + System.setProperty("type.metadata.dir", tempDir.toAbsolutePath().toString()); System.setProperty("dw.metadatahelper.all.auths", "A,B,C,D,T,U,V,W,X,Y,Z"); - log.info("using tempFolder " + tempDir); logic = new ShardQueryLogic(); logic.setMetadataTableName(QueryTestTableHelper.MODEL_TABLE_NAME); @@ -189,189 +106,139 @@ public void setup() throws Exception { logic.setMetadataHelperFactory(new MetadataHelperFactory()); logic.setDateIndexHelperFactory(new DateIndexHelperFactory()); logic.setMaxEvaluationPipelines(1); - deserializer = new KryoDocumentDeserializer(); - } - - public void dumpTable(String tableName) throws Exception { - Scanner s = ScannerHelper.createScanner(client, tableName, authSet); - Range r = new Range(); - s.setRange(r); - for (Entry entry : s) { - if (log.isDebugEnabled()) { - log.debug(entry.getKey() + " " + entry.getValue()); - } - } - } - - private void drive() throws Exception { - // might be called multiple times by a single test, so clear results - results.clear(); - - setupQuery(); - executeQuery(); - assertResults(); - } - - private void setupQuery() throws Exception { - log.debug("runTestQuery"); - log.trace("Creating QueryImpl"); - QueryImpl settings = new QueryImpl(); - - settings.setBeginDate(format.parse("20091231")); - settings.setEndDate(format.parse("20150101")); - settings.setPagesize(Integer.MAX_VALUE); - settings.setQueryAuthorizations(auths.serialize()); - settings.setQuery(query); - settings.setParameters(extraParameters); - settings.setId(UUID.randomUUID()); - - log.debug("query: " + settings.getQuery()); - log.debug("logic: " + settings.getQueryLogicName()); - GenericQueryConfiguration config = logic.initialize(client, settings, authSet); - logic.setupQuery(config); + setClientForTest(clientForTest); + givenDate("20091231", "20150101"); + givenParameter("hit.list", "true"); } - private void executeQuery() { - for (Entry entry : logic) { - Document document = deserializer.apply(entry).getValue(); - results.add(document); - } - } - - private void assertResults() { - Assert.assertEquals(expectedResultCount, results.size()); - for (Document result : results) { - assertResult(result); - } - } - - private void assertResult(Document result) { - boolean validated = hitTermAssertions.assertHitTerms(result); - Assert.assertEquals(hitTermAssertions.hitTermExpected(), validated); + @AfterEach + public void tearDown() { + System.clearProperty("type.metadata.dir"); } @Test public void testUnion() throws Exception { // sanity check that expects two documents - query = "UUID == 'First' || UUID == 'Second'"; - hitTermAssertions.withRequiredAnyOf("UUID.0:First", "UUID.0:Second"); - expectedResultCount = 2; - drive(); + givenQuery("UUID == 'First' || UUID == 'Second'"); + expectHitTermsRequiredAnyOf("UUID.0:First", "UUID.0:Second"); + expectResultCount(2); + planAndExecuteQuery(); - extraParameters.put("return.fields", "*"); - drive(); + givenParameter("return.fields", "*"); + planAndExecuteQuery(); } @Test public void testDoubleNestedUnion() throws Exception { // look for FOO or BAR, expecting the hit_terms to be in the right places - query = "( UUID == 'First' || UUID == 'Second' ) && ( FOO == 'FOO' || BAR == 'BAR' )"; - hitTermAssertions.withRequiredAnyOf("UUID.0:First", "UUID.0:Second"); - hitTermAssertions.withRequiredAnyOf("FOO.0:FOO", "BAR.0:BAR"); - expectedResultCount = 2; - drive(); - - extraParameters.put("return.fields", "*"); - drive(); + givenQuery("( UUID == 'First' || UUID == 'Second' ) && ( FOO == 'FOO' || BAR == 'BAR' )"); + expectHitTermsRequiredAnyOf("UUID.0:First", "UUID.0:Second"); + expectHitTermsRequiredAnyOf("FOO.0:FOO", "BAR.0:BAR"); + expectResultCount(2); + planAndExecuteQuery(); + + givenParameter("return.fields", "*"); + planAndExecuteQuery(); } @Test public void testNestedUnionWithAnchorOfDifferentGroupingContext() throws Exception { // should find NAME0 in different grouping contexts, but the hit terms should be correct - query = "( UUID == 'First' || UUID == 'Second' ) && NAME == 'NAME0'"; - hitTermAssertions.withRequiredAnyOf("UUID.0:First", "UUID.0:Second"); - hitTermAssertions.withRequiredAnyOf("NAME.0:NAME0", "NAME.1:NAME0"); - expectedResultCount = 2; - drive(); - - extraParameters.put("return.fields", "*"); - drive(); + givenQuery("( UUID == 'First' || UUID == 'Second' ) && NAME == 'NAME0'"); + expectHitTermsRequiredAnyOf("UUID.0:First", "UUID.0:Second"); + expectHitTermsRequiredAnyOf("NAME.0:NAME0", "NAME.1:NAME0"); + expectResultCount(2); + planAndExecuteQuery(); + + givenParameter("return.fields", "*"); + planAndExecuteQuery(); } @Test public void testNestedUnionFirstDocumentReturned() throws Exception { // this may get initial hits in Second, but will return only First. Makes sure that hits from Second are not included in First - query = "( UUID == 'First' || UUID == 'Second' ) && NAME == 'Haiqu' && FOO == 'FOO'"; - hitTermAssertions.withRequiredAllOf("UUID.0:First", "NAME.2:Haiqu", "FOO.0:FOO"); - expectedResultCount = 1; - drive(); + givenQuery("( UUID == 'First' || UUID == 'Second' ) && NAME == 'Haiqu' && FOO == 'FOO'"); + expectHitTermsRequiredAllOf("UUID.0:First", "NAME.2:Haiqu", "FOO.0:FOO"); + expectResultCount(1); + planAndExecuteQuery(); - extraParameters.put("return.fields", "*"); - drive(); + givenParameter("return.fields", "*"); + planAndExecuteQuery(); } @Test public void testNestedUnionSecondDocumentReturned() throws Exception { // this may get initial hits in First, but will return only Second. Makes sure that hits from First are not included in Second - query = "( UUID == 'First' || UUID == 'Second' ) && NAME == 'Haiqu' && BAR == 'BAR'"; - hitTermAssertions.withRequiredAllOf("UUID.0:Second", "NAME.2:Haiqu", "BAR.0:BAR"); - expectedResultCount = 1; - drive(); + givenQuery("( UUID == 'First' || UUID == 'Second' ) && NAME == 'Haiqu' && BAR == 'BAR'"); + expectHitTermsRequiredAllOf("UUID.0:Second", "NAME.2:Haiqu", "BAR.0:BAR"); + expectResultCount(1); + planAndExecuteQuery(); - extraParameters.put("return.fields", "*"); - drive(); + givenParameter("return.fields", "*"); + planAndExecuteQuery(); } @Test public void testLargeNestedUnionStillOnlySecondDocumentReturned() throws Exception { // try to pull in hits from Third, should still hit only Second - query = "( UUID == 'First' || UUID == 'Second' || UUID == 'Third') && NAME == 'Haiqu' && BAR == 'BAR'"; - hitTermAssertions.withRequiredAllOf("UUID.0:Second", "NAME.2:Haiqu", "BAR.0:BAR"); - expectedResultCount = 1; - drive(); + givenQuery("( UUID == 'First' || UUID == 'Second' || UUID == 'Third') && NAME == 'Haiqu' && BAR == 'BAR'"); + expectHitTermsRequiredAllOf("UUID.0:Second", "NAME.2:Haiqu", "BAR.0:BAR"); + expectResultCount(1); + planAndExecuteQuery(); - extraParameters.put("return.fields", "*"); - drive(); + givenParameter("return.fields", "*"); + planAndExecuteQuery(); } @Test public void testNestedUnionWithFilterAnchorTerms() throws Exception { - query = "( UUID == 'First' || UUID == 'Second' || UUID == 'Third') && filter:includeRegex(NAME,'Haiqu') && filter:includeRegex(BAR,'BAR')"; - hitTermAssertions.withRequiredAllOf("UUID.0:Second", "NAME.2:Haiqu", "BAR.0:BAR"); - expectedResultCount = 1; - drive(); + givenQuery("( UUID == 'First' || UUID == 'Second' || UUID == 'Third') && filter:includeRegex(NAME,'Haiqu') && filter:includeRegex(BAR,'BAR')"); + expectHitTermsRequiredAllOf("UUID.0:Second", "NAME.2:Haiqu", "BAR.0:BAR"); + expectResultCount(1); + planAndExecuteQuery(); - extraParameters.put("return.fields", "*"); - drive(); + givenParameter("return.fields", "*"); + planAndExecuteQuery(); } @Test public void testSecondAndBarBar() throws Exception { - query = "UUID == 'Second' && BAR == 'BAR'"; - hitTermAssertions.withRequiredAllOf("UUID.0:Second", "BAR.0:BAR"); - expectedResultCount = 1; - drive(); + givenQuery("UUID == 'Second' && BAR == 'BAR'"); + expectHitTermsRequiredAllOf("UUID.0:Second", "BAR.0:BAR"); + expectResultCount(1); + planAndExecuteQuery(); - extraParameters.put("return.fields", "*"); - drive(); + givenParameter("return.fields", "*"); + planAndExecuteQuery(); } @Test public void testFilterOccurrenceFunction() throws Exception { - query = "NAME == 'Haiqu' && BAR == 'BAR' && filter:occurrence(NAME, '==', 3)"; - hitTermAssertions.withRequiredAllOf("BAR.0:BAR", "NAME.2:Haiqu"); - expectedResultCount = 1; - drive(); + givenQuery("NAME == 'Haiqu' && BAR == 'BAR' && filter:occurrence(NAME, '==', 3)"); + expectHitTermsRequiredAllOf("BAR.0:BAR", "NAME.2:Haiqu"); + expectResultCount(1); + planAndExecuteQuery(); - extraParameters.put("return.fields", "*"); - drive(); + givenParameter("return.fields", "*"); + planAndExecuteQuery(); } @Test public void testFilterIsNotNull() throws Exception { - query = "UUID == 'First' && filter:isNotNull(NAME)"; - hitTermAssertions.withRequiredAnyOf("UUID.0:First"); - expectedResultCount = 1; - drive(); + givenQuery("UUID == 'First' && filter:isNotNull(NAME)"); + expectHitTermsRequiredAnyOf("UUID.0:First"); + expectResultCount(1); + planAndExecuteQuery(); - extraParameters.put("return.fields", "*"); - drive(); + givenParameter("return.fields", "*"); + planAndExecuteQuery(); } private static class MoreTestData { private static final Type lcNoDiacriticsType = new LcNoDiacriticsType(); + private static final IndexIngestUtil ingestUtil = new IndexIngestUtil(); protected static final String datatype = "test"; protected static final String date = "20130101"; @@ -381,8 +248,10 @@ private static class MoreTestData { protected static final long timeStamp = 1356998400000L; /** + * Writes test data using the document range only; {@link IndexIngestUtil} derives the other shard index table variants (NO_UID_INDEX, TRUNCATED_INDEX, + * etc.) that {@link AbstractQueryTest} iterates over. */ - public static void writeItAll(AccumuloClient client, WhatKindaRange range) throws Exception { + public static void writeItAll(AccumuloClient client) throws Exception { BatchWriter bw = null; BatchWriterConfig bwConfig = new BatchWriterConfig().setMaxMemory(1000L).setMaxLatency(1, TimeUnit.SECONDS).setMaxWriteThreads(1); Mutation mutation; @@ -430,20 +299,17 @@ public static void writeItAll(AccumuloClient client, WhatKindaRange range) throw // write shard index table: bw = client.createBatchWriter(TableName.SHARD_INDEX, bwConfig); mutation = new Mutation(lcNoDiacriticsType.normalize("First")); - mutation.put("UUID".toUpperCase(), shard + "\u0000" + datatype, columnVisibility, timeStamp, - range == WhatKindaRange.SHARD ? getValueForNuthinAndYourHitsForFree() : getValueForBuilderFor(firstUID)); + mutation.put("UUID".toUpperCase(), shard + "\u0000" + datatype, columnVisibility, timeStamp, getValueForBuilderFor(firstUID)); bw.addMutation(mutation); bw = client.createBatchWriter(TableName.SHARD_INDEX, bwConfig); mutation = new Mutation(lcNoDiacriticsType.normalize("Second")); - mutation.put("UUID".toUpperCase(), shard + "\u0000" + datatype, columnVisibility, timeStamp, - range == WhatKindaRange.SHARD ? getValueForNuthinAndYourHitsForFree() : getValueForBuilderFor(secondUID)); + mutation.put("UUID".toUpperCase(), shard + "\u0000" + datatype, columnVisibility, timeStamp, getValueForBuilderFor(secondUID)); bw.addMutation(mutation); bw = client.createBatchWriter(TableName.SHARD_INDEX, bwConfig); mutation = new Mutation(lcNoDiacriticsType.normalize("Third")); - mutation.put("UUID".toUpperCase(), shard + "\u0000" + datatype, columnVisibility, timeStamp, - range == WhatKindaRange.SHARD ? getValueForNuthinAndYourHitsForFree() : getValueForBuilderFor(thirdUID)); + mutation.put("UUID".toUpperCase(), shard + "\u0000" + datatype, columnVisibility, timeStamp, getValueForBuilderFor(thirdUID)); bw.addMutation(mutation); } finally { @@ -457,20 +323,17 @@ public static void writeItAll(AccumuloClient client, WhatKindaRange range) throw // write the reverse index table: bw = client.createBatchWriter(TableName.SHARD_RINDEX, bwConfig); mutation = new Mutation(new StringBuilder(lcNoDiacriticsType.normalize("First")).reverse()); - mutation.put("UUID".toUpperCase(), shard + "\u0000" + datatype, columnVisibility, timeStamp, - range == WhatKindaRange.SHARD ? getValueForNuthinAndYourHitsForFree() : getValueForBuilderFor(firstUID)); + mutation.put("UUID".toUpperCase(), shard + "\u0000" + datatype, columnVisibility, timeStamp, getValueForBuilderFor(firstUID)); bw.addMutation(mutation); bw = client.createBatchWriter(TableName.SHARD_RINDEX, bwConfig); mutation = new Mutation(new StringBuilder(lcNoDiacriticsType.normalize("Second")).reverse()); - mutation.put("UUID".toUpperCase(), shard + "\u0000" + datatype, columnVisibility, timeStamp, - range == WhatKindaRange.SHARD ? getValueForNuthinAndYourHitsForFree() : getValueForBuilderFor(secondUID)); + mutation.put("UUID".toUpperCase(), shard + "\u0000" + datatype, columnVisibility, timeStamp, getValueForBuilderFor(secondUID)); bw.addMutation(mutation); bw = client.createBatchWriter(TableName.SHARD_RINDEX, bwConfig); mutation = new Mutation(new StringBuilder(lcNoDiacriticsType.normalize("Third")).reverse()); - mutation.put("UUID".toUpperCase(), shard + "\u0000" + datatype, columnVisibility, timeStamp, - range == WhatKindaRange.SHARD ? getValueForNuthinAndYourHitsForFree() : getValueForBuilderFor(thirdUID)); + mutation.put("UUID".toUpperCase(), shard + "\u0000" + datatype, columnVisibility, timeStamp, getValueForBuilderFor(thirdUID)); bw.addMutation(mutation); } finally { @@ -546,6 +409,9 @@ public static void writeItAll(AccumuloClient client, WhatKindaRange range) throw bw.close(); } } + + Authorizations auths = new Authorizations("A"); + ingestUtil.write(client, auths); } } @@ -558,13 +424,4 @@ private static Value getValueForBuilderFor(String... in) { builder.setIGNORE(false); return new Value(builder.build().toByteArray()); } - - private static Value getValueForNuthinAndYourHitsForFree() { - Uid.List.Builder builder = Uid.List.newBuilder(); - // technically this value could be zero due to delete mutations, depending on the combiner - // set on the table the key may or may not be deleted. - builder.setCOUNT(50); // better not be zero!!!! - builder.setIGNORE(true); // better be true!!! - return new Value(builder.build().toByteArray()); - } }