From 0767441d2f97f4797f51fa57d8acbd00c2e9aecc Mon Sep 17 00:00:00 2001 From: "chris.lockard" Date: Tue, 2 Jun 2026 12:38:45 -0700 Subject: [PATCH 1/2] Make solr field/schema resolvers respect the tokenized field of attributes --- .../data/impl/types/CoreAttributes.java | 2 +- .../source/solr/DynamicSchemaResolver.java | 66 +++--- .../source/solr/SolrFilterDelegate.java | 13 +- .../solr/DynamicSchemaResolverTest.java | 204 +++++++++++++++++- .../source/solr/SolrFilterDelegateTest.java | 34 ++- .../solr/provider/SolrProviderQuery.java | 2 +- .../solr/provider/SolrProviderSpatial.java | 2 +- .../solr/provider/SolrProviderTestUtil.java | 12 +- .../org/codice/solr/query/SchemaField.java | 10 +- .../solr/query/SchemaFieldResolver.java | 16 +- .../solr/query/SolrQueryFilterVisitor.java | 40 ++-- .../query/SolrQueryFilterVisitorTest.java | 35 ++- 12 files changed, 352 insertions(+), 84 deletions(-) diff --git a/catalog/core/catalog-core-api-impl/src/main/java/ddf/catalog/data/impl/types/CoreAttributes.java b/catalog/core/catalog-core-api-impl/src/main/java/ddf/catalog/data/impl/types/CoreAttributes.java index d1e223df60cd..c55d10604cf2 100644 --- a/catalog/core/catalog-core-api-impl/src/main/java/ddf/catalog/data/impl/types/CoreAttributes.java +++ b/catalog/core/catalog-core-api-impl/src/main/java/ddf/catalog/data/impl/types/CoreAttributes.java @@ -206,7 +206,7 @@ public class CoreAttributes implements Core, MetacardType { METACARD_TAGS, true /* indexed */, true /* stored */, - false /* tokenized */, + true /* tokenized */, true /* multivalued */, BasicTypes.STRING_TYPE)); descriptors.add( diff --git a/catalog/solr/catalog-solr-core/src/main/java/ddf/catalog/source/solr/DynamicSchemaResolver.java b/catalog/solr/catalog-solr-core/src/main/java/ddf/catalog/source/solr/DynamicSchemaResolver.java index 58a0beccfd0c..6e0587c6842c 100644 --- a/catalog/solr/catalog-solr-core/src/main/java/ddf/catalog/source/solr/DynamicSchemaResolver.java +++ b/catalog/solr/catalog-solr-core/src/main/java/ddf/catalog/source/solr/DynamicSchemaResolver.java @@ -178,9 +178,6 @@ public DynamicSchemaResolver(List additionalFields) { textSortCharacterLimit = 127; } fieldsCache.add(Metacard.ID + SchemaFields.TEXT_SUFFIX); - fieldsCache.add(Metacard.ID + SchemaFields.TEXT_SUFFIX + SchemaFields.TOKENIZED); - fieldsCache.add( - Metacard.ID + SchemaFields.TEXT_SUFFIX + SchemaFields.TOKENIZED + SchemaFields.HAS_CASE); fieldsCache.add(SchemaFields.METACARD_TYPE_FIELD_NAME); fieldsCache.add(SchemaFields.METACARD_TYPE_OBJECT_FIELD_NAME); @@ -273,10 +270,10 @@ void addFields(Metacard metacard, SolrInputDocument solrInputDocument) if (AttributeFormat.XML.equals(format) && solrInputDocument.getFieldValue( formatIndexName + getSpecialIndexSuffix(AttributeFormat.STRING)) - == null) { + == null + && ad.isTokenized()) { List parsedTexts = parseTextFrom(attributeValues); - - // parsedTexts => *_txt_tokenized + // parsedTexts => *_txt_tokenized - only if attribute is tokenized String specialStringIndexName = ad.getName() + getFieldSuffix(AttributeFormat.STRING) @@ -294,12 +291,14 @@ void addFields(Metacard metacard, SolrInputDocument solrInputDocument) solrInputDocument.addField( ad.getName() + getFieldSuffix(AttributeFormat.STRING), truncatedValues); - // *_txt_tokenized - solrInputDocument.addField( - ad.getName() - + getFieldSuffix(AttributeFormat.STRING) - + getSpecialIndexSuffix(AttributeFormat.STRING), - attributeValues); + // *_txt_tokenized - only if attribute is tokenized + if (ad.isTokenized()) { + solrInputDocument.addField( + ad.getName() + + getFieldSuffix(AttributeFormat.STRING) + + getSpecialIndexSuffix(AttributeFormat.STRING), + attributeValues); + } } else if (AttributeFormat.OBJECT.equals(format)) { ByteArrayOutputStream byteArrayOS = new ByteArrayOutputStream(); List byteArrays = new ArrayList<>(); @@ -594,13 +593,12 @@ String getField( Map enabledFeatures) { final String fieldSuffix = getFieldSuffix(format); - String fieldName = - propertyName - + fieldSuffix - + (isSearchedAsExactValue ? "" : getSpecialIndexSuffix(format, enabledFeatures)); + String baseName = propertyName + fieldSuffix; + String extendedName = + baseName + (isSearchedAsExactValue ? "" : getSpecialIndexSuffix(format, enabledFeatures)); - if (fieldsCache.contains(fieldName)) { - return fieldName; + if (fieldsCache.contains(extendedName)) { + return extendedName; } switch (format) { @@ -613,13 +611,14 @@ String getField( default: break; } - + // if we have the base name in the cache but not the extended name assume the extended doesn't + // exist otherwise fallback to the old convention + String bestGuess = fieldsCache.contains(baseName) ? baseName : extendedName; LOGGER.debug( "Could not find exact schema field name for [{}], attempting to search with [{}]", propertyName, - fieldName); - - return fieldName; + bestGuess); + return bestGuess; } private String getFieldSuffix(AttributeFormat format) { @@ -658,8 +657,11 @@ String getCaseSensitiveField( && mappedPropertyName.endsWith(SchemaFields.PHONETICS)) { return mappedPropertyName; } - // TODO We can check if this field really does exist - return mappedPropertyName + SchemaFields.HAS_CASE; + String potentialField = mappedPropertyName + SchemaFields.HAS_CASE; + if (fieldsCache.contains(potentialField)) { + return potentialField; + } + return fieldsCache.contains(mappedPropertyName) ? mappedPropertyName : potentialField; } private String getSpecialIndexSuffix(AttributeFormat format) { @@ -706,6 +708,18 @@ private void addToFieldsCache(AttributeDescriptor descriptor) { fieldsCache.add(descriptor.getName() + schemaFields.getFieldSuffix(format)); + if (format.equals(AttributeFormat.GEOMETRY)) { + fieldsCache.add( + descriptor.getName() + + schemaFields.getFieldSuffix(format) + + getSpecialIndexSuffix(format)); + return; + } + + if (!descriptor.isTokenized()) { + return; + } + if (!getSpecialIndexSuffix(format).equals("")) { fieldsCache.add( descriptor.getName() @@ -892,9 +906,7 @@ private Set getAnyTextFields() { .collect(Collectors.toList()); } - return fields.stream() - .map(field -> field + SchemaFields.TEXT_SUFFIX) - .collect(Collectors.toSet()); + return new HashSet<>(fields); } private Set getAnyGeoFields() { diff --git a/catalog/solr/catalog-solr-core/src/main/java/ddf/catalog/source/solr/SolrFilterDelegate.java b/catalog/solr/catalog-solr-core/src/main/java/ddf/catalog/source/solr/SolrFilterDelegate.java index b7a74323ae52..0f4d108c8ddd 100644 --- a/catalog/solr/catalog-solr-core/src/main/java/ddf/catalog/source/solr/SolrFilterDelegate.java +++ b/catalog/solr/catalog-solr-core/src/main/java/ddf/catalog/source/solr/SolrFilterDelegate.java @@ -313,19 +313,14 @@ public SolrQuery propertyIsEqualTo(String propertyName, String literal, boolean private String wildcardSolrQuery( String searchPhrase, String propertyName, boolean isCaseSensitive, boolean isExact) { String solrQuery; - String tokenized = resolver.getSpecialIndexSuffix(AttributeFormat.STRING, enabledFeatures); if (Metacard.ANY_TEXT.equals(propertyName)) { solrQuery = resolver .anyTextFields() .map( - field -> { - if (!isExact) { - return field + tokenized; - } else { - return field; - } - }) + field -> + resolver.getField( + field, AttributeFormat.STRING, isExact, Collections.emptyMap())) .map( textField -> { if (isCaseSensitive && !isExact) { @@ -339,7 +334,7 @@ private String wildcardSolrQuery( } else { String field = getMappedPropertyName(propertyName, AttributeFormat.STRING, true); if (!isExact) { - field += tokenized; + field = getMappedPropertyName(propertyName, AttributeFormat.STRING, false); } if (isCaseSensitive && !isExact) { field = resolver.getCaseSensitiveField(field, enabledFeatures); diff --git a/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/DynamicSchemaResolverTest.java b/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/DynamicSchemaResolverTest.java index 6d092180abac..4da2eb9943c2 100644 --- a/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/DynamicSchemaResolverTest.java +++ b/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/DynamicSchemaResolverTest.java @@ -19,6 +19,7 @@ import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; @@ -54,7 +55,7 @@ public class DynamicSchemaResolverTest { - private static final int INITIAL_FIELDS_CACHE_COUNT = 5; + private static final int INITIAL_FIELDS_CACHE_COUNT = 3; private static final ObjectMapper METACARD_TYPE_MAPPER = MetacardTypeMapperFactory.newObjectMapper(); @@ -180,7 +181,7 @@ public void testAnyTextFieldPropertyParsing() throws Exception { // Perform Test List fields = resolver.anyTextFields().collect(Collectors.toList()); Truth.assertThat(fields) - .containsExactly("metadata_txt", "title_txt", "description_txt", "ext.extracted.text_txt"); + .containsExactly("metadata", "title", "description", "ext.extracted.text"); } @Test @@ -301,4 +302,203 @@ public void getFieldExactValue() { dynamicSchemaResolver.getField("unknown", AttributeFormat.STRING, true, enabledFeatures), is("unknown_txt")); } + + @Test + public void testAddFieldsWithXmlAttributeNotTokenized() throws Exception { + // Setup - XML attribute that is NOT tokenized + String metacardTypeName = "test"; + Set attributeDescriptors = new HashSet<>(1); + String attributeName = "metadata"; + boolean indexed = true; + boolean stored = true; + boolean tokenized = false; + boolean multiValued = false; + + attributeDescriptors.add( + new TestAttributeDescriptorImpl( + attributeName, + attributeName, + indexed, + stored, + tokenized, + multiValued, + BasicTypes.XML_TYPE)); + + Attribute mockAttribute = mock(Attribute.class); + when(mockAttribute.getValue()).thenReturn("test"); + when(mockAttribute.getValues()).thenReturn(Collections.singletonList("test")); + + Metacard mockMetacard = mock(Metacard.class, RETURNS_DEEP_STUBS); + when(mockMetacard.getMetacardType().getName()).thenReturn(metacardTypeName); + when(mockMetacard.getMetacardType().getAttributeDescriptors()).thenReturn(attributeDescriptors); + when(mockMetacard.getAttribute(attributeName)).thenReturn(mockAttribute); + + SolrInputDocument solrInputDocument = new SolrInputDocument(); + DynamicSchemaResolver resolver = new DynamicSchemaResolver(); + + // Perform Test + resolver.addFields(mockMetacard, solrInputDocument); + + // Verify - tokenized=false means the special string index should NOT be populated + assertThat(solrInputDocument.getFieldValue("metadata_txt_tokenized"), is(nullValue())); + } + + @Test + public void testAddFieldsWithXmlAttributeTokenized() throws Exception { + // Setup - XML attribute that IS tokenized + String metacardTypeName = "test"; + Set attributeDescriptors = new HashSet<>(1); + String attributeName = "metadata"; + boolean indexed = true; + boolean stored = true; + boolean tokenized = true; + boolean multiValued = false; + + attributeDescriptors.add( + new TestAttributeDescriptorImpl( + attributeName, + attributeName, + indexed, + stored, + tokenized, + multiValued, + BasicTypes.XML_TYPE)); + + Attribute mockAttribute = mock(Attribute.class); + when(mockAttribute.getValue()).thenReturn("test"); + when(mockAttribute.getValues()).thenReturn(Collections.singletonList("test")); + + Metacard mockMetacard = mock(Metacard.class, RETURNS_DEEP_STUBS); + when(mockMetacard.getMetacardType().getName()).thenReturn(metacardTypeName); + when(mockMetacard.getMetacardType().getAttributeDescriptors()).thenReturn(attributeDescriptors); + when(mockMetacard.getAttribute(attributeName)).thenReturn(mockAttribute); + + SolrInputDocument solrInputDocument = new SolrInputDocument(); + DynamicSchemaResolver resolver = new DynamicSchemaResolver(); + + // Perform Test + resolver.addFields(mockMetacard, solrInputDocument); + + // Verify - tokenized=true means the special string index SHOULD be populated + assertThat(solrInputDocument.getFieldValue("metadata_txt_tokenized"), notNullValue()); + } + + @Test + public void testAddFieldsWithStringAttributeNotTokenizedAddsTxtNotTokenizedField() + throws Exception { + // Setup - STRING attribute that is NOT tokenized + String metacardTypeName = "test"; + Set attributeDescriptors = new HashSet<>(1); + String attributeName = "title"; + boolean indexed = true; + boolean stored = true; + boolean tokenized = false; + boolean multiValued = false; + + attributeDescriptors.add( + new TestAttributeDescriptorImpl( + attributeName, + attributeName, + indexed, + stored, + tokenized, + multiValued, + BasicTypes.STRING_TYPE)); + + Attribute mockAttribute = mock(Attribute.class); + when(mockAttribute.getValue()).thenReturn("test value"); + when(mockAttribute.getValues()).thenReturn(Collections.singletonList("test value")); + + Metacard mockMetacard = mock(Metacard.class, RETURNS_DEEP_STUBS); + when(mockMetacard.getMetacardType().getName()).thenReturn(metacardTypeName); + when(mockMetacard.getMetacardType().getAttributeDescriptors()).thenReturn(attributeDescriptors); + when(mockMetacard.getAttribute(attributeName)).thenReturn(mockAttribute); + + SolrInputDocument solrInputDocument = new SolrInputDocument(); + DynamicSchemaResolver resolver = new DynamicSchemaResolver(); + + // Perform Test + resolver.addFields(mockMetacard, solrInputDocument); + + // Verify - tokenized=false means the tokenized field should NOT be populated + assertThat(solrInputDocument.getFieldValue("title_txt"), is("test value")); + assertThat(solrInputDocument.getFieldValue("title_txt_tokenized"), is(nullValue())); + } + + @Test + public void testAddFieldsWithStringAttributeTokenizedAddsBothTxtAndTokenizedField() + throws Exception { + // Setup - STRING attribute that IS tokenized + String metacardTypeName = "test"; + Set attributeDescriptors = new HashSet<>(1); + String attributeName = "title"; + boolean indexed = true; + boolean stored = true; + boolean tokenized = true; + boolean multiValued = false; + + attributeDescriptors.add( + new TestAttributeDescriptorImpl( + attributeName, + attributeName, + indexed, + stored, + tokenized, + multiValued, + BasicTypes.STRING_TYPE)); + + Attribute mockAttribute = mock(Attribute.class); + when(mockAttribute.getValue()).thenReturn("test value"); + when(mockAttribute.getValues()).thenReturn(Collections.singletonList("test value")); + + Metacard mockMetacard = mock(Metacard.class, RETURNS_DEEP_STUBS); + when(mockMetacard.getMetacardType().getName()).thenReturn(metacardTypeName); + when(mockMetacard.getMetacardType().getAttributeDescriptors()).thenReturn(attributeDescriptors); + when(mockMetacard.getAttribute(attributeName)).thenReturn(mockAttribute); + + SolrInputDocument solrInputDocument = new SolrInputDocument(); + DynamicSchemaResolver resolver = new DynamicSchemaResolver(); + + // Perform Test + resolver.addFields(mockMetacard, solrInputDocument); + + // Verify - tokenized=true means both fields SHOULD be populated + assertThat(solrInputDocument.getFieldValue("title_txt"), is("test value")); + assertThat(solrInputDocument.getFieldValue("title_txt_tokenized"), is("test value")); + } + + @Test + public void testGetFieldNumericalFallsBackToRequestedSuffix() { + // When no numerical field exists in cache, should fall back to the requested suffix + // fieldsCache is empty for a new resolver + assertThat( + dynamicSchemaResolver.getField( + "unknown", AttributeFormat.INTEGER, false, Collections.emptyMap()), + is("unknown_int")); + assertThat( + dynamicSchemaResolver.getField( + "unknown", AttributeFormat.LONG, false, Collections.emptyMap()), + is("unknown_lng")); + assertThat( + dynamicSchemaResolver.getField( + "unknown", AttributeFormat.DOUBLE, false, Collections.emptyMap()), + is("unknown_dbl")); + assertThat( + dynamicSchemaResolver.getField( + "unknown", AttributeFormat.FLOAT, false, Collections.emptyMap()), + is("unknown_flt")); + assertThat( + dynamicSchemaResolver.getField( + "unknown", AttributeFormat.SHORT, false, Collections.emptyMap()), + is("unknown_shr")); + } + + @Test + public void testGetFieldXmlReturnsTextPathSuffix() { + // XML format should use TEXT_PATH (_tpt) suffix + assertThat( + dynamicSchemaResolver.getField( + "metadata", AttributeFormat.XML, false, Collections.emptyMap()), + is("metadata_xml_tpt")); + } } diff --git a/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/SolrFilterDelegateTest.java b/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/SolrFilterDelegateTest.java index 6411247594f3..c4ecc00778ad 100644 --- a/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/SolrFilterDelegateTest.java +++ b/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/SolrFilterDelegateTest.java @@ -232,7 +232,9 @@ public void reservedSpecialCharactersIsLike() { .thenReturn(SchemaFields.TOKENIZED); when(mockResolver.getCaseSensitiveField("testProperty_txt_tokenized", Collections.emptyMap())) .thenReturn("testProperty_txt_tokenized_tokenized"); - + when(mockResolver.getField( + "testProperty", AttributeFormat.STRING, false, Collections.emptyMap())) + .thenReturn("testProperty_txt_tokenized"); // when searching for like reserved characters SolrQuery likeQuery = toTest.propertyIsLike("testProperty", "+ - && || ! ( ) { } [ ] ^ \" ~ : \\*?", true); @@ -445,6 +447,13 @@ public void testPropertyIsLikeTermAndWildcard() { .thenReturn(Collections.singletonList("metadata_txt").stream()); when(mockResolver.getSpecialIndexSuffix(AttributeFormat.STRING, Collections.emptyMap())) .thenReturn(SchemaFields.TOKENIZED); + when(mockResolver.getField( + "metadata_txt", AttributeFormat.STRING, false, Collections.emptyMap())) + .thenReturn("metadata_txt_tokenized"); + when(mockResolver.getField("metadata", AttributeFormat.STRING, true, Collections.emptyMap())) + .thenReturn("metadata_txt"); + when(mockResolver.getField("metadata", AttributeFormat.STRING, false, Collections.emptyMap())) + .thenReturn("metadata_txt_tokenized"); String searchPhrase = "abc-123*"; String expectedQuery = "(" + TOKENIZED_METADATA_FIELD + ":(abc\\-123*))"; @@ -486,6 +495,13 @@ public void testPropertyIsLikeWildcardNoTokens() { .thenReturn(Collections.singletonList("metadata_txt").stream()); when(mockResolver.getSpecialIndexSuffix(AttributeFormat.STRING, Collections.emptyMap())) .thenReturn(SchemaFields.TOKENIZED); + when(mockResolver.getField( + "metadata_txt", AttributeFormat.STRING, false, Collections.emptyMap())) + .thenReturn("metadata_txt_tokenized"); + when(mockResolver.getField("metadata", AttributeFormat.STRING, true, Collections.emptyMap())) + .thenReturn("metadata_txt"); + when(mockResolver.getField("metadata", AttributeFormat.STRING, false, Collections.emptyMap())) + .thenReturn("metadata_txt_tokenized"); String searchPhrase = "title*"; String expectedQuery = "(" + TOKENIZED_METADATA_FIELD + ":(title*))"; @@ -502,6 +518,13 @@ public void testPropertyIsLikeMultipleTermsWithWildcard() { .thenReturn(Collections.singletonList("metadata_txt").stream()); when(mockResolver.getSpecialIndexSuffix(AttributeFormat.STRING, Collections.emptyMap())) .thenReturn(SchemaFields.TOKENIZED); + when(mockResolver.getField( + "metadata_txt", AttributeFormat.STRING, false, Collections.emptyMap())) + .thenReturn("metadata_txt_tokenized"); + when(mockResolver.getField("metadata", AttributeFormat.STRING, true, Collections.emptyMap())) + .thenReturn("metadata_txt"); + when(mockResolver.getField("metadata", AttributeFormat.STRING, false, Collections.emptyMap())) + .thenReturn("metadata_txt_tokenized"); String searchPhrase = "abc 123*"; String expectedQuery = "(" + TOKENIZED_METADATA_FIELD + ":(abc 123*))"; @@ -519,6 +542,13 @@ public void testPropertyIsLikeCaseSensitiveWildcard() { .thenReturn(SchemaFields.TOKENIZED); when(mockResolver.getCaseSensitiveField("metadata_txt_tokenized", Collections.emptyMap())) .thenReturn("metadata_txt_tokenized_has_case"); + when(mockResolver.getField( + "metadata_txt", AttributeFormat.STRING, false, Collections.emptyMap())) + .thenReturn("metadata_txt_tokenized"); + when(mockResolver.getField("metadata", AttributeFormat.STRING, true, Collections.emptyMap())) + .thenReturn("metadata_txt"); + when(mockResolver.getField("metadata", AttributeFormat.STRING, false, Collections.emptyMap())) + .thenReturn("metadata_txt_tokenized"); String searchPhrase = "abc-123*"; String expectedQuery = @@ -688,6 +718,8 @@ public void testBetweenCastException() { public void testPropertyIsInProximityTo() { when(mockResolver.getField("title", AttributeFormat.STRING, true, Collections.emptyMap())) .thenReturn("title_txt"); + when(mockResolver.getField("title", AttributeFormat.STRING, false, Collections.emptyMap())) + .thenReturn("title_txt_tokenized"); when(mockResolver.getSpecialIndexSuffix(AttributeFormat.STRING, Collections.emptyMap())) .thenReturn(SchemaFields.TOKENIZED); diff --git a/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/provider/SolrProviderQuery.java b/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/provider/SolrProviderQuery.java index 8cde7db013bc..6d2bfd35f276 100644 --- a/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/provider/SolrProviderQuery.java +++ b/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/provider/SolrProviderQuery.java @@ -206,7 +206,7 @@ public void testQueryMissingSortField() throws IngestException, UnsupportedQuery create(list, provider); - Filter txtFilter = getFilterBuilder().attribute("id").like().text("*"); + Filter txtFilter = getFilterBuilder().attribute(Metacard.ANY_TEXT).like().text("*"); QueryImpl query = new QueryImpl(txtFilter); diff --git a/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/provider/SolrProviderSpatial.java b/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/provider/SolrProviderSpatial.java index 208536263d27..383d0a1b1072 100644 --- a/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/provider/SolrProviderSpatial.java +++ b/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/provider/SolrProviderSpatial.java @@ -119,7 +119,7 @@ public void testSpatialPointRadius() throws Exception { // CREATE create(list, provider); - Filter filter = getFilterBuilder().attribute(Metacard.ID).is().like().text("*"); + Filter filter = getFilterBuilder().attribute(Metacard.ANY_TEXT).is().like().text("*"); SourceResponse sourceResponse = provider.query(new QueryRequestImpl(new QueryImpl(filter))); assertEquals("Failed to find all records.", 3, sourceResponse.getResults().size()); diff --git a/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/provider/SolrProviderTestUtil.java b/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/provider/SolrProviderTestUtil.java index 01179e24ac94..d44dbb23aed3 100644 --- a/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/provider/SolrProviderTestUtil.java +++ b/catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/provider/SolrProviderTestUtil.java @@ -81,7 +81,7 @@ public static void deleteAll(int methodNameIndex, SolrCatalogProviderImpl provid QueryImpl query; SourceResponse sourceResponse; - query = new QueryImpl(filterBuilder.attribute(Metacard.ID).is().like().text("*")); + query = new QueryImpl(filterBuilder.attribute(Metacard.ANY_TEXT).is().like().text("*")); query.setPageSize(ALL_RESULTS); sourceResponse = provider.query(new QueryRequestImpl(query)); @@ -180,15 +180,15 @@ public static Set numericalDescriptors( descriptors.add( new AttributeDescriptorImpl(Metacard.ID, true, true, true, false, BasicTypes.STRING_TYPE)); descriptors.add( - new AttributeDescriptorImpl(doubleField, true, true, true, false, BasicTypes.DOUBLE_TYPE)); + new AttributeDescriptorImpl(doubleField, true, true, false, false, BasicTypes.DOUBLE_TYPE)); descriptors.add( - new AttributeDescriptorImpl(floatField, true, true, true, false, BasicTypes.FLOAT_TYPE)); + new AttributeDescriptorImpl(floatField, true, true, false, false, BasicTypes.FLOAT_TYPE)); descriptors.add( - new AttributeDescriptorImpl(intField, true, true, true, false, BasicTypes.INTEGER_TYPE)); + new AttributeDescriptorImpl(intField, true, true, false, false, BasicTypes.INTEGER_TYPE)); descriptors.add( - new AttributeDescriptorImpl(longField, true, true, true, false, BasicTypes.LONG_TYPE)); + new AttributeDescriptorImpl(longField, true, true, false, false, BasicTypes.LONG_TYPE)); descriptors.add( - new AttributeDescriptorImpl(shortField, true, true, true, false, BasicTypes.SHORT_TYPE)); + new AttributeDescriptorImpl(shortField, true, true, false, false, BasicTypes.SHORT_TYPE)); return descriptors; } diff --git a/platform/solr/solr-query/src/main/java/org/codice/solr/query/SchemaField.java b/platform/solr/solr-query/src/main/java/org/codice/solr/query/SchemaField.java index b2821e7c37c0..fd5172592baf 100644 --- a/platform/solr/solr-query/src/main/java/org/codice/solr/query/SchemaField.java +++ b/platform/solr/solr-query/src/main/java/org/codice/solr/query/SchemaField.java @@ -19,7 +19,7 @@ public class SchemaField { private String name; - private String suffix; + private String specialSuffix; private String type; @@ -36,12 +36,12 @@ public void setName(String name) { this.name = name; } - public String getSuffix() { - return suffix; + public String getSpecialSuffix() { + return specialSuffix; } - public void setSuffix(String suffix) { - this.suffix = suffix; + public void setSpecialSuffix(String specialSuffix) { + this.specialSuffix = specialSuffix; } public String getType() { diff --git a/platform/solr/solr-query/src/main/java/org/codice/solr/query/SchemaFieldResolver.java b/platform/solr/solr-query/src/main/java/org/codice/solr/query/SchemaFieldResolver.java index 22de5ceeca44..6944494c57f6 100644 --- a/platform/solr/solr-query/src/main/java/org/codice/solr/query/SchemaFieldResolver.java +++ b/platform/solr/solr-query/src/main/java/org/codice/solr/query/SchemaFieldResolver.java @@ -132,7 +132,7 @@ public String getFieldSuffix(AttributeFormat attributeFormat) { return FORMAT_TO_SUFFIX_MAP.get(attributeFormat); } - public SchemaField getSchemaField(String propertyName, boolean isSearchedAsExactValue) { + public SchemaField getSchemaField(String propertyName) { SchemaField schemaField = null; LukeRequest luke = new LukeRequest(); LukeResponse rsp; @@ -152,13 +152,13 @@ public SchemaField getSchemaField(String propertyName, boolean isSearchedAsExact String fieldType = entry.getValue().getType(); int index = StringUtils.lastIndexOfAny(entry.getKey(), FORMAT_SUFFIXES); String suffix = entry.getKey().substring(index); - if (!isSearchedAsExactValue) { - suffix = getSpecialIndexSuffix(suffix); - fieldType += suffix; - } - LOGGER.debug("field {} has type {}", entry.getKey(), fieldType); schemaField = new SchemaField(entry.getKey(), fieldType); - schemaField.setSuffix(suffix); + String specialSuffix = getSpecialIndexSuffix(suffix); + // Only add special suffix if the resulting field actually exists in the schema + String potentialFieldName = propertyName + suffix + specialSuffix; + if (fieldsInfo.containsKey(potentialFieldName)) { + schemaField.setSpecialSuffix(specialSuffix); + } return schemaField; } } @@ -185,6 +185,6 @@ private String getSpecialIndexSuffix(String suffix) { return SchemaFieldResolver.TEXT_PATH; } - return ""; + return null; } } diff --git a/platform/solr/solr-query/src/main/java/org/codice/solr/query/SolrQueryFilterVisitor.java b/platform/solr/solr-query/src/main/java/org/codice/solr/query/SolrQueryFilterVisitor.java index 89ea30c45984..52f5b991bb6f 100644 --- a/platform/solr/solr-query/src/main/java/org/codice/solr/query/SolrQueryFilterVisitor.java +++ b/platform/solr/solr-query/src/main/java/org/codice/solr/query/SolrQueryFilterVisitor.java @@ -199,8 +199,6 @@ public Object visit(PropertyIsLike filter, Object data) { String pattern = normalizePattern(filter.getLiteral(), wildcard, singleChar, escapeChar); - String mappedPropertyName = getMappedPropertyName(propertyName); - String searchPhrase = escapeSpecialCharacters(pattern); boolean isAnyText = Metacard.ANY_TEXT.equals(propertyName); @@ -208,8 +206,15 @@ public Object visit(PropertyIsLike filter, Object data) { if (isAnyText && SOLR_WILDCARD_CHAR.equals(searchPhrase)) { return new SolrQuery("*:*"); } + String solrPropertyName; + SchemaField schemaField = getSchemaField(propertyName); + if (schemaField != null && schemaField.getSpecialSuffix() != null) { + solrPropertyName = schemaField.getName() + schemaField.getSpecialSuffix(); + } else { + solrPropertyName = getMappedPropertyName(propertyName, AttributeFormat.STRING, true); + } - return new SolrQuery(mappedPropertyName + SchemaFieldResolver.TOKENIZED + ":" + searchPhrase); + return new SolrQuery(solrPropertyName + ":" + searchPhrase); } @SuppressWarnings("java:S127") @@ -262,18 +267,7 @@ String getMappedPropertyName(String propertyName) { // will have the suffix and the variations on the property name, e.g., for propertyName="user" // fieldsInfo will have keys for "user_txt", "user_txt_tokenized", and // "user_txt_tokenized_has_case" - SchemaField schemaField; - String cacheKey = solrCoreName + "." + propertyName; - if (schemaFieldsCache.containsKey(cacheKey)) { - LOGGER.debug("Getting SchemaField for propertyName {} from cache", propertyName); - schemaField = schemaFieldsCache.get(cacheKey); - } else { - LOGGER.debug("Using SchemaFieldResolver for propertyName {}", propertyName); - schemaField = schemaFieldResolver.getSchemaField(propertyName, true); - if (schemaField != null) { - schemaFieldsCache.put(cacheKey, schemaField); - } - } + SchemaField schemaField = getSchemaField(propertyName); if (schemaField != null) { mappedPropertyName = schemaField.getName(); @@ -291,6 +285,22 @@ String getMappedPropertyName(String propertyName) { return mappedPropertyName; } + SchemaField getSchemaField(String propertyName) { + SchemaField schemaField; + String cacheKey = solrCoreName + "." + propertyName; + if (schemaFieldsCache.containsKey(cacheKey)) { + LOGGER.debug("Getting SchemaField for propertyName {} from cache", propertyName); + schemaField = schemaFieldsCache.get(cacheKey); + } else { + LOGGER.debug("Using SchemaFieldResolver for propertyName {}", propertyName); + schemaField = schemaFieldResolver.getSchemaField(propertyName); + if (schemaField != null) { + schemaFieldsCache.put(cacheKey, schemaField); + } + } + return schemaField; + } + private String getMappedPropertyName( String propertyName, AttributeFormat format, boolean isSearchedAsExactString) { String specialField = FIELD_MAP.get(propertyName); diff --git a/platform/solr/solr-query/src/test/java/org/codice/solr/query/SolrQueryFilterVisitorTest.java b/platform/solr/solr-query/src/test/java/org/codice/solr/query/SolrQueryFilterVisitorTest.java index 26bfc12d3b14..3c59213b33b4 100644 --- a/platform/solr/solr-query/src/test/java/org/codice/solr/query/SolrQueryFilterVisitorTest.java +++ b/platform/solr/solr-query/src/test/java/org/codice/solr/query/SolrQueryFilterVisitorTest.java @@ -129,15 +129,15 @@ public void testInSingleParam() throws Exception { public void testUnsupportedQuery() throws Exception { Filter filter = ECQL.toFilter("property LIKE 'val*'"); SolrQuery solrQuery = (SolrQuery) filter.accept(solrVisitor, null); - assertThat(solrQuery.getQuery().trim(), equalTo("property_txt_tokenized:val*")); + assertThat(solrQuery.getQuery().trim(), equalTo("property_txt:val*")); } @Test public void testGetMappedPropertyNameCache() { SchemaFieldResolver mockResolver = mock(SchemaFieldResolver.class); SchemaField schema = new SchemaField("testField_int", "tint"); - schema.setSuffix("_int"); - when(mockResolver.getSchemaField("testField", true)).thenReturn(schema); + schema.setSpecialSuffix("_int"); + when(mockResolver.getSchemaField("testField")).thenReturn(schema); solrVisitor = new SolrQueryFilterVisitor("alerts", mockResolver); String propertyName = solrVisitor.getMappedPropertyName("testField"); @@ -145,7 +145,7 @@ public void testGetMappedPropertyNameCache() { propertyName = solrVisitor.getMappedPropertyName("testField"); assertThat(propertyName, is("testField_int")); - verify(mockResolver, times(1)).getSchemaField("testField", true); + verify(mockResolver, times(1)).getSchemaField("testField"); } @Test @@ -153,7 +153,7 @@ public void testGetMappedPropertyNameNullNotCached() { SchemaFieldResolver mockResolver = mock(SchemaFieldResolver.class); // returning null simulates no entry in SOLR to query schema for "testField" - when(mockResolver.getSchemaField("testField2", true)).thenReturn(null); + when(mockResolver.getSchemaField("testField2")).thenReturn(null); when(mockResolver.getFieldSuffix(AttributeFormat.STRING)).thenReturn("_txt"); solrVisitor = new SolrQueryFilterVisitor("alerts", mockResolver); @@ -162,12 +162,31 @@ public void testGetMappedPropertyNameNullNotCached() { // simulates an entry in SOLR to query schema for "testField" SchemaField schema = new SchemaField("testField2_int", "tint"); - schema.setSuffix("_int"); + schema.setSpecialSuffix("_int"); - when(mockResolver.getSchemaField("testField2", true)).thenReturn(schema); + when(mockResolver.getSchemaField("testField2")).thenReturn(schema); propertyName = solrVisitor.getMappedPropertyName("testField2"); assertThat(propertyName, is("testField2_int")); - verify(mockResolver, times(2)).getSchemaField("testField2", true); + verify(mockResolver, times(2)).getSchemaField("testField2"); + } + + @Test + public void testGetMappedPropertyNameWithTokenizedSpecialSuffix() { + SchemaFieldResolver mockResolver = mock(SchemaFieldResolver.class); + + // Simulates a field that exists with a tokenized special index + // e.g., metadata_txt exists and metadata_txt_tokenized also exists + SchemaField schema = new SchemaField("metadata_txt", "string"); + schema.setSpecialSuffix(SchemaFieldResolver.TOKENIZED); + when(mockResolver.getSchemaField("metadata")).thenReturn(schema); + when(mockResolver.getFieldSuffix(AttributeFormat.STRING)).thenReturn("_txt"); + solrVisitor = new SolrQueryFilterVisitor("alerts", mockResolver); + + String propertyName = solrVisitor.getMappedPropertyName("metadata"); + // Should return the full tokenized field name + assertThat(propertyName, is("metadata_txt_tokenized")); + + verify(mockResolver, times(1)).getSchemaField("metadata"); } } From 41123f6023c71e3be7a445d1079ce9ef1b2a85be Mon Sep 17 00:00:00 2001 From: "chris.lockard" Date: Thu, 4 Jun 2026 11:57:56 -0700 Subject: [PATCH 2/2] Fix tests --- .../query/SolrQueryFilterVisitorTest.java | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/platform/solr/solr-query/src/test/java/org/codice/solr/query/SolrQueryFilterVisitorTest.java b/platform/solr/solr-query/src/test/java/org/codice/solr/query/SolrQueryFilterVisitorTest.java index 3c59213b33b4..d8fa7bc4f87f 100644 --- a/platform/solr/solr-query/src/test/java/org/codice/solr/query/SolrQueryFilterVisitorTest.java +++ b/platform/solr/solr-query/src/test/java/org/codice/solr/query/SolrQueryFilterVisitorTest.java @@ -29,6 +29,7 @@ import org.apache.solr.common.util.NamedList; import org.geotools.api.filter.Filter; import org.geotools.api.filter.Or; +import org.geotools.filter.text.cql2.CQLException; import org.geotools.filter.text.ecql.ECQL; import org.junit.Before; import org.junit.Test; @@ -172,21 +173,38 @@ public void testGetMappedPropertyNameNullNotCached() { } @Test - public void testGetMappedPropertyNameWithTokenizedSpecialSuffix() { + public void testGetMappedPropertyName() { SchemaFieldResolver mockResolver = mock(SchemaFieldResolver.class); - - // Simulates a field that exists with a tokenized special index - // e.g., metadata_txt exists and metadata_txt_tokenized also exists SchemaField schema = new SchemaField("metadata_txt", "string"); schema.setSpecialSuffix(SchemaFieldResolver.TOKENIZED); when(mockResolver.getSchemaField("metadata")).thenReturn(schema); - when(mockResolver.getFieldSuffix(AttributeFormat.STRING)).thenReturn("_txt"); - solrVisitor = new SolrQueryFilterVisitor("alerts", mockResolver); - + solrVisitor = new SolrQueryFilterVisitor("alert", mockResolver); String propertyName = solrVisitor.getMappedPropertyName("metadata"); - // Should return the full tokenized field name - assertThat(propertyName, is("metadata_txt_tokenized")); - + assertThat(propertyName, is("metadata_txt")); verify(mockResolver, times(1)).getSchemaField("metadata"); } + + @Test + public void testPropertyIsLikeWithSpecialSuffix() throws CQLException { + SchemaFieldResolver mockResolver = mock(SchemaFieldResolver.class); + SchemaField schema = new SchemaField("metadata2_txt", "string"); + schema.setSpecialSuffix(SchemaFieldResolver.TOKENIZED); + when(mockResolver.getSchemaField("metadata2")).thenReturn(schema); + solrVisitor = new SolrQueryFilterVisitor("alert", mockResolver); + Filter filter = ECQL.toFilter("metadata2 LIKE 'val*'"); + SolrQuery solrQuery = (SolrQuery) filter.accept(solrVisitor, null); + assertThat(solrQuery.getQuery().trim(), equalTo("metadata2_txt_tokenized:val*")); + } + + @Test + public void testPropertyIsLikeWithoutSpecialSuffix() throws CQLException { + SchemaFieldResolver mockResolver = mock(SchemaFieldResolver.class); + SchemaField schema = new SchemaField("metadata3_txt", "string"); + when(mockResolver.getSchemaField("metadata3")).thenReturn(schema); + when(mockResolver.getFieldSuffix(AttributeFormat.STRING)).thenReturn("_txt"); + solrVisitor = new SolrQueryFilterVisitor("alert", mockResolver); + Filter filter = ECQL.toFilter("metadata3 LIKE 'val*'"); + SolrQuery solrQuery = (SolrQuery) filter.accept(solrVisitor, null); + assertThat(solrQuery.getQuery().trim(), equalTo("metadata3_txt:val*")); + } }