diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java index 8830a27c8550..38a728e912c5 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java @@ -3010,6 +3010,64 @@ void get_assetsCountsWithNoParent(TestNamespace ns) { assertNotNull(counts); } + @Test + void get_assetsCountsPaginationSlicesAndPreservesTotal(TestNamespace ns) throws Exception { + OpenMetadataClient client = SdkClients.adminClient(); + + CreateGlossary createGlossary = + new CreateGlossary() + .withName(ns.prefix("asset_count_pagination_glossary")) + .withDescription("Glossary for asset count pagination test"); + Glossary glossary = client.glossaries().create(createGlossary); + + int termCount = 5; + java.util.List createdTermFqns = new java.util.ArrayList<>(); + for (int i = 0; i < termCount; i++) { + CreateGlossaryTerm req = + new CreateGlossaryTerm() + .withName(ns.prefix("count_pagination_term_" + i)) + .withGlossary(glossary.getFullyQualifiedName()) + .withDescription("Pagination term " + i); + createdTermFqns.add(createEntity(req).getFullyQualifiedName()); + } + + String fullCounts = getAssetCounts(client, glossary.getFullyQualifiedName()); + ObjectMapper mapper = new ObjectMapper(); + JsonNode unpaged = mapper.readTree(fullCounts); + for (String fqn : createdTermFqns) { + assertTrue(unpaged.has(fqn), "Unpaged response should contain term " + fqn); + } + + String firstPageBody = getAssetCountsWithPaging(client, glossary.getFullyQualifiedName(), 2, 0); + JsonNode firstPage = mapper.readTree(firstPageBody); + assertEquals( + 2, + firstPage.size(), + "First page should contain at most `limit` glossary terms when limit=2"); + + String secondPageBody = + getAssetCountsWithPaging(client, glossary.getFullyQualifiedName(), 2, 2); + JsonNode secondPage = mapper.readTree(secondPageBody); + assertTrue( + secondPage.size() <= 2, + "Second page should contain at most `limit` glossary terms when limit=2"); + + java.util.Set firstPageKeys = new java.util.HashSet<>(); + firstPage.fieldNames().forEachRemaining(firstPageKeys::add); + secondPage + .fieldNames() + .forEachRemaining( + key -> + assertFalse( + firstPageKeys.contains(key), + "Pages should not overlap: " + key + " appeared on both pages")); + + String beyondEndBody = + getAssetCountsWithPaging(client, glossary.getFullyQualifiedName(), 5, 10000); + JsonNode beyondEnd = mapper.readTree(beyondEndBody); + assertEquals(0, beyondEnd.size(), "Offset past the end should return an empty asset-count map"); + } + @Test void get_termAssetsById(TestNamespace ns) { OpenMetadataClient client = SdkClients.adminClient(); @@ -3176,6 +3234,21 @@ private String getAssetCounts(OpenMetadataClient client, String parent) { HttpMethod.GET, "/v1/glossaryTerms/assets/counts", null, optionsBuilder.build()); } + private String getAssetCountsWithPaging( + OpenMetadataClient client, String parent, int limit, int offset) { + RequestOptions.Builder optionsBuilder = + RequestOptions.builder() + .queryParam("limit", String.valueOf(limit)) + .queryParam("offset", String.valueOf(offset)); + if (parent != null) { + optionsBuilder.queryParam("parent", parent); + } + return client + .getHttpClient() + .executeForString( + HttpMethod.GET, "/v1/glossaryTerms/assets/counts", null, optionsBuilder.build()); + } + private String getTermAssetsById(OpenMetadataClient client, String id) { RequestOptions.Builder optionsBuilder = RequestOptions.builder(); optionsBuilder.queryParam("limit", "10"); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index 62c088b7f5b5..4ca24acf2c5a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -55,6 +55,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -116,10 +117,12 @@ import org.openmetadata.service.resources.glossary.GlossaryTermResource; import org.openmetadata.service.resources.settings.SettingsCache; import org.openmetadata.service.search.DefaultInheritedFieldEntitySearch; +import org.openmetadata.service.search.EntityBuilderConstant; import org.openmetadata.service.search.InheritedFieldEntitySearch; import org.openmetadata.service.search.InheritedFieldEntitySearch.InheritedFieldQuery; import org.openmetadata.service.search.InheritedFieldEntitySearch.InheritedFieldResult; import org.openmetadata.service.search.PropagationDescriptor; +import org.openmetadata.service.search.QueryFilterBuilder; import org.openmetadata.service.security.AuthorizationException; import org.openmetadata.service.security.policyevaluator.PolicyConditionUpdater; import org.openmetadata.service.util.EntityUtil; @@ -191,43 +194,66 @@ public ResultList getGlossaryTermAssetsByName( } public Map getAllGlossaryTermsWithAssetsCount(String parent) { + return getGlossaryTermsAssetsCount(parent, null, null).counts(); + } + + public GlossaryTermAssetCountResult getGlossaryTermsAssetsCount( + String parent, Integer limit, Integer offset) { if (inheritedFieldEntitySearch == null) { LOG.warn("Search unavailable for glossary term asset counts"); - return new HashMap<>(); + return new GlossaryTermAssetCountResult(new LinkedHashMap<>(), 0); } - List fqns; - if (parent != null && !parent.isEmpty()) { - fqns = - daoCollection.glossaryTermDAO().getNestedTerms(parent).stream() - .map(json -> JsonUtils.readTree(json).path("fullyQualifiedName").asText()) - .collect(Collectors.toList()); - } else { - fqns = - listAll(getFields("fullyQualifiedName"), new ListFilter(null)).stream() - .map(GlossaryTerm::getFullyQualifiedName) - .collect(Collectors.toList()); - } + List orderedFqns = listGlossaryTermFqns(parent); + int total = orderedFqns.size(); + List pageFqns = sliceFqns(orderedFqns, limit, offset); - Map glossaryTermAssetCounts = new HashMap<>(); + if (pageFqns.isEmpty()) { + return new GlossaryTermAssetCountResult(new LinkedHashMap<>(), total); + } - for (String fqn : fqns) { - InheritedFieldQuery query = InheritedFieldQuery.forGlossaryTerm(fqn, 0, 0); + Map aggregatedCounts = fetchAssetCountsByGlossaryTermFqn(); + LinkedHashMap pageCounts = new LinkedHashMap<>(); + for (String fqn : pageFqns) { + pageCounts.put(fqn, aggregatedCounts.getOrDefault(fqn, 0)); + } + return new GlossaryTermAssetCountResult(pageCounts, total); + } - Integer count = - inheritedFieldEntitySearch.getCountForField( - query, - () -> { - LOG.warn("Search fallback for glossary term {} asset count. Returning 0.", fqn); - return 0; - }); + private List listGlossaryTermFqns(String parent) { + if (parent != null && !parent.isEmpty()) { + return daoCollection.glossaryTermDAO().getNestedTerms(parent).stream() + .map(json -> JsonUtils.readTree(json).path("fullyQualifiedName").asText()) + .sorted() + .collect(Collectors.toList()); + } + return listAll(getFields("fullyQualifiedName"), new ListFilter(null)).stream() + .map(GlossaryTerm::getFullyQualifiedName) + .sorted() + .collect(Collectors.toList()); + } - glossaryTermAssetCounts.put(fqn, count); + private static List sliceFqns(List fqns, Integer limit, Integer offset) { + if (limit == null && offset == null) { + return fqns; } + int total = fqns.size(); + int from = offset == null ? 0 : Math.max(0, offset); + if (from >= total) { + return Collections.emptyList(); + } + int to = limit == null ? total : Math.min(total, from + Math.max(0, limit)); + return fqns.subList(from, to); + } - return glossaryTermAssetCounts; + private Map fetchAssetCountsByGlossaryTermFqn() { + String queryFilter = QueryFilterBuilder.buildGenericAssetsCountFilter(TAGS_FQN, true); + return inheritedFieldEntitySearch.getAggregatedCountsByField( + TAGS_FQN, queryFilter, EntityBuilderConstant.MAX_AGGREGATE_SIZE); } + public record GlossaryTermAssetCountResult(LinkedHashMap counts, int total) {} + public Map getRelationTypeUsageCounts() { Map usageCounts = new HashMap<>(); List counts = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java index e7e2eef75782..5f5ca9436353 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java @@ -107,6 +107,7 @@ public class GlossaryTermResource extends EntityResource result = repository.getAllGlossaryTermsWithAssetsCount(parent); - return Response.ok(result).build(); + GlossaryTermRepository.GlossaryTermAssetCountResult result = + repository.getGlossaryTermsAssetsCount(parent, limit, offset); + return Response.ok(result.counts()).header(TOTAL_COUNT_HEADER, result.total()).build(); } @GET diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/glossaryAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/glossaryAPI.ts index 4b1bc0c1ffb3..c2d30c614dd8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/glossaryAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/glossaryAPI.ts @@ -322,17 +322,49 @@ export const getGlossaryTermAssets = async ( return response.data; }; +export type GlossaryTermAssetCountsParams = { + parent?: string; + limit?: number; + offset?: number; +}; + +export type GlossaryTermAssetCountsResponse = { + data: Record; + total: number; +}; + export const getGlossaryTermsAssetCounts = async ( - parent?: string + parentOrParams?: string | GlossaryTermAssetCountsParams ): Promise> => { + const params: GlossaryTermAssetCountsParams = + typeof parentOrParams === 'string' + ? { parent: parentOrParams } + : parentOrParams ?? {}; const response = await APIClient.get>( '/glossaryTerms/assets/counts', - { params: parent ? { parent } : undefined } + { params } ); return response.data; }; +export const getGlossaryTermsAssetCountsPage = async ( + params: GlossaryTermAssetCountsParams +): Promise => { + const response = await APIClient.get>( + '/glossaryTerms/assets/counts', + { params } + ); + const totalHeader = + response.headers?.['x-total-count'] ?? response.headers?.['X-Total-Count']; + const parsedTotal = totalHeader ? Number(totalHeader) : NaN; + const total = Number.isFinite(parsedTotal) + ? parsedTotal + : Object.keys(response.data ?? {}).length; + + return { data: response.data ?? {}, total }; +}; + export const searchGlossaryTerms = async (search: string, page = 1) => { const apiUrl = `/search/query?q=${search ?? ''}`;