From 1355a3f393aa302c4cd3d989ea5c2545915b5bdb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 27 Feb 2026 10:16:57 +0000 Subject: [PATCH 01/11] Initial plan From 186616211b40f0c33835545019ee624581edadf9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 27 Feb 2026 10:32:14 +0000 Subject: [PATCH 02/11] Refactor ExtraMetadata: migrate to SparseFields and remove ExtraMetadata model Co-authored-by: etj <717359+etj@users.noreply.github.com> --- geonode/api/resourcebase_api.py | 8 -- geonode/api/tests.py | 32 -------- geonode/base/api/serializers.py | 20 ----- geonode/base/api/tests.py | 72 ----------------- geonode/base/api/views.py | 81 +------------------ ...8_migrate_extrametadata_to_sparsefields.py | 52 ++++++++++++ geonode/base/models.py | 11 +-- geonode/base/utils.py | 31 ------- geonode/documents/api/tests.py | 11 --- geonode/geoapps/api/tests.py | 11 --- geonode/layers/api/tests.py | 11 --- geonode/maps/api/tests.py | 11 --- geonode/resource/manager.py | 2 - geonode/resource/utils.py | 8 -- geonode/settings.py | 34 -------- 15 files changed, 54 insertions(+), 341 deletions(-) create mode 100644 geonode/base/migrations/0098_migrate_extrametadata_to_sparsefields.py diff --git a/geonode/api/resourcebase_api.py b/geonode/api/resourcebase_api.py index cf98190ac8e..e3a94505aec 100644 --- a/geonode/api/resourcebase_api.py +++ b/geonode/api/resourcebase_api.py @@ -142,10 +142,6 @@ def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): if "app_type__in" in filters: orm_filters.update({"resource_type": filters["app_type__in"].lower()}) - _metadata = {f"metadata__{_k}": _v for _k, _v in filters.items() if _k.startswith("metadata__")} - if _metadata: - orm_filters.update({"metadata_filters": _metadata}) - if "extent" in filters: orm_filters.update({"extent": filters["extent"]}) orm_filters["f_method"] = filters["f_method"] if "f_method" in filters else "and" @@ -165,7 +161,6 @@ def apply_filters(self, request, applicable_filters): keywords = applicable_filters.pop("keywords__slug__in", None) metadata_only = applicable_filters.pop("metadata_only", False) filtering_method = applicable_filters.pop("f_method", "and") - metadata_filters = applicable_filters.pop("metadata_filters", None) if filtering_method == "or": filters = Q() for f in applicable_filters.items(): @@ -207,9 +202,6 @@ def apply_filters(self, request, applicable_filters): if keywords: filtered = self.filter_h_keywords(filtered, keywords) - if metadata_filters: - filtered = filtered.filter(**metadata_filters) - # return filtered return get_visible_resources( filtered, diff --git a/geonode/api/tests.py b/geonode/api/tests.py index 71559cbe835..9cafd084939 100644 --- a/geonode/api/tests.py +++ b/geonode/api/tests.py @@ -37,7 +37,6 @@ from geonode.layers.models import Dataset from geonode.documents.models import Document from geonode.base.models import ( - ExtraMetadata, Thesaurus, ThesaurusLabel, ThesaurusKeyword, @@ -534,37 +533,6 @@ def test_category_filters(self): self.assertValidJSONResponse(resp) self.assertEqual(len(self.deserialize(resp)["objects"]), 5) - def test_metadata_filters(self): - """Test category filtering""" - _r = Dataset.objects.first() - _m = ExtraMetadata.objects.create( - resource=_r, - metadata={ - "name": "metadata-updated", - "slug": "metadata-slug-updated", - "help_text": "this is the help text-updated", - "field_type": "str-updated", - "value": "my value-updated", - "category": "category", - }, - ) - - list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) - _r.metadata.add(_m) - # check we get the correct layers number returnered filtering on one - # and then two different categories - filter_url = f"{list_url}?metadata__category=category" - - resp = self.api_client.get(filter_url) - self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)["objects"]), 1) - - filter_url = f"{list_url}?metadata__category=not-existing-category" - - resp = self.api_client.get(filter_url) - self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)["objects"]), 0) - def test_tag_filters(self): """Test keywords filtering""" list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) diff --git a/geonode/base/api/serializers.py b/geonode/base/api/serializers.py index fa1add81264..522164c70cc 100644 --- a/geonode/base/api/serializers.py +++ b/geonode/base/api/serializers.py @@ -57,7 +57,6 @@ SpatialRepresentationType, ThesaurusKeyword, ThesaurusKeywordLabel, - ExtraMetadata, LinkedResource, ) from geonode.documents.models import Document @@ -282,23 +281,6 @@ def get_attribute(self, instance): return build_absolute_uri(instance.detail_url) -class ExtraMetadataSerializer(DynamicModelSerializer): - class Meta: - model = ExtraMetadata - name = "ExtraMetadata" - fields = ("pk", "metadata") - - def to_representation(self, obj): - if isinstance(obj, QuerySet): - out = [] - for el in obj: - out.append({**{"id": el.id}, **el.metadata}) - return out - elif isinstance(obj, list): - return obj - return {**{"id": obj.id}, **obj.metadata} - - class ThumbnailUrlField(DynamicComputedField): def __init__(self, **kwargs): super().__init__(**kwargs) @@ -672,7 +654,6 @@ class ResourceBaseSerializer(MultiLangOutputMixin, DynamicModelSerializer): links = DynamicRelationField(LinksSerializer, source="id", read_only=True) # Deferred fields - metadata = ComplexDynamicRelationField(ExtraMetadataSerializer, many=True, deferred=True) data = DataBlobField(DataBlobSerializer, source="id", deferred=True, required=False) executions = DynamicRelationField( ResourceExecutionRequestSerializer, source="id", deferred=True, required=False, read_only=True @@ -754,7 +735,6 @@ class Meta: "sourcetype", "is_copyable", "blob", - "metadata", "executions", "linked_resources", "download_url", diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py index 2e4ab46f127..aa6ab6600a2 100644 --- a/geonode/base/api/tests.py +++ b/geonode/base/api/tests.py @@ -71,7 +71,6 @@ ResourceBase, TopicCategory, ThesaurusKeyword, - ExtraMetadata, RestrictionCodeType, License, Group, @@ -3084,77 +3083,6 @@ def test_metadata_uploaded_preserve_can_be_updated(self): self.assertTrue(response.json()["resource"]["metadata_uploaded_preserve"]) -class TestExtraMetadataBaseApi(GeoNodeBaseTestSupport): - def setUp(self): - self.layer = create_single_dataset("single_layer") - self.metadata = { - "filter_header": "Foo Filter header", - "field_name": "metadata-name", - "field_label": "this is the help text", - "field_value": "foo", - } - m = ExtraMetadata.objects.create(resource=self.layer, metadata=self.metadata) - self.layer.metadata.add(m) - self.mdata = ExtraMetadata.objects.first() - - def test_get_will_return_the_list_of_extra_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse("base-resources-extra-metadata", args=[self.layer.id]) - response = self.client.get(url, content_type="application/json") - self.assertTrue(200, response.status_code) - expected = [{**{"id": self.mdata.id}, **self.metadata}] - self.assertEqual(expected, response.json()) - - def test_put_will_update_the_whole_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse("base-resources-extra-metadata", args=[self.layer.id]) - input_metadata = { - "id": self.mdata.id, - "filter_header": "Foo Filter header", - "field_name": "metadata-updated", - "field_label": "this is the help text", - "field_value": "foo", - } - response = self.client.put(url, data=[input_metadata], content_type="application/json") - self.assertTrue(200, response.status_code) - self.assertEqual([input_metadata], response.json()) - - def test_post_will_add_new_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse("base-resources-extra-metadata", args=[self.layer.id]) - input_metadata = { - "filter_header": "Foo Filter header", - "field_name": "metadata-updated", - "field_label": "this is the help text", - "field_value": "foo", - } - response = self.client.post(url, data=[input_metadata], content_type="application/json") - self.assertTrue(201, response.status_code) - self.assertEqual(2, len(response.json())) - - def test_delete_will_delete_single_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse("base-resources-extra-metadata", args=[self.layer.id]) - response = self.client.delete(url, data=[self.mdata.id], content_type="application/json") - self.assertTrue(200, response.status_code) - self.assertEqual([], response.json()) - - def test_user_without_view_perms_cannot_see_the_endpoint(self): - from geonode.resource.manager import resource_manager - - self.client.login(username="bobby", password="bob") - resource_manager.remove_permissions(self.layer.uuid, instance=self.layer.get_self_resource()) - url = reverse("base-resources-extra-metadata", args=[self.layer.id]) - response = self.client.get(url, content_type="application/json") - self.assertTrue(401, response.status_code) - - perm_spec = {"users": {"bobby": ["view_resourcebase"]}, "groups": {}} - self.layer.set_permissions(perm_spec) - url = reverse("base-resources-extra-metadata", args=[self.layer.id]) - response = self.client.get(url, content_type="application/json") - self.assertTrue(200, response.status_code) - - class TestApiLinkedResources(GeoNodeBaseTestSupport): @classmethod def setUpClass(cls) -> None: diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index cda55692ddc..0f03ae69871 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -65,7 +65,6 @@ TopicCategory, ThesaurusKeyword, Configuration, - ExtraMetadata, LinkedResource, ) from geonode.base.api.filters import ( @@ -107,12 +106,11 @@ TopicCategorySerializer, RegionSerializer, ThesaurusKeywordSerializer, - ExtraMetadataSerializer, LinkedResourceSerializer, ) from geonode.people.api.serializers import UserSerializer from .pagination import GeoNodeApiPagination -from geonode.base.utils import validate_extra_metadata, patch_perms +from geonode.base.utils import patch_perms from geonode.assets.models import Asset from geonode.assets.utils import create_asset_and_link, unlink_asset from geonode.assets.handlers import asset_handler_registry @@ -1282,83 +1280,6 @@ def set_thumbnail(self, request, pk, *args, **kwargs): return Response({"thumbnail_url": resource.thumbnail_url}) return Response("Unable to set thumbnail", status=status.HTTP_400_BAD_REQUEST) - @extend_schema( - methods=["get", "put", "delete", "post"], description="Get/Update/Delete/Add extra metadata for resource" - ) - @action( - detail=True, - methods=["get", "put", "delete", "post"], - permission_classes=[IsOwnerOrAdmin, UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}})], - url_path=r"extra_metadata", # noqa - url_name="extra-metadata", - ) - def extra_metadata(self, request, pk, *args, **kwargs): - _obj = get_object_or_404(ResourceBase, pk=pk) - - if request.method == "GET": - # get list of available metadata - queryset = _obj.metadata.all() - _filters = [{f"metadata__{key}": value} for key, value in request.query_params.items()] - if _filters: - queryset = queryset.filter(**_filters[0]) - return Response(ExtraMetadataSerializer().to_representation(queryset)) - if not request.method == "DELETE": - try: - extra_metadata = validate_extra_metadata(request.data, _obj) - except Exception as e: - return Response(status=500, data=e.args[0]) - - if request.method == "PUT": - """ - update specific metadata. The ID of the metadata is required to perform the update - [ - { - "id": 1, - "name": "foo_name", - "slug": "foo_sug", - "help_text": "object", - "field_type": "int", - "value": "object", - "category": "object" - } - ] - """ - for _m in extra_metadata: - _id = _m.pop("id") - ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id=_id).update(metadata=_m) - logger.info("metadata updated for the selected resource") - _obj.refresh_from_db() - return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all())) - elif request.method == "DELETE": - # delete single metadata - """ - Expect a payload with the IDs of the metadata that should be deleted. Payload be like: - [4, 3] - """ - ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id__in=request.data).delete() - _obj.refresh_from_db() - return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all())) - elif request.method == "POST": - # add new metadata - """ - [ - { - "name": "foo_name", - "slug": "foo_sug", - "help_text": "object", - "field_type": "int", - "value": "object", - "category": "object" - } - ] - """ - for _m in extra_metadata: - new_m = ExtraMetadata.objects.create(resource=_obj, metadata=_m) - new_m.save() - _obj.metadata.add(new_m) - _obj.refresh_from_db() - return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all()), status=201) - @action( detail=True, methods=["get"], diff --git a/geonode/base/migrations/0098_migrate_extrametadata_to_sparsefields.py b/geonode/base/migrations/0098_migrate_extrametadata_to_sparsefields.py new file mode 100644 index 00000000000..234d4b9644c --- /dev/null +++ b/geonode/base/migrations/0098_migrate_extrametadata_to_sparsefields.py @@ -0,0 +1,52 @@ +import json +import logging + +from django.db import migrations + +logger = logging.getLogger(__name__) + + +def migrate_extrametadata_to_sparsefields(apps, schema_editor): + """ + Migrate ExtraMetadata entries to SparseField entries. + + Each ExtraMetadata object (with its JSON dict) is stored as a single + SparseField entry with name 'extra_metadata_' and value as JSON string. + Entries whose serialized JSON exceeds 1024 characters are skipped with a warning. + """ + ExtraMetadata = apps.get_model("base", "ExtraMetadata") + SparseField = apps.get_model("metadata", "SparseField") + + for extra_meta in ExtraMetadata.objects.select_related("resource").iterator(): + name = f"extra_metadata_{extra_meta.pk}" + value = json.dumps(extra_meta.metadata) + if len(value) > 1024: + logger.warning( + f"ExtraMetadata pk={extra_meta.pk} skipped during migration to SparseField: " + f"serialized value exceeds 1024 characters" + ) + continue + SparseField.objects.get_or_create( + resource=extra_meta.resource, + name=name, + defaults={"value": value}, + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("base", "0097_alter_link_asset"), + ("metadata", "0001_initial"), + ] + + operations = [ + migrations.RunPython(migrate_extrametadata_to_sparsefields, migrations.RunPython.noop), + migrations.RemoveField( + model_name="resourcebase", + name="metadata", + ), + migrations.DeleteModel( + name="ExtraMetadata", + ), + ] diff --git a/geonode/base/models.py b/geonode/base/models.py index 6ead4558734..3d4fc801d0e 100644 --- a/geonode/base/models.py +++ b/geonode/base/models.py @@ -679,9 +679,6 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): data_quality_statement_help_text = _( "general explanation of the data producer's knowledge about the lineage of a" " dataset" ) - extra_metadata_help_text = _( - 'Additional metadata, must be in format [ {"metadata_key": "metadata_value"}, {"metadata_key": "metadata_value"} ]' - ) # internal fields uuid = models.CharField(max_length=36, unique=True, default=uuid.uuid4) title = models.CharField(_("title"), max_length=255, help_text=_("name by which the cited resource is known")) @@ -887,10 +884,6 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): subtype = models.CharField(max_length=128, null=True, blank=True) - metadata = models.ManyToManyField( - "ExtraMetadata", verbose_name=_("Extra Metadata"), null=True, blank=True, help_text=extra_metadata_help_text - ) - objects = ResourceBaseManager() class Meta: @@ -2137,6 +2130,4 @@ class GroupGeoLimit(models.Model): wkt = models.TextField(db_column="wkt", blank=True) -class ExtraMetadata(models.Model): - resource = models.ForeignKey(ResourceBase, null=False, blank=False, on_delete=models.CASCADE) - metadata = JSONField(null=True, default=dict, blank=True) + diff --git a/geonode/base/utils.py b/geonode/base/utils.py index f5b543b5066..d421230341e 100644 --- a/geonode/base/utils.py +++ b/geonode/base/utils.py @@ -22,16 +22,12 @@ # Standard Modules import re -import json import logging -from schema import Schema from dateutil.parser import isoparse from datetime import datetime, timedelta # Django functionality -from django.conf import settings from django.contrib.auth import get_user_model -from django.core.exceptions import ValidationError # Geonode functionality from geonode.layers.models import Dataset @@ -177,33 +173,6 @@ def get_resource(resource_base): return resource_base.get_real_instance() -def validate_extra_metadata(data, instance): - if not data: - return data - - # starting validation of extra metadata passed via JSON - # if schema for metadata validation is not defined, an error is raised - resource_type = instance.polymorphic_ctype.model if instance.polymorphic_ctype else instance.class_name.lower() - extra_metadata_validation_schema = settings.EXTRA_METADATA_SCHEMA.get(resource_type, None) - if not extra_metadata_validation_schema: - raise ValidationError(f"EXTRA_METADATA_SCHEMA validation schema is not available for resource {resource_type}") - # starting json structure validation. The Field can contain multiple metadata - try: - if isinstance(data, str): - data = json.loads(data) - except Exception: - raise ValidationError("The value provided for the Extra metadata field is not a valid JSON") - - # looping on all the single metadata provided. If it doen't match the schema an error is raised - for _index, _metadata in enumerate(data): - try: - Schema(extra_metadata_validation_schema).validate(_metadata) - except Exception as e: - raise ValidationError(f"{e} at index {_index} for input json: {json.dumps(_metadata)}") - # conerted because in this case, we can store a well formated json instead of the user input - return data - - def remove_country_from_languagecode(language: str): """Remove country code (us) from language name (en-us) >>> remove_country_from_lanugecode("en-us") diff --git a/geonode/documents/api/tests.py b/geonode/documents/api/tests.py index 42f677a189e..c699d7221aa 100644 --- a/geonode/documents/api/tests.py +++ b/geonode/documents/api/tests.py @@ -83,17 +83,6 @@ def test_documents(self): # import json # logger.error(f"{json.dumps(layers_data)}") - def test_extra_metadata_included_with_param(self): - resource = Document.objects.first() - url = urljoin(f"{reverse('documents-list')}/", f"{resource.pk}") - data = {"include[]": "metadata"} - - response = self.client.get(url, format="json", data=data) - self.assertIsNotNone(response.data["document"].get("metadata")) - - response = self.client.get(url, format="json") - self.assertNotIn("metadata", response.data["document"]) - def test_creation_return_error_if_file_is_not_passed(self): """ If file_path is not available, should raise error diff --git a/geonode/geoapps/api/tests.py b/geonode/geoapps/api/tests.py index 7fb78d4160e..11a86393c03 100644 --- a/geonode/geoapps/api/tests.py +++ b/geonode/geoapps/api/tests.py @@ -114,17 +114,6 @@ def test_geoapp_listing_advertised(self): GeoApp.objects.update(advertised=True) - def test_extra_metadata_included_with_param(self): - _app = GeoApp.objects.first() - url = urljoin(f"{reverse('geoapps-list')}/", f"{_app.pk}") - data = {"include[]": "metadata"} - - response = self.client.get(url, format="json", data=data) - self.assertIsNotNone(response.data["geoapp"].get("metadata")) - - response = self.client.get(url, format="json") - self.assertNotIn("metadata", response.data["geoapp"]) - def test_geoapps_crud(self): """ Ensure we can create/update GeoApps. diff --git a/geonode/layers/api/tests.py b/geonode/layers/api/tests.py index a68e61c68ec..27c6992c3bf 100644 --- a/geonode/layers/api/tests.py +++ b/geonode/layers/api/tests.py @@ -209,17 +209,6 @@ def test_dataset_listing_advertised(self): Dataset.objects.update(advertised=True) - def test_extra_metadata_included_with_param(self): - _dataset = Dataset.objects.first() - url = urljoin(f"{reverse('datasets-list')}/", f"{_dataset.pk}") - data = {"include[]": "metadata"} - - response = self.client.get(url, format="json", data=data) - self.assertIsNotNone(response.data["dataset"].get("metadata")) - - response = self.client.get(url, format="json") - self.assertNotIn("metadata", response.data["dataset"]) - def test_get_dataset_related_maps_and_maplayers(self): dataset = Dataset.objects.first() assign_perm("base.view_resourcebase", get_anonymous_user(), dataset.get_self_resource()) diff --git a/geonode/maps/api/tests.py b/geonode/maps/api/tests.py index 24c154d5bf1..24781461654 100644 --- a/geonode/maps/api/tests.py +++ b/geonode/maps/api/tests.py @@ -129,17 +129,6 @@ def test_maps(self): self.assertEqual(response.data["map"]["maplayers"][0]["order"], 0) self.assertEqual(response.data["map"]["maplayers"][0]["opacity"], 1.0) - def test_extra_metadata_included_with_param(self): - resource = Map.objects.first() - url = urljoin(f"{reverse('maps-list')}/", f"{resource.pk}") - data = {"include[]": "metadata"} - - response = self.client.get(url, format="json", data=data) - self.assertIsNotNone(response.data["map"].get("metadata")) - - response = self.client.get(url, format="json") - self.assertNotIn("map", response.data["map"]) - def test_patch_map(self): """ Patch to maps// diff --git a/geonode/resource/manager.py b/geonode/resource/manager.py index 528f3d6628b..9d60f2579c4 100644 --- a/geonode/resource/manager.py +++ b/geonode/resource/manager.py @@ -397,7 +397,6 @@ def update( keywords: list = [], custom: dict = {}, notify: bool = True, - extra_metadata: list = [], *args, **kwargs, ) -> ResourceBase: @@ -439,7 +438,6 @@ def update( regions=regions, keywords=keywords, vals=vals, - extra_metadata=extra_metadata, ) ji = custom.get("jsoninstance", None) diff --git a/geonode/resource/utils.py b/geonode/resource/utils.py index 2e8edb6173a..546a49d9455 100644 --- a/geonode/resource/utils.py +++ b/geonode/resource/utils.py @@ -37,7 +37,6 @@ from ..base import enumerations from ..base.models import ( - ExtraMetadata, Link, License, ResourceBase, @@ -142,7 +141,6 @@ def update_resource( regions: list = [], keywords: list = [], vals: dict = {}, - extra_metadata: list = [], ): if xml_file: instance.metadata_xml = open(xml_file).read() @@ -277,12 +275,6 @@ def update_resource( # Refresh from DB instance.refresh_from_db() - if extra_metadata: - instance.metadata.all().delete() - for _m in extra_metadata: - new_m = ExtraMetadata.objects.create(resource=instance, metadata=_m) - instance.metadata.add(new_m) - return instance diff --git a/geonode/settings.py b/geonode/settings.py index 1150c4247d7..794a39e8cdc 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -25,7 +25,6 @@ import logging import subprocess import dj_database_url -from schema import Optional from urllib.parse import urlparse, urljoin # @@ -2069,39 +2068,6 @@ def get_geonode_catalogue_service(): DEFAULT_AUTO_FIELD = "django.db.models.AutoField" -""" -Default schema used to store extra and dynamic metadata for the resource -""" - -DEFAULT_EXTRA_METADATA_SCHEMA = { - Optional("id"): int, - "filter_header": object, - "field_name": object, - "field_label": object, - "field_value": object, -} - -""" -If present, will extend the available metadata schema used for store -new value for each resource. By default overrided the existing one. -The expected schema is the same as the default -""" -CUSTOM_METADATA_SCHEMA = os.getenv("CUSTOM_METADATA_SCHEMA ", {}) - -""" -Variable used to actually get the expected metadata schema for each resource_type. -In this way, each resource type can have a different metadata schema -""" -EXTRA_METADATA_SCHEMA = { - **{ - "map": os.getenv("MAP_EXTRA_METADATA_SCHEMA", DEFAULT_EXTRA_METADATA_SCHEMA), - "dataset": os.getenv("DATASET_EXTRA_METADATA_SCHEMA", DEFAULT_EXTRA_METADATA_SCHEMA), - "document": os.getenv("DOCUMENT_EXTRA_METADATA_SCHEMA", DEFAULT_EXTRA_METADATA_SCHEMA), - "geoapp": os.getenv("GEOAPP_EXTRA_METADATA_SCHEMA", DEFAULT_EXTRA_METADATA_SCHEMA), - }, - **CUSTOM_METADATA_SCHEMA, -} - """ List of modules that implement custom metadata storers that will be called when the metadata of a resource is saved """ From cfc914c27efc9c7745d695ce83f68536d40ffc7f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 19 Mar 2026 18:09:02 +0000 Subject: [PATCH 03/11] Fix flake8 errors: remove unused imports and trailing blank line Co-authored-by: etj <717359+etj@users.noreply.github.com> --- geonode/base/api/serializers.py | 2 +- geonode/base/api/views.py | 1 - geonode/base/models.py | 3 --- geonode/geoapps/api/tests.py | 2 -- 4 files changed, 1 insertion(+), 7 deletions(-) diff --git a/geonode/base/api/serializers.py b/geonode/base/api/serializers.py index 522164c70cc..812b7e98ac3 100644 --- a/geonode/base/api/serializers.py +++ b/geonode/base/api/serializers.py @@ -27,7 +27,7 @@ from django.contrib.auth.models import Group from django.forms.models import model_to_dict from django.contrib.auth import get_user_model -from django.db.models.query import QuerySet + from geonode.assets.utils import get_default_asset, is_asset_deletable from geonode.metadata.multilang.serializers import MultiLangOutputMixin from geonode.people import Roles diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index 0f03ae69871..1ad9fa915de 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -89,7 +89,6 @@ from geonode.resource.manager import resource_manager from .permissions import ( - IsOwnerOrAdmin, IsManagerEditOrAdmin, ResourceBasePermissionsFilter, UserHasPerms, diff --git a/geonode/base/models.py b/geonode/base/models.py index 3d4fc801d0e..c6b356d0352 100644 --- a/geonode/base/models.py +++ b/geonode/base/models.py @@ -2128,6 +2128,3 @@ class GroupGeoLimit(models.Model): group = models.ForeignKey(GroupProfile, null=False, blank=False, on_delete=models.CASCADE) resource = models.ForeignKey(ResourceBase, null=False, blank=False, on_delete=models.CASCADE) wkt = models.TextField(db_column="wkt", blank=True) - - - diff --git a/geonode/geoapps/api/tests.py b/geonode/geoapps/api/tests.py index 11a86393c03..a6198e47283 100644 --- a/geonode/geoapps/api/tests.py +++ b/geonode/geoapps/api/tests.py @@ -19,8 +19,6 @@ import json import logging from unittest.mock import MagicMock -from urllib.parse import urljoin - from django.contrib.auth import get_user_model from django.urls import reverse from django.test import override_settings From c1a9e810d53e46f79e4799aad89edcb0459d22ac Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 07:56:45 +0000 Subject: [PATCH 04/11] Remove 'metadata' field from DatasetSerializer.Meta.fields Agent-Logs-Url: https://github.com/GeoNode/geonode/sessions/d5187544-0996-458a-88b7-66628254cbf4 Co-authored-by: etj <717359+etj@users.noreply.github.com> --- geonode/layers/api/serializers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/geonode/layers/api/serializers.py b/geonode/layers/api/serializers.py index 5a95554c978..e004af66f52 100644 --- a/geonode/layers/api/serializers.py +++ b/geonode/layers/api/serializers.py @@ -171,7 +171,6 @@ class Meta: + ( "uuid", "name", - "metadata", "attribute_set", "charset", "is_mosaic", From 6a89c72fa347a3e8ff9d6abe2d0069a4ba594203 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 May 2026 15:43:23 +0000 Subject: [PATCH 05/11] Restore EXTRA_METADATA_SCHEMA=\{\} in settings.py for mapstore client compatibility Agent-Logs-Url: https://github.com/GeoNode/geonode/sessions/31247663-06ab-4d8e-99b8-13df81f898db Co-authored-by: etj <717359+etj@users.noreply.github.com> --- geonode/settings.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/geonode/settings.py b/geonode/settings.py index 8fc7ff81315..02ac444a5f0 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -2075,6 +2075,13 @@ def get_geonode_catalogue_service(): # 'geonode.resource.regions_storer.spatial_predicate_region_assignor', ] +""" +Kept for backward compatibility with geonode_mapstore_client and other packages +that may still reference this setting. ExtraMetadata has been removed in favour +of SparseFields; this empty dict prevents AttributeError on startup. +""" +EXTRA_METADATA_SCHEMA = {} + MULTILANG_FIELDS = ( # "title", # "abstract", From 3c4e8c9d56464acaa7acca6871bd8722afb5a37d Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 May 2026 18:00:10 +0200 Subject: [PATCH 06/11] Add deprecated backward-compatible ExtraMetadata API adapters (#14133) * Initial plan * Add deprecated backward-compatible ExtraMetadata API adapters Add isolated deprecated adapters that re-expose the old /extra_metadata/ endpoint and metadata serializer field using SparseField as storage. - New file: deprecated_extra_metadata.py (self-contained, easy to remove) - DeprecatedExtraMetadataMixin: restores GET/PUT/POST/DELETE endpoint - DeprecatedExtraMetadataField: restores metadata deferred field - All marked @deprecated(version="4.4.0") with deprecation warnings - Tests covering all CRUD operations and serializer field Agent-Logs-Url: https://github.com/GeoNode/geonode/sessions/39c9c964-1dcf-4194-8452-fac2b2f27292 Co-authored-by: etj <717359+etj@users.noreply.github.com> * Address code review: extract value limit constant, fix redundant branch Agent-Logs-Url: https://github.com/GeoNode/geonode/sessions/39c9c964-1dcf-4194-8452-fac2b2f27292 Co-authored-by: etj <717359+etj@users.noreply.github.com> * Restore deprecated EXTRA_METADATA_SCHEMA settings and address review comments - Restore EXTRA_METADATA_SCHEMA, DEFAULT_EXTRA_METADATA_SCHEMA, CUSTOM_METADATA_SCHEMA settings with deprecation markers (required by mapstore-client) - Restore 'from schema import Optional' import in settings.py - Fix docstring removal checklist (step 3 + add step 4 for settings) - Add logger.warning to DeprecatedExtraMetadataField.get_attribute() - Fix query param filtering to skip known non-filter params - Handle IntegrityError on concurrent POST via UUID fallback - Fix trailing slash in test URL helper Agent-Logs-Url: https://github.com/GeoNode/geonode/sessions/b6b2de77-7b42-4630-995a-d2c46d014e84 Co-authored-by: etj <717359+etj@users.noreply.github.com> * Fix trailing space in CUSTOM_METADATA_SCHEMA env var, extract skip params constant Agent-Logs-Url: https://github.com/GeoNode/geonode/sessions/b6b2de77-7b42-4630-995a-d2c46d014e84 Co-authored-by: etj <717359+etj@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: etj <717359+etj@users.noreply.github.com> --- geonode/base/api/deprecated_extra_metadata.py | 346 ++++++++++++++++++ geonode/base/api/serializers.py | 4 + geonode/base/api/tests.py | 149 ++++++++ geonode/base/api/views.py | 5 +- geonode/settings.py | 27 ++ 5 files changed, 530 insertions(+), 1 deletion(-) create mode 100644 geonode/base/api/deprecated_extra_metadata.py diff --git a/geonode/base/api/deprecated_extra_metadata.py b/geonode/base/api/deprecated_extra_metadata.py new file mode 100644 index 00000000000..e0d3ec942b8 --- /dev/null +++ b/geonode/base/api/deprecated_extra_metadata.py @@ -0,0 +1,346 @@ +# ######################################################################### +# +# Copyright (C) 2025 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# ######################################################################### +""" +Deprecated backward-compatible adapters for the ExtraMetadata API. + +These adapters re-expose the old ``/extra_metadata/`` endpoint and the +``metadata`` serializer field using :class:`SparseField` as the storage +backend. They are marked **deprecated** and should be removed after the +deprecation period. + +To remove these adapters: + 1. Delete this file. + 2. Remove the ``metadata`` field and import from ``serializers.py``. + 3. Remove ``DeprecatedExtraMetadataMixin`` and its import from ``views.py``. + 4. Remove deprecated settings from ``settings.py`` + (``DEFAULT_EXTRA_METADATA_SCHEMA``, ``CUSTOM_METADATA_SCHEMA``, + ``EXTRA_METADATA_SCHEMA``, and the ``from schema import Optional`` import). +""" + +import json +import logging +import uuid +import warnings + +from deprecated import deprecated +from rest_framework.decorators import action +from rest_framework.response import Response +from drf_spectacular.utils import extend_schema + +from dynamic_rest.fields.fields import DynamicComputedField + +from django.db import IntegrityError + +from geonode.base.models import ResourceBase +from geonode.metadata.models import SparseField + +logger = logging.getLogger(__name__) + +DEPRECATION_VERSION = "4.4.0" +DEPRECATION_REASON = ( + "The extra_metadata API is deprecated and will be removed in a future " + "version. Use the sparse fields API instead." +) +SPARSE_FIELD_PREFIX = "extra_metadata_" +# SparseField.value is CharField(max_length=1024); entries exceeding this +# limit cannot be stored and are silently skipped with a log warning. +SPARSE_FIELD_VALUE_MAX_LENGTH = 1024 + +# Query parameter names that should *not* be treated as legacy +# ``metadata__`` filters in the deprecated GET endpoint. +_NON_FILTER_QUERY_PARAMS = { + "api_preset", "page", "page_size", "format", "include[]", + "exclude[]", "sort[]", +} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _sparse_fields_for_resource(resource): + """Return SparseField entries that represent migrated ExtraMetadata.""" + return SparseField.objects.filter( + resource=resource, + name__startswith=SPARSE_FIELD_PREFIX, + ) + + +def _sparse_to_legacy(sparse_field): + """Convert a SparseField into the old ExtraMetadata representation. + + Returns ``{"id": , ...metadata_dict}`` so existing consumers see the + same shape they used to get. + """ + try: + metadata = json.loads(sparse_field.value) if sparse_field.value else {} + except (json.JSONDecodeError, TypeError): + metadata = {} + return {**{"id": sparse_field.pk}, **metadata} + + +def _next_sparse_name(resource): + """Generate the next available ``extra_metadata_`` name. + + Falls back to a UUID-based suffix if a name collision occurs (e.g. under + concurrent requests), keeping the ``extra_metadata_`` prefix intact. + The numeric part is still preferred for readability. + """ + existing = ( + SparseField.objects.filter( + resource=resource, + name__startswith=SPARSE_FIELD_PREFIX, + ) + .order_by("-name") + .values_list("name", flat=True) + ) + max_n = 0 + for name in existing: + suffix = name[len(SPARSE_FIELD_PREFIX):] + try: + max_n = max(max_n, int(suffix)) + except (ValueError, TypeError): + pass + candidate = f"{SPARSE_FIELD_PREFIX}{max_n + 1}" + # Guard against a race: if the candidate already exists, use a UUID suffix + if SparseField.objects.filter(resource=resource, name=candidate).exists(): + candidate = f"{SPARSE_FIELD_PREFIX}{uuid.uuid4().hex[:12]}" + return candidate + + +# --------------------------------------------------------------------------- +# Deprecated serializer field (for ``metadata`` on ResourceBaseSerializer) +# --------------------------------------------------------------------------- + + +class DeprecatedExtraMetadataField(DynamicComputedField): + """Deferred computed field that reconstructs the legacy ``metadata`` + representation from :class:`SparseField` entries. + + .. deprecated:: 4.4.0 + Use the sparse fields API instead. + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + @deprecated(version=DEPRECATION_VERSION, reason=DEPRECATION_REASON) + def get_attribute(self, instance): + warnings.warn(DEPRECATION_REASON, DeprecationWarning, stacklevel=2) + logger.warning(DEPRECATION_REASON) + try: + qs = _sparse_fields_for_resource(instance) + return [_sparse_to_legacy(sf) for sf in qs] + except Exception as e: + logger.exception(e) + return [] + + +# --------------------------------------------------------------------------- +# Deprecated extra_metadata view action mixin +# --------------------------------------------------------------------------- + + +class DeprecatedExtraMetadataMixin: + """Mixin that adds the deprecated ``extra_metadata`` action back to a + ``ViewSet``. + + Import this mixin and add it to your ViewSet's bases to restore the + old ``/{pk}/extra_metadata/`` endpoint backed by SparseFields. + + .. deprecated:: 4.4.0 + Use the sparse fields API instead. + """ + + @extend_schema( + methods=["get", "put", "delete", "post"], + description=( + "[DEPRECATED] Get/Update/Delete/Add extra metadata for a resource. " + "Use the sparse fields API instead." + ), + deprecated=True, + ) + @action( + detail=True, + methods=["get", "put", "delete", "post"], + url_path=r"extra_metadata", + url_name="extra-metadata", + ) + def extra_metadata(self, request, pk, *args, **kwargs): + """Deprecated endpoint – delegates to SparseField storage.""" + warnings.warn(DEPRECATION_REASON, DeprecationWarning, stacklevel=2) + logger.warning(DEPRECATION_REASON) + + resource = ResourceBase.objects.filter(pk=pk).first() + if resource is None: + return Response({"detail": "Not found."}, status=404) + + if request.method == "GET": + return self._extra_metadata_get(request, resource) + elif request.method == "POST": + return self._extra_metadata_post(request, resource) + elif request.method == "PUT": + return self._extra_metadata_put(request, resource) + elif request.method == "DELETE": + return self._extra_metadata_delete(request, resource) + + # -- GET ---------------------------------------------------------------- + + @staticmethod + def _extra_metadata_get(request, resource): + qs = _sparse_fields_for_resource(resource) + # Support the old query-param filtering (e.g. ?field_name=value) + for key, value in request.query_params.items(): + if key in _NON_FILTER_QUERY_PARAMS or key.startswith("filter{"): + continue + # Old API used metadata__=value JSONField lookups. + # We approximate this by filtering on the JSON string. + filtered = [] + for sf in qs: + try: + meta = json.loads(sf.value) if sf.value else {} + except (json.JSONDecodeError, TypeError): + continue + if str(meta.get(key)) == str(value): + filtered.append(sf) + qs = filtered + break # Old API only used the first filter pair + + if isinstance(qs, list): + return Response([_sparse_to_legacy(sf) for sf in qs]) + return Response([_sparse_to_legacy(sf) for sf in qs.iterator()]) + + # -- POST --------------------------------------------------------------- + + @staticmethod + def _extra_metadata_post(request, resource): + data = request.data + if isinstance(data, str): + try: + data = json.loads(data) + except (json.JSONDecodeError, TypeError): + return Response( + {"detail": "Invalid JSON payload."}, + status=400, + ) + + if not isinstance(data, list): + return Response( + {"detail": "Expected a JSON list of metadata objects."}, + status=400, + ) + + for meta_dict in data: + if not isinstance(meta_dict, dict): + continue + meta_dict.pop("id", None) + value = json.dumps(meta_dict) + if len(value) > SPARSE_FIELD_VALUE_MAX_LENGTH: + logger.warning( + "extra_metadata entry skipped: serialized value exceeds " + f"{SPARSE_FIELD_VALUE_MAX_LENGTH} characters" + ) + continue + name = _next_sparse_name(resource) + try: + SparseField.objects.create( + resource=resource, name=name, value=value + ) + except IntegrityError: + # Concurrent request created the same name; retry with UUID + name = f"{SPARSE_FIELD_PREFIX}{uuid.uuid4().hex[:12]}" + SparseField.objects.create( + resource=resource, name=name, value=value + ) + + result = [_sparse_to_legacy(sf) for sf in _sparse_fields_for_resource(resource)] + return Response(result, status=201) + + # -- PUT ---------------------------------------------------------------- + + @staticmethod + def _extra_metadata_put(request, resource): + data = request.data + if isinstance(data, str): + try: + data = json.loads(data) + except (json.JSONDecodeError, TypeError): + return Response( + {"detail": "Invalid JSON payload."}, + status=400, + ) + + if not isinstance(data, list): + return Response( + {"detail": "Expected a JSON list of metadata objects."}, + status=400, + ) + + for meta_dict in data: + if not isinstance(meta_dict, dict): + continue + sf_id = meta_dict.pop("id", None) + if sf_id is None: + continue + value = json.dumps(meta_dict) + if len(value) > SPARSE_FIELD_VALUE_MAX_LENGTH: + logger.warning( + "extra_metadata entry skipped: serialized value exceeds " + f"{SPARSE_FIELD_VALUE_MAX_LENGTH} characters" + ) + continue + SparseField.objects.filter( + pk=sf_id, + resource=resource, + name__startswith=SPARSE_FIELD_PREFIX, + ).update(value=value) + + result = [_sparse_to_legacy(sf) for sf in _sparse_fields_for_resource(resource)] + return Response(result) + + # -- DELETE ------------------------------------------------------------- + + @staticmethod + def _extra_metadata_delete(request, resource): + data = request.data + if isinstance(data, str): + try: + data = json.loads(data) + except (json.JSONDecodeError, TypeError): + return Response( + {"detail": "Invalid JSON payload."}, + status=400, + ) + + if not isinstance(data, list): + return Response( + {"detail": "Expected a JSON list of IDs."}, + status=400, + ) + + ids = [int(i) for i in data if isinstance(i, (int, str)) and str(i).isdigit()] + SparseField.objects.filter( + pk__in=ids, + resource=resource, + name__startswith=SPARSE_FIELD_PREFIX, + ).delete() + + result = [_sparse_to_legacy(sf) for sf in _sparse_fields_for_resource(resource)] + return Response(result) diff --git a/geonode/base/api/serializers.py b/geonode/base/api/serializers.py index 812b7e98ac3..6d8094518d0 100644 --- a/geonode/base/api/serializers.py +++ b/geonode/base/api/serializers.py @@ -67,6 +67,7 @@ from geonode.assets.handlers import asset_handler_registry from geonode.utils import build_absolute_uri from geonode.security.utils import get_resources_with_perms, get_geoapp_subtypes +from geonode.base.api.deprecated_extra_metadata import DeprecatedExtraMetadataField from geonode.resource.models import ExecutionRequest from django.contrib.gis.geos import Polygon from geonode.security.registry import permissions_registry @@ -654,6 +655,8 @@ class ResourceBaseSerializer(MultiLangOutputMixin, DynamicModelSerializer): links = DynamicRelationField(LinksSerializer, source="id", read_only=True) # Deferred fields + # Deprecated: use the sparse fields API instead of ``metadata``. + metadata = DeprecatedExtraMetadataField(deferred=True, read_only=True) data = DataBlobField(DataBlobSerializer, source="id", deferred=True, required=False) executions = DynamicRelationField( ResourceExecutionRequestSerializer, source="id", deferred=True, required=False, read_only=True @@ -735,6 +738,7 @@ class Meta: "sourcetype", "is_copyable", "blob", + "metadata", # Deprecated: use sparse fields API instead "executions", "linked_resources", "download_url", diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py index 13c291cbfcb..d94f4864a7a 100644 --- a/geonode/base/api/tests.py +++ b/geonode/base/api/tests.py @@ -4231,3 +4231,152 @@ def test_map_layer_permission_caching(self): # Check that the permissions in the layers are the same for layer1, layer2 in zip(data1["map"]["maplayers"], data2["map"]["maplayers"]): self.assertEqual(layer1["dataset"]["perms"], layer2["dataset"]["perms"]) + + +class DeprecatedExtraMetadataApiTest(GeoNodeBaseTestSupport): + """Tests for the deprecated backward-compatible ExtraMetadata API adapters. + + These adapters re-expose the old ``/extra_metadata/`` endpoint and the + ``metadata`` serializer field using SparseField as the storage backend. + """ + + def setUp(self): + super().setUp() + self.admin = get_user_model().objects.get(username="admin") + self.dataset = create_single_dataset("deprecated_em_test") + + def _url(self, pk=None): + pk = pk or self.dataset.pk + return urljoin( + f"{reverse('base-resources-list')}/", + f"{pk}/extra_metadata/", + ) + + def test_get_empty_extra_metadata(self): + """GET should return an empty list when no extra metadata exists.""" + self.client.login(username="admin", password="admin") + response = self.client.get(self._url()) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), []) + + def test_post_extra_metadata(self): + """POST should create new extra metadata entries.""" + from geonode.metadata.models import SparseField + + self.client.login(username="admin", password="admin") + payload = [ + {"field_name": "test_field", "field_value": "test_value"}, + {"field_name": "another", "field_value": "value2"}, + ] + response = self.client.post( + self._url(), + data=json.dumps(payload), + content_type="application/json", + ) + self.assertEqual(response.status_code, 201) + result = response.json() + self.assertEqual(len(result), 2) + # Each entry should have an id and the metadata fields + for item in result: + self.assertIn("id", item) + self.assertIn("field_name", item) + self.assertIn("field_value", item) + + # Verify SparseField entries were created + sf_count = SparseField.objects.filter( + resource=self.dataset.resourcebase_ptr, + name__startswith="extra_metadata_", + ).count() + self.assertEqual(sf_count, 2) + + def test_get_returns_posted_metadata(self): + """GET after POST should return the created metadata.""" + self.client.login(username="admin", password="admin") + payload = [{"field_name": "myfield", "field_value": "myvalue"}] + self.client.post( + self._url(), + data=json.dumps(payload), + content_type="application/json", + ) + response = self.client.get(self._url()) + self.assertEqual(response.status_code, 200) + result = response.json() + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["field_name"], "myfield") + self.assertEqual(result[0]["field_value"], "myvalue") + + def test_put_updates_existing_metadata(self): + """PUT should update an existing entry by id.""" + self.client.login(username="admin", password="admin") + # Create first + payload = [{"field_name": "original", "field_value": "v1"}] + response = self.client.post( + self._url(), + data=json.dumps(payload), + content_type="application/json", + ) + created_id = response.json()[0]["id"] + + # Update + update_payload = [ + {"id": created_id, "field_name": "updated", "field_value": "v2"} + ] + response = self.client.put( + self._url(), + data=json.dumps(update_payload), + content_type="application/json", + ) + self.assertEqual(response.status_code, 200) + result = response.json() + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["field_name"], "updated") + self.assertEqual(result[0]["field_value"], "v2") + + def test_delete_removes_metadata(self): + """DELETE should remove entries by id.""" + self.client.login(username="admin", password="admin") + # Create + payload = [ + {"field_name": "to_delete", "field_value": "val"}, + {"field_name": "to_keep", "field_value": "keep"}, + ] + response = self.client.post( + self._url(), + data=json.dumps(payload), + content_type="application/json", + ) + items = response.json() + delete_id = items[0]["id"] + + # Delete one + response = self.client.delete( + self._url(), + data=json.dumps([delete_id]), + content_type="application/json", + ) + self.assertEqual(response.status_code, 200) + result = response.json() + self.assertEqual(len(result), 1) + self.assertNotEqual(result[0]["id"], delete_id) + + def test_metadata_field_in_serializer(self): + """The deprecated ``metadata`` field should appear when requested + via include[] and return data from SparseField entries.""" + from geonode.metadata.models import SparseField + + self.client.login(username="admin", password="admin") + SparseField.objects.create( + resource=self.dataset.resourcebase_ptr, + name="extra_metadata_1", + value=json.dumps({"field_name": "test", "field_value": "val"}), + ) + url = f"{reverse('base-resources-list')}/{self.dataset.pk}?include[]=metadata" + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + data = response.json().get("resource", response.json()) + self.assertIn("metadata", data) + metadata = data["metadata"] + self.assertIsInstance(metadata, list) + if metadata: + self.assertIn("id", metadata[0]) + self.assertIn("field_name", metadata[0]) diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index 6263803677a..6f9d3037242 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -107,6 +107,7 @@ from geonode.people.api.serializers import UserSerializer from .pagination import GeoNodeApiPagination from geonode.base.utils import patch_perms +from geonode.base.api.deprecated_extra_metadata import DeprecatedExtraMetadataMixin from geonode.assets.models import Asset from geonode.assets.utils import create_asset_and_link, unlink_asset from geonode.assets.handlers import asset_handler_registry @@ -288,7 +289,9 @@ def replace_presets(self, request): request.GET._mutable = False -class ResourceBaseViewSet(ApiPresetsInitializer, MultiLangViewMixin, DynamicModelViewSet): +class ResourceBaseViewSet( + ApiPresetsInitializer, MultiLangViewMixin, DeprecatedExtraMetadataMixin, DynamicModelViewSet +): """ API endpoint that allows base resources to be viewed or edited. """ diff --git a/geonode/settings.py b/geonode/settings.py index 77b6d45d779..6c7ac0856b7 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -25,6 +25,7 @@ import logging import subprocess import dj_database_url +from schema import Optional from urllib.parse import urlparse, urljoin # @@ -2163,6 +2164,32 @@ def get_geonode_catalogue_service(): DEFAULT_AUTO_FIELD = "django.db.models.AutoField" +# ---- Deprecated ExtraMetadata settings (kept for backward compatibility) ---- +# These settings are deprecated and will be removed in a future version. +# The ExtraMetadata model has been replaced by SparseField. +# External clients (e.g. mapstore-client) may still reference these settings. + +DEFAULT_EXTRA_METADATA_SCHEMA = { + Optional("id"): int, + "filter_header": object, + "field_name": object, + "field_label": object, + "field_value": object, +} + +CUSTOM_METADATA_SCHEMA = os.getenv("CUSTOM_METADATA_SCHEMA", {}) + +EXTRA_METADATA_SCHEMA = { + **{ + "map": os.getenv("MAP_EXTRA_METADATA_SCHEMA", DEFAULT_EXTRA_METADATA_SCHEMA), + "dataset": os.getenv("DATASET_EXTRA_METADATA_SCHEMA", DEFAULT_EXTRA_METADATA_SCHEMA), + "document": os.getenv("DOCUMENT_EXTRA_METADATA_SCHEMA", DEFAULT_EXTRA_METADATA_SCHEMA), + "geoapp": os.getenv("GEOAPP_EXTRA_METADATA_SCHEMA", DEFAULT_EXTRA_METADATA_SCHEMA), + }, + **CUSTOM_METADATA_SCHEMA, +} +# ---- End deprecated ExtraMetadata settings ---- + """ List of modules that implement custom metadata storers that will be called when the metadata of a resource is saved """ From 87de498eb5891b5ccf51cd0effff7e1dc0a46e9e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 May 2026 16:31:41 +0000 Subject: [PATCH 07/11] Fix E303 flake8: remove extra blank line in views.py after merge conflict resolution Agent-Logs-Url: https://github.com/GeoNode/geonode/sessions/a845ef0f-e7e2-42db-8de0-04f61f0838e1 Co-authored-by: etj <717359+etj@users.noreply.github.com> --- geonode/base/api/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index 6f9d3037242..ecdc6959e61 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -1212,7 +1212,6 @@ def set_thumbnail(self, request, pk, *args, **kwargs): return Response({"thumbnail_url": resource.thumbnail_url}) return Response("Unable to set thumbnail", status=status.HTTP_400_BAD_REQUEST) - @action( detail=True, methods=["get"], From 3183588340e550c07567d0145cab7808c24131eb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 May 2026 16:54:46 +0000 Subject: [PATCH 08/11] Apply black formatting to deprecated_extra_metadata.py, views.py, tests.py Agent-Logs-Url: https://github.com/GeoNode/geonode/sessions/8a0d75b5-2c96-4456-9d56-6bef8adfdd7f Co-authored-by: etj <717359+etj@users.noreply.github.com> --- geonode/base/api/deprecated_extra_metadata.py | 22 +++++++++---------- geonode/base/api/tests.py | 8 ++----- geonode/base/api/views.py | 5 +---- 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/geonode/base/api/deprecated_extra_metadata.py b/geonode/base/api/deprecated_extra_metadata.py index e0d3ec942b8..06596d679cd 100644 --- a/geonode/base/api/deprecated_extra_metadata.py +++ b/geonode/base/api/deprecated_extra_metadata.py @@ -65,8 +65,13 @@ # Query parameter names that should *not* be treated as legacy # ``metadata__`` filters in the deprecated GET endpoint. _NON_FILTER_QUERY_PARAMS = { - "api_preset", "page", "page_size", "format", "include[]", - "exclude[]", "sort[]", + "api_preset", + "page", + "page_size", + "format", + "include[]", + "exclude[]", + "sort[]", } @@ -113,7 +118,7 @@ def _next_sparse_name(resource): ) max_n = 0 for name in existing: - suffix = name[len(SPARSE_FIELD_PREFIX):] + suffix = name[len(SPARSE_FIELD_PREFIX) :] try: max_n = max(max_n, int(suffix)) except (ValueError, TypeError): @@ -172,8 +177,7 @@ class DeprecatedExtraMetadataMixin: @extend_schema( methods=["get", "put", "delete", "post"], description=( - "[DEPRECATED] Get/Update/Delete/Add extra metadata for a resource. " - "Use the sparse fields API instead." + "[DEPRECATED] Get/Update/Delete/Add extra metadata for a resource. " "Use the sparse fields API instead." ), deprecated=True, ) @@ -260,15 +264,11 @@ def _extra_metadata_post(request, resource): continue name = _next_sparse_name(resource) try: - SparseField.objects.create( - resource=resource, name=name, value=value - ) + SparseField.objects.create(resource=resource, name=name, value=value) except IntegrityError: # Concurrent request created the same name; retry with UUID name = f"{SPARSE_FIELD_PREFIX}{uuid.uuid4().hex[:12]}" - SparseField.objects.create( - resource=resource, name=name, value=value - ) + SparseField.objects.create(resource=resource, name=name, value=value) result = [_sparse_to_legacy(sf) for sf in _sparse_fields_for_resource(resource)] return Response(result, status=201) diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py index d94f4864a7a..b4a1a1e6944 100644 --- a/geonode/base/api/tests.py +++ b/geonode/base/api/tests.py @@ -1993,10 +1993,8 @@ def test_set_resource_thumbnail(self): self.assertEqual(response.json(), "The url must be of an image with format (png, jpeg or jpg)") # using Base64 data as an ASCII byte string - data["file"] = ( - "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAABHNCSVQICAgI\ + data["file"] = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAABHNCSVQICAgI\ fAhkiAAAABl0RVh0U29mdHdhcmUAZ25vbWUtc2NyZWVuc2hvdO8Dvz4AAAANSURBVAiZYzAxMfkPAALYAZzx61+bAAAAAElFTkSuQmCC" - ) with patch("geonode.base.models.is_monochromatic_image") as _mck: _mck.return_value = False response = self.client.put(url, data=data, format="json") @@ -4318,9 +4316,7 @@ def test_put_updates_existing_metadata(self): created_id = response.json()[0]["id"] # Update - update_payload = [ - {"id": created_id, "field_name": "updated", "field_value": "v2"} - ] + update_payload = [{"id": created_id, "field_name": "updated", "field_value": "v2"}] response = self.client.put( self._url(), data=json.dumps(update_payload), diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index ecdc6959e61..95a61b6f899 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -113,7 +113,6 @@ from geonode.assets.handlers import asset_handler_registry from geonode.utils import get_supported_datasets_file_types - logger = logging.getLogger(__name__) @@ -289,9 +288,7 @@ def replace_presets(self, request): request.GET._mutable = False -class ResourceBaseViewSet( - ApiPresetsInitializer, MultiLangViewMixin, DeprecatedExtraMetadataMixin, DynamicModelViewSet -): +class ResourceBaseViewSet(ApiPresetsInitializer, MultiLangViewMixin, DeprecatedExtraMetadataMixin, DynamicModelViewSet): """ API endpoint that allows base resources to be viewed or edited. """ From 2f245ad67b57ee1a78e5243f1fdeb331c0bdbeab Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 May 2026 17:48:35 +0000 Subject: [PATCH 09/11] Remove drf_spectacular import from deprecated_extra_metadata.py Agent-Logs-Url: https://github.com/GeoNode/geonode/sessions/5fc50f73-190e-46f5-b7fd-2d885ed21c7c Co-authored-by: etj <717359+etj@users.noreply.github.com> --- geonode/base/api/deprecated_extra_metadata.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/geonode/base/api/deprecated_extra_metadata.py b/geonode/base/api/deprecated_extra_metadata.py index 06596d679cd..a73f7f7e9cc 100644 --- a/geonode/base/api/deprecated_extra_metadata.py +++ b/geonode/base/api/deprecated_extra_metadata.py @@ -41,7 +41,6 @@ from deprecated import deprecated from rest_framework.decorators import action from rest_framework.response import Response -from drf_spectacular.utils import extend_schema from dynamic_rest.fields.fields import DynamicComputedField @@ -174,13 +173,6 @@ class DeprecatedExtraMetadataMixin: Use the sparse fields API instead. """ - @extend_schema( - methods=["get", "put", "delete", "post"], - description=( - "[DEPRECATED] Get/Update/Delete/Add extra metadata for a resource. " "Use the sparse fields API instead." - ), - deprecated=True, - ) @action( detail=True, methods=["get", "put", "delete", "post"], From fecb49ec6cb8dc77a6afabd3a90397e58535e121 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 13 May 2026 12:17:48 +0000 Subject: [PATCH 10/11] Renumber migration to 0100 to fix leaf node conflict with 0099_resourcebase_auth_config Agent-Logs-Url: https://github.com/GeoNode/geonode/sessions/cb857a4d-d999-4228-ad98-fe4648e898aa Co-authored-by: etj <717359+etj@users.noreply.github.com> --- ...efields.py => 0100_migrate_extrametadata_to_sparsefields.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename geonode/base/migrations/{0098_migrate_extrametadata_to_sparsefields.py => 0100_migrate_extrametadata_to_sparsefields.py} (96%) diff --git a/geonode/base/migrations/0098_migrate_extrametadata_to_sparsefields.py b/geonode/base/migrations/0100_migrate_extrametadata_to_sparsefields.py similarity index 96% rename from geonode/base/migrations/0098_migrate_extrametadata_to_sparsefields.py rename to geonode/base/migrations/0100_migrate_extrametadata_to_sparsefields.py index 234d4b9644c..b6c8c13591f 100644 --- a/geonode/base/migrations/0098_migrate_extrametadata_to_sparsefields.py +++ b/geonode/base/migrations/0100_migrate_extrametadata_to_sparsefields.py @@ -36,7 +36,7 @@ def migrate_extrametadata_to_sparsefields(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ("base", "0097_alter_link_asset"), + ("base", "0099_resourcebase_auth_config"), ("metadata", "0001_initial"), ] From 3c15e1039f479e5fb00f2425d6f2f45a8272c87c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 13 May 2026 13:45:57 +0000 Subject: [PATCH 11/11] Apply black==24.4.2 formatting to base/api/tests.py Agent-Logs-Url: https://github.com/GeoNode/geonode/sessions/a710ea51-7e89-45d3-a6d4-74229bd12a5c Co-authored-by: etj <717359+etj@users.noreply.github.com> --- geonode/base/api/tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py index b4a1a1e6944..a64947c2bc5 100644 --- a/geonode/base/api/tests.py +++ b/geonode/base/api/tests.py @@ -1993,8 +1993,10 @@ def test_set_resource_thumbnail(self): self.assertEqual(response.json(), "The url must be of an image with format (png, jpeg or jpg)") # using Base64 data as an ASCII byte string - data["file"] = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAABHNCSVQICAgI\ + data["file"] = ( + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAABHNCSVQICAgI\ fAhkiAAAABl0RVh0U29mdHdhcmUAZ25vbWUtc2NyZWVuc2hvdO8Dvz4AAAANSURBVAiZYzAxMfkPAALYAZzx61+bAAAAAElFTkSuQmCC" + ) with patch("geonode.base.models.is_monochromatic_image") as _mck: _mck.return_value = False response = self.client.put(url, data=data, format="json")