diff --git a/.github/workflows/default-tests.yml b/.github/workflows/default-tests.yml index aed362ab..0014ab1d 100644 --- a/.github/workflows/default-tests.yml +++ b/.github/workflows/default-tests.yml @@ -33,6 +33,7 @@ jobs: - name: Setup Micromamba ${{ matrix.python-version }} uses: mamba-org/setup-micromamba@d7c9bd84e824b79d2af72a2d4196c7f4300d3476 # v3.0.0 with: + micromamba-version: '2.6.0-0' environment-name: TEST init-shell: bash create-args: >- diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3010af88..6edcd4b8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: - id: add-trailing-comma - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.12 + rev: v0.15.13 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -50,7 +50,7 @@ repos: )$ - repo: https://github.com/woodruffw/zizmor-pre-commit - rev: v1.24.1 + rev: v1.25.2 hooks: - id: zizmor diff --git a/compliance_checker/base.py b/compliance_checker/base.py index 8b676a49..9762be77 100644 --- a/compliance_checker/base.py +++ b/compliance_checker/base.py @@ -15,22 +15,12 @@ import validators from lxml import etree from netCDF4 import Dataset -from owslib.namespaces import Namespaces -from owslib.swe.observation.sos100 import SensorObservationService_1_0_0 -from owslib.swe.sensor.sml import SensorML from typing_extensions import deprecated from compliance_checker import __version__ from compliance_checker.util import kvp_convert -def get_namespaces(): - n = Namespaces() - ns = n.get_namespaces(["ogc", "sml", "gml", "sos", "swe", "xlink"]) - ns["ows"] = n.get_namespace("ows110") - return ns - - def csv_splitter(input_string): """ csv_splitter(input_string) @@ -207,28 +197,6 @@ def std_check(cls, dataset, name): return name in dataset.ncattrs() -@deprecated( - "The ioos_sos checker is deprecated and will be removed in the next compliance-checker version.", -) -class BaseSOSGCCheck: - """ - Base class for SOS-GetCapabilities supporting Check Suites. - """ - - supported_ds = [SensorObservationService_1_0_0] - - -@deprecated( - "The ioos_sos checker is deprecated and will be removed in the next compliance-checker version.", -) -class BaseSOSDSCheck: - """ - Base class for SOS-DescribeSensor supporting Check Suites. - """ - - supported_ds = [SensorML] - - class Result: """ Holds the result of a check method. diff --git a/compliance_checker/ioos.py b/compliance_checker/ioos.py index 77c4c256..c7dbb869 100644 --- a/compliance_checker/ioos.py +++ b/compliance_checker/ioos.py @@ -6,9 +6,6 @@ from numbers import Number import validators -from lxml.etree import XPath -from owslib.namespaces import Namespaces -from typing_extensions import deprecated import compliance_checker.cf.util as cfutil from compliance_checker import base @@ -16,8 +13,6 @@ from compliance_checker.base import ( BaseCheck, BaseNCCheck, - BaseSOSDSCheck, - BaseSOSGCCheck, Result, TestCtx, check_has, @@ -1725,266 +1720,3 @@ def check_instrument_make_model_calib_date(self, ds): ) return results - - -@deprecated( - "The ioos_sos checker is deprecated and will be removed in the next compliance-checker version.", -) -class IOOSBaseSOSCheck(BaseCheck): - _cc_spec = "ioos_sos" - _cc_spec_version = "0.1" - _cc_description = "IOOS Inventory Metadata checks for the Sensor Observation System (SOS). Checks SOS functions GetCapabilities and DescribeSensor." - register_checker = True - # requires login - _cc_url = "http://sdf.ndbc.noaa.gov/sos/" - - -@deprecated( - "The ioos_sos checker is deprecated and will be removed in the next compliance-checker version.", -) -class IOOSSOSGCCheck(BaseSOSGCCheck, IOOSBaseSOSCheck): - # set up namespaces for XPath - ns = Namespaces().get_namespaces(["sos", "gml", "xlink"]) - ns["ows"] = Namespaces().get_namespace("ows110") - - @check_has(BaseCheck.HIGH) - def check_high(self, ds): - return [] - - @check_has(BaseCheck.MEDIUM) - def check_recommended(self, ds): - return [ - ( - "service_contact_email", - XPath( - "/sos:Capabilities/ows:ServiceProvider/ows:ServiceContact/ows:ContactInfo/ows:Address/ows:ElectronicMailAddress", - namespaces=self.ns, - ), - ), - ( - "service_contact_name", - XPath( - "/sos:Capabilities/ows:ServiceProvider/ows:ServiceContact/ows:IndividualName", - namespaces=self.ns, - ), - ), - ( - "service_provider_name", - XPath( - "/sos:Capabilities/ows:ServiceProvider/ows:ProviderName", - namespaces=self.ns, - ), - ), - ( - "service_title", - XPath( - "/sos:Capabilities/ows:ServiceProvider/ows:ProviderName", - namespaces=self.ns, - ), - ), - ( - "service_type_name", - XPath( - "/sos:Capabilities/ows:ServiceIdentification/ows:ServiceType", - namespaces=self.ns, - ), - ), - ( - "service_type_version", - XPath( - "/sos:Capabilities/ows:ServiceIdentification/ows:ServiceTypeVersion", - namespaces=self.ns, - ), - ), - # ds.identification[0].observed_properties has this as well, but - # don't want to try to shoehorn a function here - # ('variable_names', len(ds.identification[0].observed_properties) > 0) - ( - "variable_names", - XPath( - "/sos:Capabilities/sos:Contents/sos:ObservationOfferingList/sos:ObservationOffering/sos:observedProperty", - namespaces=self.ns, - ), - ), - ( - "data_format_template_version", - XPath( - "/sos:Capabilities/ows:OperationsMetadata/ows:ExtendedCapabilities/gml:metaDataProperty[@xlink:title='ioosTemplateVersion']/gml:version", - namespaces=self.ns, - ), - ), - ] - - @check_has(BaseCheck.LOW) - def check_suggested(self, ds): - return ["altitude_units"] - - -@deprecated( - "The ioos_sos checker is deprecated and will be removed in the next compliance-checker version.", -) -class IOOSSOSDSCheck(BaseSOSDSCheck, IOOSBaseSOSCheck): - # set up namespaces for XPath - ns = Namespaces().get_namespaces(["sml", "swe", "gml", "xlink"]) - - @check_has(BaseCheck.HIGH) - def check_high(self, ds): - return [ - ( - "platform_sponsor", - XPath( - "/sml:SensorML/sml:member/sml:System/sml:classification/sml:ClassifierList/sml:classifier[@name='sponsor']/sml:Term/sml:value", - namespaces=self.ns, - ), - ), - ( - "platform_type", - XPath( - "/sml:SensorML/sml:member/sml:System/sml:classification/sml:ClassifierList/sml:classifier[@name='platformType']/sml:Term/sml:value", - namespaces=self.ns, - ), - ), - ( - "station_publisher_name", - XPath( - "/sml:SensorML/sml:member/sml:System/sml:contact/sml:ContactList/sml:member[@xlink:role='http://mmisw.org/ont/ioos/definition/publisher']/sml:ResponsibleParty/sml:organizationName", - namespaces=self.ns, - ), - ), - ( - "station_publisher_email", - XPath( - "/sml:SensorML/sml:member/sml:System/sml:contact/sml:ContactList/sml:member[@xlink:role='http://mmisw.org/ont/ioos/definition/publisher']/sml:ResponsibleParty/sml:contactInfo/address/sml:electronicMailAddress", - namespaces=self.ns, - ), - ), - ( - "station_id", - XPath( - "/sml:SensorML/sml:member/sml:System/sml:identification/sml:IdentifierList/sml:identifier[@name='stationID']/sml:Term/sml:value", - namespaces=self.ns, - ), - ), - ( - "station_long_name", - XPath( - "/sml:SensorML/sml:member/sml:System/sml:identification/sml:IdentifierList/sml:identifier[@name='longName']/sml:Term/sml:value", - namespaces=self.ns, - ), - ), - ( - "station_short_name", - XPath( - "/sml:SensorML/sml:member/sml:System/sml:identification/sml:IdentifierList/sml:identifier[@name='shortName']/sml:Term/sml:value", - namespaces=self.ns, - ), - ), - ( - "station_wmo_id", - XPath( - '/sml:SensorML/sml:member/sml:System/sml:identification/sml:IdentifierList/sml:identifier/sml:Term[@definition="http://mmisw.org/ont/ioos/definition/wmoID"]/sml:value', - namespaces=self.ns, - ), - ), - ( - "time_period", - XPath( - "/sml:SensorML/sml:member/sml:System/sml:capabilities[@name='observationTimeRange']/swe:DataRecord/swe:field[@name='observationTimeRange']/swe:TimeRange/swe:value", - namespaces=self.ns, - ), - ), - ( - "operator_email", - XPath( - "/sml:SensorML/sml:member/sml:System/sml:contact/sml:ContactList/sml:member[@xlink:role='http://mmisw.org/ont/ioos/definition/operator']/sml:ResponsibleParty/sml:contactInfo/address/sml:electronicMailAddress", - namespaces=self.ns, - ), - ), - ( - "operator_name", - XPath( - "/sml:SensorML/sml:member/sml:System/sml:contact/sml:ContactList/sml:member[@xlink:role='http://mmisw.org/ont/ioos/definition/operator']/sml:ResponsibleParty/sml:organizationName", - namespaces=self.ns, - ), - ), - ( - "station_description", - XPath( - "/sml:SensorML/sml:member/sml:System/gml:description", - namespaces=self.ns, - ), - ), - # replaced with lon/lat with point - ( - "station_location_point", - XPath( - "/sml:SensorML/sml:member/sml:System/sml:location/gml:Point/gml:pos", - namespaces=self.ns, - ), - ), - ] - - @check_has(BaseCheck.MEDIUM) - def check_recommended(self, ds): - return [ - ( - "sensor_descriptions", - XPath( - "/sml:SensorML/sml:member/sml:System/sml:components/sml:ComponentList/sml:component/sml:System/gml:description", - namespaces=self.ns, - ), - ), - ( - "sensor_ids", - XPath( - "/sml:SensorML/sml:member/sml:System/sml:components/sml:ComponentList/sml:component/sml:System/@gml:id", - namespaces=self.ns, - ), - ), - ( - "sensor_names", - XPath( - "/sml:SensorML/sml:member/sml:System/sml:components/sml:ComponentList/sml:component/@name", - namespaces=self.ns, - ), - ), - ( - "data_format_template_version", - XPath( - "/sml:SensorML/sml:capabilities/swe:SimpleDataRecord/swe:field[@name='ioosTemplateVersion']/swe:Text/swe:value", - namespaces=self.ns, - ), - ), - ( - "variable_names", - XPath( - "/sml:SensorML/sml:member/sml:System/sml:components/sml:ComponentList/sml:component/sml:System/sml:outputs/sml:OutputList/sml:output/swe:Quantity/@definition", - namespaces=self.ns, - ), - ), - ( - "variable_units", - XPath( - "/sml:SensorML/sml:member/sml:System/sml:components/sml:ComponentList/sml:component/sml:System/sml:outputs/sml:OutputList/sml:output/swe:Quantity/swe:uom/@code", - namespaces=self.ns, - ), - ), - ( - "network_id", - XPath( - "/sml:SensorML/sml:member/sml:System/sml:capabilities[@name='networkProcedures']/swe:SimpleDataRecord/gml:metaDataProperty/@xlink:href", - namespaces=self.ns, - ), - ), - ( - "operator_sector", - XPath( - "/sml:SensorML/sml:member/sml:System/sml:classification/sml:ClassifierList/sml:classifier[@name='operatorSector']/sml:Term/sml:value", - namespaces=self.ns, - ), - ), - ] - - @check_has(BaseCheck.LOW) - def check_suggested(self, ds): - return [] diff --git a/compliance_checker/suite.py b/compliance_checker/suite.py index 0d0a57b7..dc2b14c9 100644 --- a/compliance_checker/suite.py +++ b/compliance_checker/suite.py @@ -19,10 +19,7 @@ from urllib.parse import urlparse import requests -from lxml import etree as ET from netCDF4 import Dataset -from owslib.sos import SensorObservationService -from owslib.swe.sensor.sml import SensorML from packaging.version import parse from compliance_checker import __version__, tempnc @@ -309,7 +306,6 @@ def _get_valid_checkers(self, ds, checker_names): args = [(name, self.checkers[name]) for name in checker_names if name in self.checkers] valid = [] - all_checked = {a[1] for a in args} # only class types checker_queue = set(args) while len(checker_queue): name, a = checker_queue.pop() @@ -318,13 +314,6 @@ def _get_valid_checkers(self, ds, checker_names): if type(ds) in a().supported_ds: valid.append((name, a)) - # add subclasses of SOS checks - if "ioos_sos" in name: - for subc in a.__subclasses__(): - if subc not in all_checked: - all_checked.add(subc) - checker_queue.add((name, subc)) - return valid @classmethod @@ -734,24 +723,6 @@ def process_table(res, check): has_printed = True return "\n".join(proc_strs) - def process_doc(self, doc): - """ - Attempt to parse an xml string conforming to either an SOS or SensorML - dataset and return the results - """ - xml_doc = ET.fromstring(doc) - if xml_doc.tag == "{http://www.opengis.net/sos/1.0}Capabilities": - ds = SensorObservationService(None, xml=doc) - # SensorObservationService does not store the etree doc root, - # so maybe use monkey patching here for now? - ds._root = xml_doc - - elif xml_doc.tag == "{http://www.opengis.net/sensorML/1.0.1}SensorML": - ds = SensorML(xml_doc) - else: - raise ValueError(f"Unrecognized XML root element: {xml_doc.tag}") - return ds - def generate_dataset(self, cdl_path): """ Use ncgen to generate a netCDF file from a .cdl file @@ -793,8 +764,7 @@ def generate_dataset(self, cdl_path): def load_dataset(self, ds_str): """ - Returns an instantiated instance of either a netCDF file or an SOS - mapped DS object. + Returns an instantiated instance of a netCDF-like file mapped DS object. :param str ds_str: URL of the resource to load """ @@ -825,7 +795,7 @@ def check_remote_netcdf(self, ds_str): def load_remote_dataset(self, ds_str): """ - Returns a dataset instance for the remote resource, either OPeNDAP or SOS + Returns a dataset instance for the remote resource. :param str ds_str: URL to the remote resource """ @@ -855,15 +825,10 @@ def load_remote_dataset(self, ds_str): elif opendap.is_opendap(ds_str): return Dataset(ds_str) - # Check if the HTTP response is XML, if it is, it's likely SOS so - # we'll attempt to parse the response as SOS. - # Some SOS servers don't seem to support HEAD requests. - # Issue GET instead if we reach here and can't get the response response = requests.get(ds_str, allow_redirects=True, timeout=60) content_type = response.headers.get("content-type") - if content_type.split(";")[0] == "text/xml": - return self.process_doc(response.content) - elif content_type.split(";")[0] == "application/x-netcdf": + + if content_type.split(";")[0] == "application/x-netcdf": return Dataset( urlparse(response.url).path, memory=response.content, diff --git a/compliance_checker/tests/test_ioos_sos.py b/compliance_checker/tests/test_ioos_sos.py deleted file mode 100644 index 1ccae85b..00000000 --- a/compliance_checker/tests/test_ioos_sos.py +++ /dev/null @@ -1,82 +0,0 @@ -import os - -from mocket import mocketize -from mocket.mockhttp import Entry - -from compliance_checker.runner import ComplianceChecker -from compliance_checker.suite import CheckSuite - - -# TODO: Use inheritance to eliminate redundant code in test setup, etc -class TestIOOSSOSGetCapabilities: - def setup_method(self): - with open( - os.path.join( - os.path.dirname(__file__), - "data/http_mocks/ncsos_getcapabilities.xml", - ), - ) as f: - self.resp = f.read() - # need to monkey patch checkers prior to running tests, or no checker - # classes will show up - CheckSuite().load_all_available_checkers() - - @mocketize - def test_retrieve_getcaps(self): - """Method that simulates retrieving SOS GetCapabilities""" - url = "http://data.oceansmap.com/thredds/sos/caricoos_ag/VIA/VIA.ncml" - Entry.single_register( - method=Entry.GET, - uri=url, - body=self.resp, - headers={"content-type": "text/xml"}, - ) - Entry.single_register( - Entry.HEAD, - url, - headers={"content-type": "text/xml"}, - body="HTTP/1.1 200", - ) - ComplianceChecker.run_checker(url, ["ioos_sos"], 1, "normal") - - -class TestIOOSSOSDescribeSensor: - def setup_method(self): - with open( - os.path.join( - os.path.dirname(__file__), - "data/http_mocks/ncsos_describesensor.xml", - ), - ) as f: - self.resp = f.read() - # need to monkey patch checkers prior to running tests, or no checker - # classes will show up - CheckSuite().load_all_available_checkers() - - @mocketize - def test_retrieve_describesensor(self): - """Method that simulates retrieving SOS DescribeSensor""" - url = ( - "http://data.oceansmap.com/thredds/sos/caricoos_ag/VIA/VIA.ncml?" - "request=describesensor" - "&service=sos" - "&procedure=urn:ioos:station:ncsos:VIA" - "&outputFormat=text/xml%3Bsubtype%3D%22sensorML/1.0.1/profiles/ioos_sos/1.0%22" - "&version=1.0.0" - ) - Entry.single_register( - method=Entry.GET, - uri=url, - body=self.resp, - headers={"content-type": "text/xml"}, - ) - Entry.single_register( - method=Entry.HEAD, - uri=url, - body="HTTP/1.1 200", - headers={"content-type": "text/xml"}, - ) - # need to mock out the HEAD response so that compliance checker - # recognizes this as some sort of XML doc instead of an OPeNDAP - # source - ComplianceChecker.run_checker(url, ["ioos_sos"], 1, "normal") diff --git a/docs/source/conf.py b/docs/source/conf.py index c23f6b8e..a81f7523 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -170,4 +170,5 @@ linkcheck_ignore = [ "https://github.com/ioos/compliance-checker/issues/.*", "urn:lsid.*", + "https://mmisw.org/ont/.*", ] diff --git a/pyproject.toml b/pyproject.toml index 9fb35d35..d41e7629 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,6 @@ entry-points."compliance_checker.suites"."cf-1.9" = "compliance_checker.cf:CF1_9 entry-points."compliance_checker.suites"."ioos-0.1" = "compliance_checker.ioos:IOOS0_1Check" entry-points."compliance_checker.suites"."ioos-1.1" = "compliance_checker.ioos:IOOS1_1Check" entry-points."compliance_checker.suites"."ioos-1.2" = "compliance_checker.ioos:IOOS1_2Check" -entry-points."compliance_checker.suites".ioos_sos = "compliance_checker.ioos:IOOSBaseSOSCheck" [tool.setuptools] packages = [