Task/add headline data to public api ingestion/cdd 3359

ingestion/consumer.py

@@ -27,6 +27,7 @@
 API_TIME_SERIES_MODEL = MetricsAPIInterface.get_api_timeseries()
 CORE_TIME_SERIES_MODEL = MetricsAPIInterface.get_core_timeseries()
 CORE_HEADLINE_MODEL = MetricsAPIInterface.get_core_headline()
+API_HEADLINE_MODEL = MetricsAPIInterface.get_api_headline()
 
 
 class SupportingModelsLookup(NamedTuple):
@@ -115,6 +116,7 @@ def __init__(
         age_manager: Manager = DEFAULT_AGE_MANAGER,
         stratum_manager: Manager = DEFAULT_STRATUM_MANAGER,
         core_headline_manager: Manager = CORE_HEADLINE_MODEL.objects,
+        api_headline_manager: Manager = API_HEADLINE_MODEL.objects,
         core_timeseries_manager: Manager = CORE_TIME_SERIES_MODEL.objects,
         api_timeseries_manager: Manager = API_TIME_SERIES_MODEL.objects,
     ):
@@ -133,6 +135,7 @@ def __init__(
         self.age_manager = age_manager
         self.stratum_manager = stratum_manager
         self.core_headline_manager = core_headline_manager
+        self.api_headline_manager = api_headline_manager
         self.core_timeseries_manager = core_timeseries_manager
         self.api_timeseries_manager = api_timeseries_manager
 
@@ -345,7 +348,7 @@ def update_supporting_models(self) -> SupportingModelsLookup:
             age_id=age.id,
         )
 
-    def process_core_headlines(self) -> None:
+    def process_core_and_api_headlines(self) -> None:
         """Creates `CoreHeadline` database records from the ingested data after stale records are deleted ahead of time.
 
         Notes:
@@ -361,7 +364,7 @@ def process_core_headlines(self) -> None:
 
         """
         self.clear_stale_headlines()
-        self.create_core_headlines()
+        self.create_core_and_api_headlines()
 
     def process_core_and_api_timeseries(self) -> None:
         """Creates `APITimeSeries` and `CoreTimeSeries` records from the ingested data after stale records are deleted.
@@ -440,6 +443,60 @@ def create_core_headlines(self) -> None:
             model_manager=self.core_headline_manager, model_instances=core_headlines
         )
 
+    def build_api_headlines(self):
+        """Builds `APIHeadline` model instances from the ingested data
+
+        Returns:
+            List of `APIHeadline` model instances
+
+        """
+        return [
+            API_HEADLINE_MODEL(
+                theme=self.dto.parent_theme,
+                sub_theme=self.dto.child_theme,
+                topic=self.dto.topic,
+                metric=self.dto.metric,
+                metric_group=self.dto.metric_group,
+                metric_value=headline_data.metric_value,
+                geography=self.dto.geography,
+                geography_type=self.dto.geography_type,
+                geography_code=self.dto.geography_code,
+                stratum=self.dto.stratum,
+                sex=self.dto.sex,
+                age=self.dto.age,
+                period_start=headline_data.period_start,
+                period_end=headline_data.period_end,
+                refresh_date=self.dto.refresh_date,
+                embargo=headline_data.embargo,
+                upper_confidence=headline_data.upper_confidence,
+                lower_confidence=headline_data.lower_confidence,
+                force_write=headline_data.force_write,
+                is_public=headline_data.is_public,
+            )
+            for headline_data in self.dto.data
+        ]
+
+    def create_api_headlines(self):
+        """Builds `APIHeadline` model instances from the ingested data
+
+        Returns:
+            List of `APIHeadline` model instances
+
+        """
+        api_headlines = self.build_api_headlines()
+        return create_records(
+            model_manager=self.api_headline_manager, model_instances=api_headlines
+        )
+
+    def create_core_and_api_headlines(self) -> None:
+        """Creates `APIHeadline` and `CoreHeadline` records from the ingested data after stale records are deleted.
+
+        Returns:
+            None
+        """
+        self.create_core_headlines()
+        self.create_api_headlines()
+
     def build_core_time_series(self) -> list[CORE_TIME_SERIES_MODEL]:
         """Builds `CoreTimeSeries` model instances from the ingested data
 
@@ -505,12 +562,42 @@ def create_core_time_series(self) -> None:
         )
 
     def clear_stale_headlines(self):
-        """Deletes all stale records for the `CoreHeadline` records relevant to the ingested dataset
+        """Deletes all stale records for both `CoreHeadline` and `APIHeadline` relevant to the ingested dataset
 
         Returns:
             None
 
         """
+        self._clear_stale_api_headlines()
+        self._clear_stale_core_headlines()
+
+    def _clear_stale_api_headlines(self):
+        """Deletes all stale records for the `APIHeadline` records relevant to the ingested dataset
+
+        Returns:
+            None
+        """
+        params = {
+            "theme": self.dto.parent_theme,
+            "sub_theme": self.dto.child_theme,
+            "topic": self.dto.topic,
+            "metric": self.dto.metric,
+            "geography": self.dto.geography,
+            "geography_type": self.dto.geography_type,
+            "geography_code": self.dto.geography_code,
+            "stratum": self.dto.stratum,
+            "sex": self.dto.sex,
+            "age": self.dto.age,
+        }
+        self.api_headline_manager.delete_superseded_data(**params, is_public=True)
+        self.api_headline_manager.delete_superseded_data(**params, is_public=False)
+
+    def _clear_stale_core_headlines(self):
+        """Deletes all stale records for the `CoreHeadline` records relevant to the ingested dataset
+
+        Returns:
+            None
+        """
         params = {
             "topic": self.dto.topic,
             "metric": self.dto.metric,

ingestion/data_transfer_models/headline.py

@@ -20,6 +20,7 @@ class InboundHeadlineSpecificFields(BaseModel):
     embargo: datetime.datetime | None
     metric_value: float
     lower_confidence: float | None = None
+    force_write: bool = False
     is_public: bool = True
 
     @field_validator("embargo")

ingestion/file_ingestion.py

@@ -39,7 +39,7 @@ def data_ingester(*, data: INCOMING_DATA_TYPE, filename: str) -> None:
     consumer = Consumer(source_data=data, filename=filename)
 
     if consumer.is_headline_data:
-        return consumer.process_core_headlines()
+        return consumer.process_core_and_api_headlines()
 
     return consumer.process_core_and_api_timeseries()
 

ingestion/metrics_interface/interface.py

@@ -44,6 +44,10 @@ def get_stratum_manager():
     def get_core_headline_manager():
         return core_models.CoreHeadline.objects
 
+    @staticmethod
+    def get_api_headline_manager():
+        return api_models.APIHeadline.objects
+
     @staticmethod
     def get_core_timeseries_manager():
         return core_models.CoreTimeSeries.objects
@@ -60,6 +64,10 @@ def get_time_period_enum() -> TimePeriod:
     def get_core_headline():
         return core_models.CoreHeadline
 
+    @staticmethod
+    def get_api_headline():
+        return api_models.APIHeadline
+
     @staticmethod
     def get_core_timeseries():
         return core_models.CoreTimeSeries

ingestion/operations/truncated_dataset.py

@@ -33,6 +33,7 @@ def collect_all_metric_model_managers() -> tuple[models.Manager, ...]:
     """Collects all model managers associated with the metrics app"""
     return (
         MetricsAPIInterface.get_core_headline_manager(),
+        MetricsAPIInterface.get_api_headline_manager(),
         MetricsAPIInterface.get_core_timeseries_manager(),
         MetricsAPIInterface.get_api_timeseries_manager(),
         MetricsAPIInterface.get_metric_manager(),
@@ -63,6 +64,7 @@ def clear_metrics_tables() -> None:
             - Sex
             - Stratum
             - CoreHeadline
+            - APIHeadline
             - CoreTimeSeries
             - APITimeSeries