From b13f884588fa2f0c4aed2bf8ff7bd40463b4aeb8 Mon Sep 17 00:00:00 2001 From: bstorm Date: Thu, 12 Feb 2026 10:17:20 -0700 Subject: [PATCH 01/24] Initialize PR From 7230c3b5e727ee043ee20077dd30391369806937 Mon Sep 17 00:00:00 2001 From: bstorm Date: Thu, 12 Feb 2026 11:04:09 -0700 Subject: [PATCH 02/24] create test file --- gtep/tests/unit/test_gtep_data.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 gtep/tests/unit/test_gtep_data.py diff --git a/gtep/tests/unit/test_gtep_data.py b/gtep/tests/unit/test_gtep_data.py new file mode 100644 index 00000000..82e80d2c --- /dev/null +++ b/gtep/tests/unit/test_gtep_data.py @@ -0,0 +1,19 @@ +################################################################################# +# The Institute for the Design of Advanced Energy Systems Integrated Platform +# Framework (IDAES IP) was produced under the DOE Institute for the +# Design of Advanced Energy Systems (IDAES). +# +# Copyright (c) 2018-2026 by the software owners: The Regents of the +# University of California, through Lawrence Berkeley National Laboratory, +# National Technology & Engineering Solutions of Sandia, LLC, Carnegie Mellon +# University, West Virginia University Research Corporation, et al. +# All rights reserved. Please see the files COPYRIGHT.md and LICENSE.md +# for full copyright and license information. +################################################################################# +import pytest +import pyomo.common.unittest as unittest +import gtep_data + +# Instances to check +# - user input options dict +# - From 37d36d6253ed9eb1d10718aa7333485ffc1f4935 Mon Sep 17 00:00:00 2001 From: Belle Date: Thu, 5 Mar 2026 08:43:08 -0700 Subject: [PATCH 03/24] create test file for gtep_data --- gtep/tests/unit/test_gtep_data.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/gtep/tests/unit/test_gtep_data.py b/gtep/tests/unit/test_gtep_data.py index 82e80d2c..41bdc6b2 100644 --- a/gtep/tests/unit/test_gtep_data.py +++ b/gtep/tests/unit/test_gtep_data.py @@ -14,6 +14,25 @@ import pyomo.common.unittest as unittest import gtep_data -# Instances to check -# - user input options dict -# - + +class TestExpansionPlanningData(unittest.TestCase): + def test_load_prescient(): + # test that representative data can be loaded from prescient with all default loading + pass + + def test_load_prescient_with_options_input(): + # test that an options dictionary input gets used in the data loader + pass + + def test_load_prescient_generator_in_service_false(): + pass + + def test_load_prescient_branch_in_service_false(): + pass + + def test_load_prescient_storage_in_service_false(): + pass + + def test_load_prescient_4_dates(): + # test that 4 representative dates results in the correct weights + pass From b7fa41d160e8710adfffcc3bb6eaa980dfed6a66 Mon Sep 17 00:00:00 2001 From: bstorm Date: Mon, 6 Apr 2026 10:16:13 -0600 Subject: [PATCH 04/24] Add initialization test --- gtep/tests/unit/test_gtep_data.py | 52 +++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 gtep/tests/unit/test_gtep_data.py diff --git a/gtep/tests/unit/test_gtep_data.py b/gtep/tests/unit/test_gtep_data.py new file mode 100644 index 00000000..a847c0fc --- /dev/null +++ b/gtep/tests/unit/test_gtep_data.py @@ -0,0 +1,52 @@ +################################################################################# +# The Institute for the Design of Advanced Energy Systems Integrated Platform +# Framework (IDAES IP) was produced under the DOE Institute for the +# Design of Advanced Energy Systems (IDAES). +# +# Copyright (c) 2018-2026 by the software owners: The Regents of the +# University of California, through Lawrence Berkeley National Laboratory, +# National Technology & Engineering Solutions of Sandia, LLC, Carnegie Mellon +# University, West Virginia University Research Corporation, et al. +# All rights reserved. Please see the files COPYRIGHT.md and LICENSE.md +# for full copyright and license information. +################################################################################# + +import pyomo.common.unittest as unittest +import pytest +from gtep.gtep_data import ExpansionPlanningData + + +class TestExpansionPlanningData(unittest.TestCase): + def setup(): + pass + + def test_data_init(self): + # Test that the ExpansionPlanningData object initializes properly with default values + testObject = ExpansionPlanningData() + self.assertIsInstance(testObject, ExpansionPlanningData) + self.assertEqual(testObject.stages, 2) + self.assertEqual(testObject.num_reps, 4) + self.assertEqual(testObject.len_reps, 1) + self.assertEqual(testObject.num_commit, 24) + self.assertEqual(testObject.num_dispatch, 1) + self.assertEqual(testObject.duration_dispatch, 60) + + # Test that the ExpansionPlanningData object initializes properly with input values + testObject = ExpansionPlanningData(1, 2, 2, 2, 2, 15) + self.assertIsInstance(testObject, ExpansionPlanningData) + self.assertEqual(testObject.stages, 1) + self.assertEqual(testObject.num_reps, 2) + self.assertEqual(testObject.len_reps, 2) + self.assertEqual(testObject.num_commit, 2) + self.assertEqual(testObject.num_dispatch, 2) + self.assertEqual(testObject.duration_dispatch, 15) + + # Test that the ExpansionPlanningData object initializes properly with partial input values + testObject = ExpansionPlanningData(duration_dispatch=15) + self.assertIsInstance(testObject, ExpansionPlanningData) + self.assertEqual(testObject.stages, 2) + self.assertEqual(testObject.num_reps, 4) + self.assertEqual(testObject.len_reps, 1) + self.assertEqual(testObject.num_commit, 24) + self.assertEqual(testObject.num_dispatch, 1) + self.assertEqual(testObject.duration_dispatch, 15) From 3caa4c12ea7296a06848e12e0461b8834aa02d39 Mon Sep 17 00:00:00 2001 From: bstorm Date: Mon, 6 Apr 2026 13:53:01 -0600 Subject: [PATCH 05/24] Add load_prescient unit tests --- gtep/gtep_data.py | 3 +- gtep/tests/unit/test_gtep_data.py | 389 ++++++++++++++++++++++++++++-- 2 files changed, 367 insertions(+), 25 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 61bb624b..b7ec926b 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -147,6 +147,7 @@ def load_prescient( # Arbitrary time points and lengths picked for representative periods # default here allows up to 24 hours for periods self.representative_dates = representative_dates + self.representative_weights = representative_weights if not representative_weights: # set the weight for each day to the total weight divided by number of days @@ -154,7 +155,7 @@ def load_prescient( weight_per_date = int(total_weight / (len(representative_dates))) self.representative_weights = { key: weight_per_date - for date, key in enumerate(self.representative_dates) + for key, date in enumerate(self.representative_dates) } time_keys = self.md.data["system"]["time_keys"] diff --git a/gtep/tests/unit/test_gtep_data.py b/gtep/tests/unit/test_gtep_data.py index a847c0fc..1224c8c0 100644 --- a/gtep/tests/unit/test_gtep_data.py +++ b/gtep/tests/unit/test_gtep_data.py @@ -11,42 +11,383 @@ # for full copyright and license information. ################################################################################# + import pyomo.common.unittest as unittest +from unittest.mock import create_autospec import pytest from gtep.gtep_data import ExpansionPlanningData +import prescient.simulator.config +import pandas as pd + + +@pytest.fixture +def simulation_objects_csv(tmp_path): + # Create a minimal simulation_objects.csv file with expected content + csv_content = """index,DAY_AHEAD +Periods_per_Step,24 +""" + csv_file = tmp_path / "simulation_objects.csv" + csv_file.write_text(csv_content) + load_file = tmp_path / "storage.csv" + load_file.write_text( + """name,bus,generator,storage_type,energy_capacity,initial_state_of_charge,end_state_of_charge,minimum_state_of_charge,charge_efficiency,discharge_efficiency,max_discharge_rate,min_discharge_rate,max_charge_rate,min_charge_rate,initial_charge_rate,initial_discharge_rate,charge_cost,discharge_cost,retention_rate_60min,ramp_up_input_60min,ramp_down_input_60min,ramp_up_output_60min,ramp_down_output_60min,in_service,capital_multiplier,extension_multiplier,investment_cost,investment_cost_kwh +100MW_400MWh_1,3,None,'battery',400,320,320,80,1,1,100,10,400,1,0,0,175,0,1,400,400,100,100,FALSE,1,1,1490.89,372.7225""" + ) + return tmp_path + + +@pytest.fixture +def mock_prescient_config(): + mock_instance = create_autospec( + prescient.simulator.config.PrescientConfig, instance=True + ) + mock_instance.set_value.return_value = None + mock_instance.num_days = 365 + mock_instance.sced_frequency_minutes = 60 + with unittest.mock.patch( + "gtep.gtep_data.PrescientConfig", return_value=mock_instance + ): + yield mock_instance -class TestExpansionPlanningData(unittest.TestCase): - def setup(): - pass + +@pytest.fixture +def mock_gmlc_provider(): + with unittest.mock.patch( + "prescient.data.providers.gmlc_data_provider.GmlcDataProvider" + ) as MockProvider: + instance = MockProvider.return_value + instance._start_time = "2020-01-01 00:00" + # Mock get_initial_actuals_model to return a model with .data attribute + model_mock = unittest.mock.Mock() + model_mock.data = { + "elements": { + "generator": {"gen1": {}, "gen2-c": {}}, + "branch": {"branch1": {}, "branch2-c": {}}, + "storage": {"stor1": {}}, + }, + "system": { + "time_keys": [ + "2020-01-28 00:00", + "2020-01-28 01:00", + "2020-01-28 02:00", + "2020-04-23 00:00", + "2020-07-05 00:00", + "2020-10-14 00:00", + "2020-12-14 00:00", + ] + }, + } + model_mock.clone_at_time_keys.side_effect = lambda keys: f"clone_{keys}" + instance.get_initial_actuals_model.return_value = model_mock + instance.populate_with_actuals.return_value = None + yield instance + + +@pytest.mark.usefixtures("mock_prescient_config", "mock_gmlc_provider") +class TestExpansionPlanningData: def test_data_init(self): # Test that the ExpansionPlanningData object initializes properly with default values testObject = ExpansionPlanningData() - self.assertIsInstance(testObject, ExpansionPlanningData) - self.assertEqual(testObject.stages, 2) - self.assertEqual(testObject.num_reps, 4) - self.assertEqual(testObject.len_reps, 1) - self.assertEqual(testObject.num_commit, 24) - self.assertEqual(testObject.num_dispatch, 1) - self.assertEqual(testObject.duration_dispatch, 60) + assert isinstance(testObject, ExpansionPlanningData) + assert testObject.stages == 2 + assert testObject.num_reps == 4 + assert testObject.len_reps == 1 + assert testObject.num_commit == 24 + assert testObject.num_dispatch == 1 + assert testObject.duration_dispatch == 60 # Test that the ExpansionPlanningData object initializes properly with input values testObject = ExpansionPlanningData(1, 2, 2, 2, 2, 15) - self.assertIsInstance(testObject, ExpansionPlanningData) - self.assertEqual(testObject.stages, 1) - self.assertEqual(testObject.num_reps, 2) - self.assertEqual(testObject.len_reps, 2) - self.assertEqual(testObject.num_commit, 2) - self.assertEqual(testObject.num_dispatch, 2) - self.assertEqual(testObject.duration_dispatch, 15) + assert isinstance(testObject, ExpansionPlanningData) + assert testObject.stages == 1 + assert testObject.num_reps == 2 + assert testObject.len_reps == 2 + assert testObject.num_commit == 2 + assert testObject.num_dispatch == 2 + assert testObject.duration_dispatch == 15 # Test that the ExpansionPlanningData object initializes properly with partial input values testObject = ExpansionPlanningData(duration_dispatch=15) - self.assertIsInstance(testObject, ExpansionPlanningData) - self.assertEqual(testObject.stages, 2) - self.assertEqual(testObject.num_reps, 4) - self.assertEqual(testObject.len_reps, 1) - self.assertEqual(testObject.num_commit, 24) - self.assertEqual(testObject.num_dispatch, 1) - self.assertEqual(testObject.duration_dispatch, 15) + assert isinstance(testObject, ExpansionPlanningData) + assert testObject.stages == 2 + assert testObject.num_reps == 4 + assert testObject.len_reps == 1 + assert testObject.num_commit == 24 + assert testObject.num_dispatch == 1 + assert testObject.duration_dispatch == 15 + + # -------------------------------------------------LOAD_PRESCIENT------------------------------------------------------------ # + def test_options_dict( + self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + ): + # Test passing in an options dictionary + testObject = ExpansionPlanningData() + # set options that are not the defaults + options = { + "num_days": 100, + "ruc_horizon": 12, + } + + testObject.load_prescient( + data_path=str(simulation_objects_csv), options_dict=options + ) + mock_prescient_config.set_value.assert_called_once() + passed_dict = mock_prescient_config.set_value.call_args[0][0] + assert passed_dict["data_path"] == str(simulation_objects_csv) + assert passed_dict["num_days"] == 100 + assert passed_dict["ruc_horizon"] == 12 + + def test_no_options_dict( + self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + ): + + # Test not passing in an options dictionary + testObject = ExpansionPlanningData() + testObject.load_prescient(data_path=str(simulation_objects_csv)) + mock_prescient_config.set_value.assert_called_once() + passed_dict = mock_prescient_config.set_value.call_args[0][0] + # Default options should be set + assert passed_dict["data_path"] == str(simulation_objects_csv) + assert passed_dict["num_days"] == 365 + assert passed_dict["ruc_horizon"] == 36 + + def test_default_representative_dates( + self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + ): + # Test no representative dates passed in, initializing with defaults + testObject = ExpansionPlanningData() + testObject.load_prescient(data_path=str(simulation_objects_csv)) + # default dates: + expected_dates = [ + "2020-01-28 00:00", + "2020-04-23 00:00", + "2020-07-05 00:00", + "2020-10-14 00:00", + ] + assert testObject.representative_dates == expected_dates + + def test_passed_representative_dates( + self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + ): + # Test new representative dates passed in, replacing defaults + testObject = ExpansionPlanningData() + testObject.load_prescient( + data_path=str(simulation_objects_csv), + representative_dates=[ + "2020-01-28 00:00", + "2020-04-23 00:00", + "2020-07-05 00:00", + "2020-12-14 00:00", + ], + ) + + expected_dates = [ + "2020-01-28 00:00", + "2020-04-23 00:00", + "2020-07-05 00:00", + "2020-12-14 00:00", + ] + assert testObject.representative_dates == expected_dates + + def test_representative_date_not_in_time_keys( + self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + ): + # Test passing invalid/not covered dates + testObject = ExpansionPlanningData() + bad_dates = [ + "2020-01-28 00:00", + "2020-04-23 00:00", + "2020-07-05 00:00", + "2099-01-01 00:00", # Not in time_keys + ] + with pytest.raises(ValueError): + testObject.load_prescient( + data_path=str(simulation_objects_csv), representative_dates=bad_dates + ) + + def test_empty_representative_dates( + self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + ): + # Test an empty list passed to overwrite defaults without setting new dates + testObject = ExpansionPlanningData() + with pytest.raises(ZeroDivisionError): + testObject.load_prescient( + data_path=str(simulation_objects_csv), representative_dates=[] + ) + + def test_invalid_representative_dates_type( + self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + ): + # Test setting None for the dates, overwriting defaults without adding new dates + testObject = ExpansionPlanningData() + with pytest.raises(TypeError): + testObject.load_prescient( + data_path=str(simulation_objects_csv), representative_dates=None + ) + + def test_default_representative_weights( + self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + ): + # Test no representative weights passed in, so the function calculates them + testObject = ExpansionPlanningData() + testObject.load_prescient(data_path=str(simulation_objects_csv)) + + total_weight = mock_prescient_config.num_days * testObject.stages + assert total_weight == (365 * 2) + expected_weight = int(total_weight / len(testObject.representative_dates)) + assert expected_weight == int((365 * 2) / 4) + + for w in testObject.representative_weights.values(): + assert w == expected_weight + + def test_no_representative_weights_passed_5_dates( + self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + ): + # Test no representative weights passed in and custom dates passed in, so the function calculates them + dates = [ + "2020-01-28 00:00", + "2020-04-23 00:00", + "2020-07-05 00:00", + "2020-10-14 00:00", + "2020-12-14 00:00", + ] + + testObject = ExpansionPlanningData() + testObject.load_prescient( + data_path=str(simulation_objects_csv), representative_dates=dates + ) + + total_weight = mock_prescient_config.num_days * testObject.stages + assert total_weight == (365 * 2) + expected_weight = int(total_weight / len(testObject.representative_dates)) + assert expected_weight == int((365 * 2) / 5) + + for w in testObject.representative_weights.values(): + assert w == expected_weight + + def test_passed_representative_weights( + self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + ): + # Test representative weights passed in + weights = {1: 91, 2: 91, 3: 91, 4: 91} + testObject = ExpansionPlanningData() + testObject.load_prescient( + data_path=str(simulation_objects_csv), representative_weights=weights + ) + assert testObject.representative_weights == weights + + def test_key_function_calls( + self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + ): + # Test that each function is called within the load_prescient function + testObject = ExpansionPlanningData() + + # Patch internal methods to track calls + with ( + unittest.mock.patch.object( + testObject, "load_default_data_settings" + ) as mock_load_defaults, + unittest.mock.patch.object( + testObject, "load_storage_csv" + ) as mock_load_storage, + ): + + testObject.load_prescient(data_path=str(simulation_objects_csv)) + + # External calls + mock_prescient_config.set_value.assert_called_once() + mock_gmlc_provider.get_initial_actuals_model.assert_called_once() + mock_gmlc_provider.populate_with_actuals.assert_called_once() + + # Internal calls + mock_load_defaults.assert_called_once() + mock_load_storage.assert_called_once_with(str(simulation_objects_csv)) + + def test_total_num_steps_calculation( + self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + ): + # Check that the total number of time steps is calculated based in the number of days in the config + mock_prescient_config.num_days = 365 + + testObject = ExpansionPlanningData() + testObject.load_prescient(data_path=str(simulation_objects_csv)) + + # period_per_step is 24 + expected_total_steps = 365 * 24 + + mock_gmlc_provider.get_initial_actuals_model.assert_called_once() + call_args = mock_gmlc_provider.get_initial_actuals_model.call_args[1] + assert call_args["num_time_steps"] == expected_total_steps + + def test_total_num_steps_calculation_2_years( + self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + ): + # Check that the total number of time steps is calculated based in the number of days in the config + mock_prescient_config.num_days = 730 + + testObject = ExpansionPlanningData() + testObject.load_prescient(data_path=str(simulation_objects_csv)) + + # period_per_step is 24 + expected_total_steps = 730 * 24 + + mock_gmlc_provider.get_initial_actuals_model.assert_called_once() + call_args = mock_gmlc_provider.get_initial_actuals_model.call_args[1] + assert call_args["num_time_steps"] == expected_total_steps + + def test_in_service_flags_set( + self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + ): + # Test in service flags are being set properly + testObject = ExpansionPlanningData() + testObject.load_prescient(data_path=str(simulation_objects_csv)) + + assert ( + testObject.md.data["elements"]["generator"]["gen2-c"]["in_service"] == False + ) + assert ( + testObject.md.data["elements"]["branch"]["branch2-c"]["in_service"] == False + ) + + def test_missing_simulation_objects_csv( + self, mock_prescient_config, mock_gmlc_provider, tmp_path + ): + # test if a data path is missing the simulation objects, it should throw an error + testObject = ExpansionPlanningData() + # tmp_path is empty, no simulation_objects.csv + with pytest.raises(FileNotFoundError): + testObject.load_prescient(data_path=str(tmp_path)) + + def test_incorrect_simulation_objects_csv( + self, mock_prescient_config, mock_gmlc_provider, tmp_path + ): + # test if simulations_objects is missing key details, it should throw an error + csv_file = tmp_path / "simulation_objects.csv" + csv_file.write_text("index,DAY_AHEAD\nWrongKey,24\n") + + testObject = ExpansionPlanningData() + with pytest.raises(KeyError): + testObject.load_prescient(data_path=str(tmp_path)) + + def test_clone_at_time_keys_called_correctly( + self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + ): + testObject = ExpansionPlanningData() + testObject.load_prescient(data_path=str(simulation_objects_csv)) + + # Patch clone_at_time_keys on the existing model instance + with unittest.mock.patch.object( + testObject.md, "clone_at_time_keys", wraps=testObject.md.clone_at_time_keys + ) as mock_clone: + + # Call load_prescient again to trigger cloning with patched method + testObject.load_prescient(data_path=str(simulation_objects_csv)) + + # Check that clone_at_time_keys was called once per representative date + expected_calls = len(testObject.representative_dates) + assert mock_clone.call_count == expected_calls + + +# -------------------------------------------------IMPORT_LOAD_SCALING------------------------------------------------------------ # From 8426aefe4712d3e0a966db7aeb4d0c8a5a253367 Mon Sep 17 00:00:00 2001 From: bstorm Date: Mon, 6 Apr 2026 14:26:01 -0600 Subject: [PATCH 06/24] Add import_load_scaling tests --- gtep/gtep_data.py | 3 +- gtep/tests/unit/test_gtep_data.py | 112 ++++++++++++++++++++---------- 2 files changed, 76 insertions(+), 39 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index b7ec926b..b23d635f 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -175,6 +175,7 @@ def import_load_scaling(self, load_file_name, forecast_years=[2025, 2030, 2035]) """ adjusted_forecast = pd.read_excel(load_file_name) + print((adjusted_forecast["year"]).unique) # check years are valid if len(forecast_years) < self.stages: @@ -188,7 +189,7 @@ def import_load_scaling(self, load_file_name, forecast_years=[2025, 2030, 2035]) adjusted_forecast_by_period = adjusted_forecast[ adjusted_forecast["year"].isin(forecast_years) - ] + ].copy() base_zones = [ "base_economic_coast", diff --git a/gtep/tests/unit/test_gtep_data.py b/gtep/tests/unit/test_gtep_data.py index 1224c8c0..0a0c75f0 100644 --- a/gtep/tests/unit/test_gtep_data.py +++ b/gtep/tests/unit/test_gtep_data.py @@ -18,10 +18,11 @@ from gtep.gtep_data import ExpansionPlanningData import prescient.simulator.config import pandas as pd +from pathlib import Path @pytest.fixture -def simulation_objects_csv(tmp_path): +def test_data_path(tmp_path): # Create a minimal simulation_objects.csv file with expected content csv_content = """index,DAY_AHEAD Periods_per_Step,24 @@ -36,6 +37,15 @@ def simulation_objects_csv(tmp_path): return tmp_path +@pytest.fixture +def actual_load_path(): + test_dir = Path(__file__).parent + # Navigate up to 'gtep' directory, then into 'data' + data_dir = test_dir.parent.parent / "data" + excel_path = data_dir / "Texas_2000" / "ERCOT-Adjusted-Forecast.xlsb" + return excel_path + + @pytest.fixture def mock_prescient_config(): mock_instance = create_autospec( @@ -120,7 +130,7 @@ def test_data_init(self): # -------------------------------------------------LOAD_PRESCIENT------------------------------------------------------------ # def test_options_dict( - self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + self, mock_prescient_config, mock_gmlc_provider, test_data_path ): # Test passing in an options dictionary testObject = ExpansionPlanningData() @@ -130,35 +140,33 @@ def test_options_dict( "ruc_horizon": 12, } - testObject.load_prescient( - data_path=str(simulation_objects_csv), options_dict=options - ) + testObject.load_prescient(data_path=str(test_data_path), options_dict=options) mock_prescient_config.set_value.assert_called_once() passed_dict = mock_prescient_config.set_value.call_args[0][0] - assert passed_dict["data_path"] == str(simulation_objects_csv) + assert passed_dict["data_path"] == str(test_data_path) assert passed_dict["num_days"] == 100 assert passed_dict["ruc_horizon"] == 12 def test_no_options_dict( - self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + self, mock_prescient_config, mock_gmlc_provider, test_data_path ): # Test not passing in an options dictionary testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=str(simulation_objects_csv)) + testObject.load_prescient(data_path=str(test_data_path)) mock_prescient_config.set_value.assert_called_once() passed_dict = mock_prescient_config.set_value.call_args[0][0] # Default options should be set - assert passed_dict["data_path"] == str(simulation_objects_csv) + assert passed_dict["data_path"] == str(test_data_path) assert passed_dict["num_days"] == 365 assert passed_dict["ruc_horizon"] == 36 def test_default_representative_dates( - self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + self, mock_prescient_config, mock_gmlc_provider, test_data_path ): # Test no representative dates passed in, initializing with defaults testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=str(simulation_objects_csv)) + testObject.load_prescient(data_path=str(test_data_path)) # default dates: expected_dates = [ "2020-01-28 00:00", @@ -169,12 +177,12 @@ def test_default_representative_dates( assert testObject.representative_dates == expected_dates def test_passed_representative_dates( - self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + self, mock_prescient_config, mock_gmlc_provider, test_data_path ): # Test new representative dates passed in, replacing defaults testObject = ExpansionPlanningData() testObject.load_prescient( - data_path=str(simulation_objects_csv), + data_path=str(test_data_path), representative_dates=[ "2020-01-28 00:00", "2020-04-23 00:00", @@ -192,7 +200,7 @@ def test_passed_representative_dates( assert testObject.representative_dates == expected_dates def test_representative_date_not_in_time_keys( - self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + self, mock_prescient_config, mock_gmlc_provider, test_data_path ): # Test passing invalid/not covered dates testObject = ExpansionPlanningData() @@ -204,35 +212,35 @@ def test_representative_date_not_in_time_keys( ] with pytest.raises(ValueError): testObject.load_prescient( - data_path=str(simulation_objects_csv), representative_dates=bad_dates + data_path=str(test_data_path), representative_dates=bad_dates ) def test_empty_representative_dates( - self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + self, mock_prescient_config, mock_gmlc_provider, test_data_path ): # Test an empty list passed to overwrite defaults without setting new dates testObject = ExpansionPlanningData() with pytest.raises(ZeroDivisionError): testObject.load_prescient( - data_path=str(simulation_objects_csv), representative_dates=[] + data_path=str(test_data_path), representative_dates=[] ) def test_invalid_representative_dates_type( - self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + self, mock_prescient_config, mock_gmlc_provider, test_data_path ): # Test setting None for the dates, overwriting defaults without adding new dates testObject = ExpansionPlanningData() with pytest.raises(TypeError): testObject.load_prescient( - data_path=str(simulation_objects_csv), representative_dates=None + data_path=str(test_data_path), representative_dates=None ) def test_default_representative_weights( - self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + self, mock_prescient_config, mock_gmlc_provider, test_data_path ): # Test no representative weights passed in, so the function calculates them testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=str(simulation_objects_csv)) + testObject.load_prescient(data_path=str(test_data_path)) total_weight = mock_prescient_config.num_days * testObject.stages assert total_weight == (365 * 2) @@ -243,7 +251,7 @@ def test_default_representative_weights( assert w == expected_weight def test_no_representative_weights_passed_5_dates( - self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + self, mock_prescient_config, mock_gmlc_provider, test_data_path ): # Test no representative weights passed in and custom dates passed in, so the function calculates them dates = [ @@ -256,7 +264,7 @@ def test_no_representative_weights_passed_5_dates( testObject = ExpansionPlanningData() testObject.load_prescient( - data_path=str(simulation_objects_csv), representative_dates=dates + data_path=str(test_data_path), representative_dates=dates ) total_weight = mock_prescient_config.num_days * testObject.stages @@ -268,18 +276,18 @@ def test_no_representative_weights_passed_5_dates( assert w == expected_weight def test_passed_representative_weights( - self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + self, mock_prescient_config, mock_gmlc_provider, test_data_path ): # Test representative weights passed in weights = {1: 91, 2: 91, 3: 91, 4: 91} testObject = ExpansionPlanningData() testObject.load_prescient( - data_path=str(simulation_objects_csv), representative_weights=weights + data_path=str(test_data_path), representative_weights=weights ) assert testObject.representative_weights == weights def test_key_function_calls( - self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + self, mock_prescient_config, mock_gmlc_provider, test_data_path ): # Test that each function is called within the load_prescient function testObject = ExpansionPlanningData() @@ -294,7 +302,7 @@ def test_key_function_calls( ) as mock_load_storage, ): - testObject.load_prescient(data_path=str(simulation_objects_csv)) + testObject.load_prescient(data_path=str(test_data_path)) # External calls mock_prescient_config.set_value.assert_called_once() @@ -303,16 +311,16 @@ def test_key_function_calls( # Internal calls mock_load_defaults.assert_called_once() - mock_load_storage.assert_called_once_with(str(simulation_objects_csv)) + mock_load_storage.assert_called_once_with(str(test_data_path)) def test_total_num_steps_calculation( - self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + self, mock_prescient_config, mock_gmlc_provider, test_data_path ): # Check that the total number of time steps is calculated based in the number of days in the config mock_prescient_config.num_days = 365 testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=str(simulation_objects_csv)) + testObject.load_prescient(data_path=str(test_data_path)) # period_per_step is 24 expected_total_steps = 365 * 24 @@ -322,13 +330,13 @@ def test_total_num_steps_calculation( assert call_args["num_time_steps"] == expected_total_steps def test_total_num_steps_calculation_2_years( - self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + self, mock_prescient_config, mock_gmlc_provider, test_data_path ): # Check that the total number of time steps is calculated based in the number of days in the config mock_prescient_config.num_days = 730 testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=str(simulation_objects_csv)) + testObject.load_prescient(data_path=str(test_data_path)) # period_per_step is 24 expected_total_steps = 730 * 24 @@ -338,11 +346,11 @@ def test_total_num_steps_calculation_2_years( assert call_args["num_time_steps"] == expected_total_steps def test_in_service_flags_set( - self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + self, mock_prescient_config, mock_gmlc_provider, test_data_path ): # Test in service flags are being set properly testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=str(simulation_objects_csv)) + testObject.load_prescient(data_path=str(test_data_path)) assert ( testObject.md.data["elements"]["generator"]["gen2-c"]["in_service"] == False @@ -372,10 +380,10 @@ def test_incorrect_simulation_objects_csv( testObject.load_prescient(data_path=str(tmp_path)) def test_clone_at_time_keys_called_correctly( - self, mock_prescient_config, mock_gmlc_provider, simulation_objects_csv + self, mock_prescient_config, mock_gmlc_provider, test_data_path ): testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=str(simulation_objects_csv)) + testObject.load_prescient(data_path=str(test_data_path)) # Patch clone_at_time_keys on the existing model instance with unittest.mock.patch.object( @@ -383,11 +391,39 @@ def test_clone_at_time_keys_called_correctly( ) as mock_clone: # Call load_prescient again to trigger cloning with patched method - testObject.load_prescient(data_path=str(simulation_objects_csv)) + testObject.load_prescient(data_path=str(test_data_path)) # Check that clone_at_time_keys was called once per representative date expected_calls = len(testObject.representative_dates) assert mock_clone.call_count == expected_calls + # -------------------------------------------------IMPORT_LOAD_SCALING------------------------------------------------------------ # + def test_import_load_scaling_normal_with_actual_file(self, actual_load_path): + # test successful passthrough of load scaling function + testObject = ExpansionPlanningData() + testObject.import_load_scaling(actual_load_path) -# -------------------------------------------------IMPORT_LOAD_SCALING------------------------------------------------------------ # + df = testObject.load_scaling + assert isinstance(df, pd.DataFrame) + expected_columns = ["year", "month", "day", "hour"] + [ + str(i) for i in range(1, 9) + ] + for col in expected_columns: + assert col in df.columns + assert not df.empty + + def test_import_load_scaling_incorrect_num_years(self, actual_load_path): + # Test value error raised if the length of forecast years is incorrect + testObject = ExpansionPlanningData(stages=3) + forecast_years = [2025, 2030] + + with pytest.raises(ValueError): + testObject.import_load_scaling(actual_load_path, forecast_years) + + def test_import_load_scaling_incorrect_years_too_early(self, actual_load_path): + # Test value error raised if the forecast years are outside the supported ranges + testObject = ExpansionPlanningData(stages=3) + forecast_years = [2019, 2030, 2055] + + with pytest.raises(ValueError): + testObject.import_load_scaling(actual_load_path, forecast_years) From 96655e598107270b4a447d36992cfbb2625f94d0 Mon Sep 17 00:00:00 2001 From: bstorm Date: Mon, 6 Apr 2026 15:05:04 -0600 Subject: [PATCH 07/24] add import_outage_data tests --- gtep/tests/unit/test_gtep_data.py | 92 +++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/gtep/tests/unit/test_gtep_data.py b/gtep/tests/unit/test_gtep_data.py index 0a0c75f0..b063a3f8 100644 --- a/gtep/tests/unit/test_gtep_data.py +++ b/gtep/tests/unit/test_gtep_data.py @@ -21,6 +21,7 @@ from pathlib import Path +# Data Path Fixture (Load Prescient) @pytest.fixture def test_data_path(tmp_path): # Create a minimal simulation_objects.csv file with expected content @@ -37,6 +38,7 @@ def test_data_path(tmp_path): return tmp_path +# Load Data File Fixture (Import Load) @pytest.fixture def actual_load_path(): test_dir = Path(__file__).parent @@ -46,6 +48,58 @@ def actual_load_path(): return excel_path +# Outage Data File Fixtures (Import Outages) +@pytest.fixture +def outage_data(tmp_path): + outage_content = """fips_code,lim_timestamp,case_4b_prob,date +48001,5/20/25 0:00,0,20-May +48025,5/20/25 3:00,0,20-May +48025,5/20/25 4:00,0,20-May +48025,5/20/25 5:00,0,20-May +48025,5/20/25 6:00,0,20-May +48025,5/20/25 7:00,0,20-May +48025,5/20/25 8:00,0,20-May +48025,5/20/25 9:00,0,20-May +48025,5/20/25 10:00,0,20-May +48027,5/20/25 20:00,0,20-May +48163,5/20/25 20:00,0.006944444,20-May""" + outage_csv = tmp_path / "outage.csv" + outage_csv.write_text(outage_content) + + return outage_csv + + +@pytest.fixture +def county_fips_csv(tmp_path): + dir_path = tmp_path / "gtep" / "data" / "123_Bus_Resil_Week" + dir_path.mkdir(parents=True, exist_ok=True) + content = """county_number,County,FIPS,, +1,Anderson,48001,, +13,Bee,48025,, +27,Burnet,48053,, +82,Frio,48163,, +""" + file = dir_path / "county_fips_match.csv" + file.write_text(content) + return file + + +@pytest.fixture +def bus_data_csv(tmp_path): + dir_path = tmp_path / "gtep" / "data" / "123_Bus_Resil_Week" + dir_path.mkdir(parents=True, exist_ok=True) + content = """Bus Number,County +1001,Anderson +1002,Bee +1003,Burnet +1004,Frio +""" + file = dir_path / "Bus_data_gen_weights_mappings.csv" + file.write_text(content) + return file + + +# Function Mocks (Load Prescient) @pytest.fixture def mock_prescient_config(): mock_instance = create_autospec( @@ -427,3 +481,41 @@ def test_import_load_scaling_incorrect_years_too_early(self, actual_load_path): with pytest.raises(ValueError): testObject.import_load_scaling(actual_load_path, forecast_years) + + # -------------------------------------------------IMPORT_OUTAGE_DATA------------------------------------------------------------ # + def test_import_outage_data( + self, outage_data, county_fips_csv, bus_data_csv, monkeypatch + ): + testObject = ExpansionPlanningData() + + # Load DataFrames from temp CSV files for patching + df_outage = pd.read_csv(outage_data) + df_county_fips = pd.read_csv(county_fips_csv) + df_bus_data = pd.read_csv(bus_data_csv) + + # Patch pd.read_csv to return appropriate DataFrame based on input path + def mock_read_csv(filepath, *args, **kwargs): + if filepath == str(outage_data): + return df_outage + elif filepath.endswith("county_fips_match.csv"): + return df_county_fips + elif filepath.endswith("Bus_data_gen_weights_mappings.csv"): + return df_bus_data + else: + raise FileNotFoundError(f"Unexpected file path: {filepath}") + + monkeypatch.setattr(pd, "read_csv", mock_read_csv) + + # Patch to_csv to avoid actual file writes during test + monkeypatch.setattr(pd.DataFrame, "to_csv", lambda self, *args, **kwargs: None) + + testObject.import_outage_data(str(outage_data)) + + assert hasattr(testObject, "bus_hours") + df = testObject.bus_hours + assert isinstance(df, pd.DataFrame) + assert set(df.columns) == {"hour", "Bus Number"} + assert pd.api.types.is_integer_dtype(df["hour"]) + assert pd.api.types.is_integer_dtype(df["Bus Number"]) + assert (df["hour"] == 20).any() + assert (df["Bus Number"] == 1004).any() From 7283093d230b38a5d7b1be1642f2ebdf16104feb Mon Sep 17 00:00:00 2001 From: bstorm Date: Mon, 6 Apr 2026 15:19:54 -0600 Subject: [PATCH 08/24] add default data settings test --- gtep/tests/unit/test_gtep_data.py | 49 ++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/gtep/tests/unit/test_gtep_data.py b/gtep/tests/unit/test_gtep_data.py index b063a3f8..fb5c3808 100644 --- a/gtep/tests/unit/test_gtep_data.py +++ b/gtep/tests/unit/test_gtep_data.py @@ -126,7 +126,10 @@ def mock_gmlc_provider(): model_mock = unittest.mock.Mock() model_mock.data = { "elements": { - "generator": {"gen1": {}, "gen2-c": {}}, + "generator": { + "gen1": {"fuel": "C", "in_service": True}, + "gen2-c": {"fuel": "None", "in_service": True}, + }, "branch": {"branch1": {}, "branch2-c": {}}, "storage": {"stor1": {}}, }, @@ -519,3 +522,47 @@ def mock_read_csv(filepath, *args, **kwargs): assert pd.api.types.is_integer_dtype(df["Bus Number"]) assert (df["hour"] == 20).any() assert (df["Bus Number"] == 1004).any() + + # -------------------------------------------------LOAD_DEFAULT_DATA_SETTINGS------------------------------------------------------------ # + def test_load_default_data_settings( + self, mock_prescient_config, mock_gmlc_provider, test_data_path + ): + testObject = ExpansionPlanningData() + testObject.load_prescient(data_path=str(test_data_path)) + + testObject.load_default_data_settings() + + # Check generators + for gen_name, gen in testObject.md.data["elements"]["generator"].items(): + if gen.get("fuel") == "C": + if gen.get("in_service") is False: + assert gen["lifetime"] == 1 + else: + assert gen["lifetime"] == 2 + else: + assert gen["lifetime"] == 3 + + # Check other fixed attributes + assert gen["spinning_reserve_frac"] == 0.1 + assert gen["quickstart_reserve_frac"] == 0.1 + assert gen["capital_multiplier"] == 1 + assert gen["extension_multiplier"] == 0 + assert gen["max_operating_reserve"] == 1 + assert gen["max_spinning_reserve"] == 1 + assert gen["max_quickstart_reserve"] == 1 + assert gen["ramp_up_rate"] == 0.1 + assert gen["ramp_down_rate"] == 0.1 + assert gen["emissions_factor"] == 1 + assert gen["start_fuel"] == 1 + assert gen["investment_cost"] == 1 + + # Check branches + for branch in testObject.md.data["elements"]["branch"].values(): + assert branch["loss_rate"] == 0 + assert branch["distance"] == 1 + assert branch["capital_cost"] == 10000000 + + # Check system + system = testObject.md.data["system"] + assert system["min_operating_reserve"] == 0.1 + assert system["min_spinning_reserve"] == 0.1 From a590f835086b42305b347363fb2ea2b5431e9735 Mon Sep 17 00:00:00 2001 From: bstorm Date: Mon, 6 Apr 2026 15:24:55 -0600 Subject: [PATCH 09/24] add load_storage_csv unit tests --- gtep/tests/unit/test_gtep_data.py | 51 +++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/gtep/tests/unit/test_gtep_data.py b/gtep/tests/unit/test_gtep_data.py index fb5c3808..d98bcbb5 100644 --- a/gtep/tests/unit/test_gtep_data.py +++ b/gtep/tests/unit/test_gtep_data.py @@ -566,3 +566,54 @@ def test_load_default_data_settings( system = testObject.md.data["system"] assert system["min_operating_reserve"] == 0.1 assert system["min_spinning_reserve"] == 0.1 + + # -------------------------------------------------LOAD_STORAGE_CSV------------------------------------------------------------ # + def test_load_storage_csv_success( + self, test_data_path, mock_gmlc_provider, mock_prescient_config + ): + testObject = ExpansionPlanningData() + # Setup md with empty data structure to avoid errors + testObject.md = mock_gmlc_provider.get_initial_actuals_model.return_value + testObject.load_storage_csv(str(test_data_path)) + + # Check that storage data was loaded into md.data["elements"]["storage"] + storage = testObject.md.data["elements"].get("storage", None) + assert storage is not None + assert isinstance(storage, dict) + + # Check that storage keys include the name from your CSV fixture + assert "100MW_400MWh_1" in storage + + # Check some expected keys in storage data + expected_keys = { + "bus", + "generator", + "storage_type", + "energy_capacity", + "initial_state_of_charge", + "investment_cost", + "investment_cost_kwh", + } + for key in expected_keys: + assert key in storage["100MW_400MWh_1"] + + def test_load_storage_csv_file_not_found( + self, tmp_path, mock_gmlc_provider, mock_prescient_config + ): + testObject = ExpansionPlanningData() + testObject.md = mock_gmlc_provider.get_initial_actuals_model.return_value + + # Use a directory without storage.csv + missing_path = tmp_path + + with unittest.mock.patch("builtins.print") as mock_print: + testObject.load_storage_csv(str(missing_path)) + + # Should print warning about missing file + mock_print.assert_called_once() + args = mock_print.call_args[0][0] + assert "does not exist" in args + + # Storage should be set to empty dict + storage = testObject.md.data["elements"].get("storage", None) + assert storage == {} From 159ecbfcb4c9c96cbc9e7ddd8dd8f0d5b571514e Mon Sep 17 00:00:00 2001 From: bstorm Date: Mon, 6 Apr 2026 16:22:44 -0600 Subject: [PATCH 10/24] work on texas_case_study tests --- gtep/gtep_data.py | 2 +- gtep/tests/unit/test_gtep_data.py | 75 ++++++++++++++++++++++++++++--- 2 files changed, 71 insertions(+), 6 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index b23d635f..6020eed3 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -360,7 +360,7 @@ def texas_case_study_updates(self, data_path): :param data_path: filepath for generator data csv file """ # check that datapath is coming from a texas case study directory - if "Texas" or "Coal" not in data_path: + if ("Texas" not in data_path) and ("Coal" not in data_path): raise ValueError("The data path provided is not a Texas case study") generator_update_path = data_path + "/gen.csv" diff --git a/gtep/tests/unit/test_gtep_data.py b/gtep/tests/unit/test_gtep_data.py index d98bcbb5..6f5a7250 100644 --- a/gtep/tests/unit/test_gtep_data.py +++ b/gtep/tests/unit/test_gtep_data.py @@ -38,6 +38,23 @@ def test_data_path(tmp_path): return tmp_path +# Texas Data Path Fixture (texas_case_study) +@pytest.fixture +def texas_data_path(tmp_path): + csv_content = """GEN UID,Bus ID,Unit Type,Fuel,capex1,capex2,capex3,fuel_cost1,fuel_cost2,fuel_cost3,fixed_ops1,fixed_ops2,fixed_ops3,var_ops1,var_ops2,var_ops3 +1,107,NUC,N,7040.489322,6966.178959,6731.224251,7.210946358,7.294679868,7.378403404,145.96,145.96,145.96,2.84,2.84,2.84 +2,100,CT,G,958.3056411,928.7012334,908.9649616,21.62404,18.03036,22.80128,26.87,26.87,26.87,1.76,1.76,1.76 +3,100,CT,G,958.3056411,928.7012334,908.9649616,21.62404,18.03036,22.80128,26.87,26.87,26.87,1.76,1.76,1.76 +4,100,CT,G,958.3056411,928.7012334,908.9649616,21.62404,18.03036,22.80128,26.87,26.87,26.87,1.76,1.76,1.76 +""" + dir_path = tmp_path / "Texas_2000" + dir_path.mkdir(parents=True, exist_ok=True) + excel_path = dir_path / "gen.csv" + excel_path.write_text(csv_content) + + return dir_path + + # Load Data File Fixture (Import Load) @pytest.fixture def actual_load_path(): @@ -127,8 +144,9 @@ def mock_gmlc_provider(): model_mock.data = { "elements": { "generator": { - "gen1": {"fuel": "C", "in_service": True}, - "gen2-c": {"fuel": "None", "in_service": True}, + "1": {"fuel": "C", "in_service": True}, + "2": {"fuel": "None", "in_service": True}, + "2-c": {"fuel": "None", "in_service": True}, }, "branch": {"branch1": {}, "branch2-c": {}}, "storage": {"stor1": {}}, @@ -409,9 +427,7 @@ def test_in_service_flags_set( testObject = ExpansionPlanningData() testObject.load_prescient(data_path=str(test_data_path)) - assert ( - testObject.md.data["elements"]["generator"]["gen2-c"]["in_service"] == False - ) + assert testObject.md.data["elements"]["generator"]["2-c"]["in_service"] == False assert ( testObject.md.data["elements"]["branch"]["branch2-c"]["in_service"] == False ) @@ -617,3 +633,52 @@ def test_load_storage_csv_file_not_found( # Storage should be set to empty dict storage = testObject.md.data["elements"].get("storage", None) assert storage == {} + + # -------------------------------------------------TEXAS_CASE_STUDY_UPDATES----------------------------------------------------------- # + def test_texas_case_study( + self, mock_gmlc_provider, mock_prescient_config, test_data_path, texas_data_path + ): + testObject = ExpansionPlanningData() + testObject.load_prescient(data_path=str(test_data_path)) + + # testObject.representative_data = [unittest.mock.Mock()] + # testObject.representative_data[0].data = testObject.md.data + + # Call the method under test + testObject.texas_case_study_updates(str(texas_data_path)) + + generator = testObject.md.data["elements"].get("generator", None) + assert generator is not None + + expected_columns = [ + "capex1", + "capex2", + "capex3", + "fuel_cost1", + "fuel_cost2", + "fuel_cost3", + "fixed_ops1", + "fixed_ops2", + "fixed_ops3", + "var_ops1", + "var_ops2", + "var_ops3", + ] + + # Check that each expected column is added to each generator + for gen_name, gen_data in generator.items(): + for col in expected_columns: + assert col in gen_data, f"Column {col} missing in generator {gen_name}" + # check example values + gen1 = generator.get("1") + assert gen1 is not None + assert abs(gen1["capex1"] - 7040.489322) < 1e-6 + assert abs(gen1["fuel_cost1"] - 7.210946358) < 1e-6 + + def test_texas_case_study_invalid_data_path(self): + # Test that an error is raised if not a Texas case Study + testObject = ExpansionPlanningData() + invalid_path = "/some/invalid/path" + + with pytest.raises(ValueError, match="not a Texas case study"): + testObject.texas_case_study_updates(invalid_path) From a40c9f2f9b608472956b9e6f9b3b52dad83d759b Mon Sep 17 00:00:00 2001 From: bstorm Date: Mon, 13 Apr 2026 12:06:30 -0600 Subject: [PATCH 11/24] replace actual load scaling file with fixture --- gtep/tests/unit/test_gtep_data.py | 47 +++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/gtep/tests/unit/test_gtep_data.py b/gtep/tests/unit/test_gtep_data.py index 6f5a7250..2e7839c3 100644 --- a/gtep/tests/unit/test_gtep_data.py +++ b/gtep/tests/unit/test_gtep_data.py @@ -38,6 +38,39 @@ def test_data_path(tmp_path): return tmp_path +# Load scaling File +@pytest.fixture +def load_scaling_data_path(tmp_path): + # create a mock ercot_adjusted_forecast file + info = { + "year": [2025, 2037, 2044], + "month": [1, 2, 9], + "day": [1, 17, 28], + "hour": [3, 1, 2], + "base_economic_coast": [10000, 13000, 21000], + "base_economic_east": [1500, 2000, 2700], + "base_economic_fwest": [5000, 8000, 9000], + "base_economic_ncent": [12000, 14000, 22000], + "base_economic_north": [1200, 1600, 2100], + "base_economic_scent": [6000, 8000, 10000], + "base_economic_south": [3000, 4500, 6000], + "base_economic_west": [1200, 1500, 1800], + "coast_net": [11000, 18000, 27000], + "east_net": [1500, 2100, 3000], + "fwest_net": [7000, 12000, 21000], + "ncent_net": [13000, 20000, 29000], + "north_net": [1500, 6000, 12000], + "scent_net": [7000, 20000, 34000], + "south_net": [3000, 6000, 8000], + "west_net": [1200, 3000, 8000], + } + df = pd.DataFrame(info) + file_path = tmp_path / "load_scale.xlsx" + + df.to_excel(file_path, index=False, header=True) + return file_path + + # Texas Data Path Fixture (texas_case_study) @pytest.fixture def texas_data_path(tmp_path): @@ -471,10 +504,10 @@ def test_clone_at_time_keys_called_correctly( assert mock_clone.call_count == expected_calls # -------------------------------------------------IMPORT_LOAD_SCALING------------------------------------------------------------ # - def test_import_load_scaling_normal_with_actual_file(self, actual_load_path): + def test_import_load_scaling_normal(self, load_scaling_data_path): # test successful passthrough of load scaling function testObject = ExpansionPlanningData() - testObject.import_load_scaling(actual_load_path) + testObject.import_load_scaling(load_scaling_data_path) df = testObject.load_scaling assert isinstance(df, pd.DataFrame) @@ -485,21 +518,23 @@ def test_import_load_scaling_normal_with_actual_file(self, actual_load_path): assert col in df.columns assert not df.empty - def test_import_load_scaling_incorrect_num_years(self, actual_load_path): + def test_import_load_scaling_incorrect_num_years(self, load_scaling_data_path): # Test value error raised if the length of forecast years is incorrect testObject = ExpansionPlanningData(stages=3) forecast_years = [2025, 2030] with pytest.raises(ValueError): - testObject.import_load_scaling(actual_load_path, forecast_years) + testObject.import_load_scaling(load_scaling_data_path, forecast_years) - def test_import_load_scaling_incorrect_years_too_early(self, actual_load_path): + def test_import_load_scaling_incorrect_years_too_early( + self, load_scaling_data_path + ): # Test value error raised if the forecast years are outside the supported ranges testObject = ExpansionPlanningData(stages=3) forecast_years = [2019, 2030, 2055] with pytest.raises(ValueError): - testObject.import_load_scaling(actual_load_path, forecast_years) + testObject.import_load_scaling(load_scaling_data_path, forecast_years) # -------------------------------------------------IMPORT_OUTAGE_DATA------------------------------------------------------------ # def test_import_outage_data( From 73dbc6bb50793d59093dd7834b5c2000acca52fc Mon Sep 17 00:00:00 2001 From: bstorm Date: Mon, 13 Apr 2026 12:27:03 -0600 Subject: [PATCH 12/24] fix texas unit test --- gtep/tests/unit/test_gtep_data.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/gtep/tests/unit/test_gtep_data.py b/gtep/tests/unit/test_gtep_data.py index 2e7839c3..3ae3ec21 100644 --- a/gtep/tests/unit/test_gtep_data.py +++ b/gtep/tests/unit/test_gtep_data.py @@ -79,6 +79,7 @@ def texas_data_path(tmp_path): 2,100,CT,G,958.3056411,928.7012334,908.9649616,21.62404,18.03036,22.80128,26.87,26.87,26.87,1.76,1.76,1.76 3,100,CT,G,958.3056411,928.7012334,908.9649616,21.62404,18.03036,22.80128,26.87,26.87,26.87,1.76,1.76,1.76 4,100,CT,G,958.3056411,928.7012334,908.9649616,21.62404,18.03036,22.80128,26.87,26.87,26.87,1.76,1.76,1.76 +2-c,100,CT,G,958.3056411,928.7012334,908.9649616,21.62404,18.03036,22.80128,26.87,26.87,26.87,1.76,1.76,1.76 """ dir_path = tmp_path / "Texas_2000" dir_path.mkdir(parents=True, exist_ok=True) @@ -676,8 +677,16 @@ def test_texas_case_study( testObject = ExpansionPlanningData() testObject.load_prescient(data_path=str(test_data_path)) - # testObject.representative_data = [unittest.mock.Mock()] - # testObject.representative_data[0].data = testObject.md.data + generators = testObject.md.data["elements"].get("generator", {}) + fixed_generators = {} + for gen_key, gen_val in generators.items(): + fixed_generators[str(gen_key)] = gen_val + testObject.md.data["elements"]["generator"] = fixed_generators + + # Setup representative_data with the expected structure + mock_data_point = unittest.mock.Mock() + mock_data_point.data = testObject.md.data + testObject.representative_data = [mock_data_point] # Call the method under test testObject.texas_case_study_updates(str(texas_data_path)) @@ -704,6 +713,7 @@ def test_texas_case_study( for gen_name, gen_data in generator.items(): for col in expected_columns: assert col in gen_data, f"Column {col} missing in generator {gen_name}" + # check example values gen1 = generator.get("1") assert gen1 is not None From ddfeba7a21259209bc17e2a951aff2736e0ac2c5 Mon Sep 17 00:00:00 2001 From: bstorm Date: Mon, 13 Apr 2026 12:27:58 -0600 Subject: [PATCH 13/24] add iloc to avoid deprecation warning --- gtep/gtep_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 6020eed3..07b95a45 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -384,5 +384,5 @@ def texas_case_study_updates(self, data_path): for gen in data_point.data["elements"]["generator"]: if not data_point.data["elements"]["generator"][gen].get(col): data_point.data["elements"]["generator"][gen][col] = float( - generator_df[generator_df["GEN UID"] == gen][col] + generator_df[generator_df["GEN UID"] == gen][col].iloc[0] ) From 5e727f40f2e36c72aec82cad38327037c8d0fc41 Mon Sep 17 00:00:00 2001 From: Belle Date: Tue, 28 Apr 2026 11:06:54 -0600 Subject: [PATCH 14/24] enforce pathlib typing --- gtep/gtep_data.py | 26 ++++++++---- gtep/tests/unit/test_gtep_data.py | 70 +++++++++++++------------------ 2 files changed, 46 insertions(+), 50 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index c6ee392b..0e05dbff 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -72,21 +72,21 @@ def load_prescient( # create prescient config object with defaults prescient_options = PrescientConfig() - # work around for prescient throwing an error with Path objects - if isinstance(data_path, Path): - data_path = str(data_path) - if options_dict is None: # set basic configurations that do not match prescient defaults options_dict = { - "data_path": data_path, + "data_path": str( + data_path + ), # work around for prescient (error with Path objects) "num_days": 365, "ruc_horizon": 36, } else: # ensure data path is included in options dictionary - options_dict["data_path"] = data_path + options_dict["data_path"] = str( + data_path + ) # work around for prescient (error with Path objects) # update configuration values based on options dictionary prescient_options.set_value(options_dict) @@ -348,8 +348,12 @@ def load_storage_csv(self, data_path): :param data_path: filepath for storage data csv file """ + # enforce pathlib object + if not isinstance(data_path, Path): + data_path = Path(data_path) + try: - storage_path = data_path + "/storage.csv" + storage_path = data_path / "storage.csv" storage_df = pd.read_csv(storage_path) storage_data = {} @@ -370,10 +374,14 @@ def texas_case_study_updates(self, data_path): :param data_path: filepath for generator data csv file """ # check that datapath is coming from a texas case study directory - if ("Texas" not in data_path) and ("Coal" not in data_path): + if ("Texas" not in str(data_path)) and ("Coal" not in str(data_path)): raise ValueError("The data path provided is not a Texas case study") - generator_update_path = data_path + "/gen.csv" + # enforce pathlib object + if not isinstance(data_path, Path): + data_path = Path(data_path) + + generator_update_path = data_path / "gen.csv" generator_df = pd.read_csv(generator_update_path) bonus_feature_list = [ "capex1", diff --git a/gtep/tests/unit/test_gtep_data.py b/gtep/tests/unit/test_gtep_data.py index 3ae3ec21..74cb0db3 100644 --- a/gtep/tests/unit/test_gtep_data.py +++ b/gtep/tests/unit/test_gtep_data.py @@ -249,7 +249,7 @@ def test_options_dict( "ruc_horizon": 12, } - testObject.load_prescient(data_path=str(test_data_path), options_dict=options) + testObject.load_prescient(data_path=test_data_path, options_dict=options) mock_prescient_config.set_value.assert_called_once() passed_dict = mock_prescient_config.set_value.call_args[0][0] assert passed_dict["data_path"] == str(test_data_path) @@ -262,7 +262,7 @@ def test_no_options_dict( # Test not passing in an options dictionary testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=str(test_data_path)) + testObject.load_prescient(data_path=test_data_path) mock_prescient_config.set_value.assert_called_once() passed_dict = mock_prescient_config.set_value.call_args[0][0] # Default options should be set @@ -275,7 +275,7 @@ def test_default_representative_dates( ): # Test no representative dates passed in, initializing with defaults testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=str(test_data_path)) + testObject.load_prescient(data_path=test_data_path) # default dates: expected_dates = [ "2020-01-28 00:00", @@ -291,7 +291,7 @@ def test_passed_representative_dates( # Test new representative dates passed in, replacing defaults testObject = ExpansionPlanningData() testObject.load_prescient( - data_path=str(test_data_path), + data_path=test_data_path, representative_dates=[ "2020-01-28 00:00", "2020-04-23 00:00", @@ -321,7 +321,7 @@ def test_representative_date_not_in_time_keys( ] with pytest.raises(ValueError): testObject.load_prescient( - data_path=str(test_data_path), representative_dates=bad_dates + data_path=test_data_path, representative_dates=bad_dates ) def test_empty_representative_dates( @@ -330,26 +330,14 @@ def test_empty_representative_dates( # Test an empty list passed to overwrite defaults without setting new dates testObject = ExpansionPlanningData() with pytest.raises(ZeroDivisionError): - testObject.load_prescient( - data_path=str(test_data_path), representative_dates=[] - ) - - def test_invalid_representative_dates_type( - self, mock_prescient_config, mock_gmlc_provider, test_data_path - ): - # Test setting None for the dates, overwriting defaults without adding new dates - testObject = ExpansionPlanningData() - with pytest.raises(TypeError): - testObject.load_prescient( - data_path=str(test_data_path), representative_dates=None - ) + testObject.load_prescient(data_path=test_data_path, representative_dates=[]) def test_default_representative_weights( self, mock_prescient_config, mock_gmlc_provider, test_data_path ): # Test no representative weights passed in, so the function calculates them testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=str(test_data_path)) + testObject.load_prescient(data_path=test_data_path) total_weight = mock_prescient_config.num_days * testObject.stages assert total_weight == (365 * 2) @@ -372,9 +360,7 @@ def test_no_representative_weights_passed_5_dates( ] testObject = ExpansionPlanningData() - testObject.load_prescient( - data_path=str(test_data_path), representative_dates=dates - ) + testObject.load_prescient(data_path=test_data_path, representative_dates=dates) total_weight = mock_prescient_config.num_days * testObject.stages assert total_weight == (365 * 2) @@ -391,7 +377,7 @@ def test_passed_representative_weights( weights = {1: 91, 2: 91, 3: 91, 4: 91} testObject = ExpansionPlanningData() testObject.load_prescient( - data_path=str(test_data_path), representative_weights=weights + data_path=test_data_path, representative_weights=weights ) assert testObject.representative_weights == weights @@ -411,7 +397,7 @@ def test_key_function_calls( ) as mock_load_storage, ): - testObject.load_prescient(data_path=str(test_data_path)) + testObject.load_prescient(data_path=test_data_path) # External calls mock_prescient_config.set_value.assert_called_once() @@ -420,7 +406,7 @@ def test_key_function_calls( # Internal calls mock_load_defaults.assert_called_once() - mock_load_storage.assert_called_once_with(str(test_data_path)) + mock_load_storage.assert_called_once_with(test_data_path) def test_total_num_steps_calculation( self, mock_prescient_config, mock_gmlc_provider, test_data_path @@ -429,7 +415,7 @@ def test_total_num_steps_calculation( mock_prescient_config.num_days = 365 testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=str(test_data_path)) + testObject.load_prescient(data_path=test_data_path) # period_per_step is 24 expected_total_steps = 365 * 24 @@ -445,7 +431,7 @@ def test_total_num_steps_calculation_2_years( mock_prescient_config.num_days = 730 testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=str(test_data_path)) + testObject.load_prescient(data_path=test_data_path) # period_per_step is 24 expected_total_steps = 730 * 24 @@ -459,7 +445,7 @@ def test_in_service_flags_set( ): # Test in service flags are being set properly testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=str(test_data_path)) + testObject.load_prescient(data_path=test_data_path) assert testObject.md.data["elements"]["generator"]["2-c"]["in_service"] == False assert ( @@ -473,7 +459,7 @@ def test_missing_simulation_objects_csv( testObject = ExpansionPlanningData() # tmp_path is empty, no simulation_objects.csv with pytest.raises(FileNotFoundError): - testObject.load_prescient(data_path=str(tmp_path)) + testObject.load_prescient(data_path=tmp_path) def test_incorrect_simulation_objects_csv( self, mock_prescient_config, mock_gmlc_provider, tmp_path @@ -484,13 +470,13 @@ def test_incorrect_simulation_objects_csv( testObject = ExpansionPlanningData() with pytest.raises(KeyError): - testObject.load_prescient(data_path=str(tmp_path)) + testObject.load_prescient(data_path=tmp_path) def test_clone_at_time_keys_called_correctly( self, mock_prescient_config, mock_gmlc_provider, test_data_path ): testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=str(test_data_path)) + testObject.load_prescient(data_path=test_data_path) # Patch clone_at_time_keys on the existing model instance with unittest.mock.patch.object( @@ -498,7 +484,7 @@ def test_clone_at_time_keys_called_correctly( ) as mock_clone: # Call load_prescient again to trigger cloning with patched method - testObject.load_prescient(data_path=str(test_data_path)) + testObject.load_prescient(data_path=test_data_path) # Check that clone_at_time_keys was called once per representative date expected_calls = len(testObject.representative_dates) @@ -550,11 +536,13 @@ def test_import_outage_data( # Patch pd.read_csv to return appropriate DataFrame based on input path def mock_read_csv(filepath, *args, **kwargs): - if filepath == str(outage_data): + if not isinstance(filepath, Path): + filepath = Path(filepath) + if filepath.name.endswith("outage.csv"): return df_outage - elif filepath.endswith("county_fips_match.csv"): + elif filepath.name.endswith("county_fips_match.csv"): return df_county_fips - elif filepath.endswith("Bus_data_gen_weights_mappings.csv"): + elif filepath.name.endswith("Bus_data_gen_weights_mappings.csv"): return df_bus_data else: raise FileNotFoundError(f"Unexpected file path: {filepath}") @@ -564,7 +552,7 @@ def mock_read_csv(filepath, *args, **kwargs): # Patch to_csv to avoid actual file writes during test monkeypatch.setattr(pd.DataFrame, "to_csv", lambda self, *args, **kwargs: None) - testObject.import_outage_data(str(outage_data)) + testObject.import_outage_data(outage_data) assert hasattr(testObject, "bus_hours") df = testObject.bus_hours @@ -580,7 +568,7 @@ def test_load_default_data_settings( self, mock_prescient_config, mock_gmlc_provider, test_data_path ): testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=str(test_data_path)) + testObject.load_prescient(data_path=test_data_path) testObject.load_default_data_settings() @@ -626,7 +614,7 @@ def test_load_storage_csv_success( testObject = ExpansionPlanningData() # Setup md with empty data structure to avoid errors testObject.md = mock_gmlc_provider.get_initial_actuals_model.return_value - testObject.load_storage_csv(str(test_data_path)) + testObject.load_storage_csv(test_data_path) # Check that storage data was loaded into md.data["elements"]["storage"] storage = testObject.md.data["elements"].get("storage", None) @@ -659,7 +647,7 @@ def test_load_storage_csv_file_not_found( missing_path = tmp_path with unittest.mock.patch("builtins.print") as mock_print: - testObject.load_storage_csv(str(missing_path)) + testObject.load_storage_csv(missing_path) # Should print warning about missing file mock_print.assert_called_once() @@ -675,7 +663,7 @@ def test_texas_case_study( self, mock_gmlc_provider, mock_prescient_config, test_data_path, texas_data_path ): testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=str(test_data_path)) + testObject.load_prescient(data_path=test_data_path) generators = testObject.md.data["elements"].get("generator", {}) fixed_generators = {} @@ -689,7 +677,7 @@ def test_texas_case_study( testObject.representative_data = [mock_data_point] # Call the method under test - testObject.texas_case_study_updates(str(texas_data_path)) + testObject.texas_case_study_updates(texas_data_path) generator = testObject.md.data["elements"].get("generator", None) assert generator is not None From 5b31e3458d5d316db3c0e97565f45285d7e1e7d8 Mon Sep 17 00:00:00 2001 From: bstorm Date: Thu, 30 Apr 2026 17:03:26 -0600 Subject: [PATCH 15/24] switch to unittest asserts and simplify mock --- gtep/tests/unit/test_gtep_data.py | 591 ++++++++---------------------- 1 file changed, 145 insertions(+), 446 deletions(-) diff --git a/gtep/tests/unit/test_gtep_data.py b/gtep/tests/unit/test_gtep_data.py index 74cb0db3..464badad 100644 --- a/gtep/tests/unit/test_gtep_data.py +++ b/gtep/tests/unit/test_gtep_data.py @@ -20,134 +20,20 @@ import pandas as pd from pathlib import Path +curr_dir = Path(__file__).resolve().parent +input_data_source = (curr_dir / ".." / ".." / "data" / "5bus").resolve() -# Data Path Fixture (Load Prescient) -@pytest.fixture -def test_data_path(tmp_path): - # Create a minimal simulation_objects.csv file with expected content - csv_content = """index,DAY_AHEAD -Periods_per_Step,24 -""" - csv_file = tmp_path / "simulation_objects.csv" - csv_file.write_text(csv_content) - load_file = tmp_path / "storage.csv" - load_file.write_text( - """name,bus,generator,storage_type,energy_capacity,initial_state_of_charge,end_state_of_charge,minimum_state_of_charge,charge_efficiency,discharge_efficiency,max_discharge_rate,min_discharge_rate,max_charge_rate,min_charge_rate,initial_charge_rate,initial_discharge_rate,charge_cost,discharge_cost,retention_rate_60min,ramp_up_input_60min,ramp_down_input_60min,ramp_up_output_60min,ramp_down_output_60min,in_service,capital_multiplier,extension_multiplier,investment_cost,investment_cost_kwh -100MW_400MWh_1,3,None,'battery',400,320,320,80,1,1,100,10,400,1,0,0,175,0,1,400,400,100,100,FALSE,1,1,1490.89,372.7225""" - ) - return tmp_path - - -# Load scaling File -@pytest.fixture -def load_scaling_data_path(tmp_path): - # create a mock ercot_adjusted_forecast file - info = { - "year": [2025, 2037, 2044], - "month": [1, 2, 9], - "day": [1, 17, 28], - "hour": [3, 1, 2], - "base_economic_coast": [10000, 13000, 21000], - "base_economic_east": [1500, 2000, 2700], - "base_economic_fwest": [5000, 8000, 9000], - "base_economic_ncent": [12000, 14000, 22000], - "base_economic_north": [1200, 1600, 2100], - "base_economic_scent": [6000, 8000, 10000], - "base_economic_south": [3000, 4500, 6000], - "base_economic_west": [1200, 1500, 1800], - "coast_net": [11000, 18000, 27000], - "east_net": [1500, 2100, 3000], - "fwest_net": [7000, 12000, 21000], - "ncent_net": [13000, 20000, 29000], - "north_net": [1500, 6000, 12000], - "scent_net": [7000, 20000, 34000], - "south_net": [3000, 6000, 8000], - "west_net": [1200, 3000, 8000], - } - df = pd.DataFrame(info) - file_path = tmp_path / "load_scale.xlsx" - - df.to_excel(file_path, index=False, header=True) - return file_path - - -# Texas Data Path Fixture (texas_case_study) -@pytest.fixture -def texas_data_path(tmp_path): - csv_content = """GEN UID,Bus ID,Unit Type,Fuel,capex1,capex2,capex3,fuel_cost1,fuel_cost2,fuel_cost3,fixed_ops1,fixed_ops2,fixed_ops3,var_ops1,var_ops2,var_ops3 -1,107,NUC,N,7040.489322,6966.178959,6731.224251,7.210946358,7.294679868,7.378403404,145.96,145.96,145.96,2.84,2.84,2.84 -2,100,CT,G,958.3056411,928.7012334,908.9649616,21.62404,18.03036,22.80128,26.87,26.87,26.87,1.76,1.76,1.76 -3,100,CT,G,958.3056411,928.7012334,908.9649616,21.62404,18.03036,22.80128,26.87,26.87,26.87,1.76,1.76,1.76 -4,100,CT,G,958.3056411,928.7012334,908.9649616,21.62404,18.03036,22.80128,26.87,26.87,26.87,1.76,1.76,1.76 -2-c,100,CT,G,958.3056411,928.7012334,908.9649616,21.62404,18.03036,22.80128,26.87,26.87,26.87,1.76,1.76,1.76 -""" - dir_path = tmp_path / "Texas_2000" - dir_path.mkdir(parents=True, exist_ok=True) - excel_path = dir_path / "gen.csv" - excel_path.write_text(csv_content) - - return dir_path - - -# Load Data File Fixture (Import Load) -@pytest.fixture -def actual_load_path(): - test_dir = Path(__file__).parent - # Navigate up to 'gtep' directory, then into 'data' - data_dir = test_dir.parent.parent / "data" - excel_path = data_dir / "Texas_2000" / "ERCOT-Adjusted-Forecast.xlsb" - return excel_path - - -# Outage Data File Fixtures (Import Outages) -@pytest.fixture -def outage_data(tmp_path): - outage_content = """fips_code,lim_timestamp,case_4b_prob,date -48001,5/20/25 0:00,0,20-May -48025,5/20/25 3:00,0,20-May -48025,5/20/25 4:00,0,20-May -48025,5/20/25 5:00,0,20-May -48025,5/20/25 6:00,0,20-May -48025,5/20/25 7:00,0,20-May -48025,5/20/25 8:00,0,20-May -48025,5/20/25 9:00,0,20-May -48025,5/20/25 10:00,0,20-May -48027,5/20/25 20:00,0,20-May -48163,5/20/25 20:00,0.006944444,20-May""" - outage_csv = tmp_path / "outage.csv" - outage_csv.write_text(outage_content) - - return outage_csv +load_scaling_file = ( + curr_dir / ".." / ".." / "data" / "Texas_2000" / "ERCOT-Adjusted-Forecast.xlsb" +).resolve() +storage_file = (curr_dir / ".." / ".." / "data" / "9_bus_GTEP_dir").resolve() -@pytest.fixture -def county_fips_csv(tmp_path): - dir_path = tmp_path / "gtep" / "data" / "123_Bus_Resil_Week" - dir_path.mkdir(parents=True, exist_ok=True) - content = """county_number,County,FIPS,, -1,Anderson,48001,, -13,Bee,48025,, -27,Burnet,48053,, -82,Frio,48163,, -""" - file = dir_path / "county_fips_match.csv" - file.write_text(content) - return file - +texas_data_path = (curr_dir / ".." / ".." / "data" / "Texas_2000").resolve -@pytest.fixture -def bus_data_csv(tmp_path): - dir_path = tmp_path / "gtep" / "data" / "123_Bus_Resil_Week" - dir_path.mkdir(parents=True, exist_ok=True) - content = """Bus Number,County -1001,Anderson -1002,Bee -1003,Burnet -1004,Frio -""" - file = dir_path / "Bus_data_gen_weights_mappings.csv" - file.write_text(content) - return file +outage_data_path = ( + curr_dir / ".." / ".." / "data" / "123_Bus_Resil_Week" / "may_20.csv" +).resolve() # Function Mocks (Load Prescient) @@ -166,81 +52,39 @@ def mock_prescient_config(): yield mock_instance -@pytest.fixture -def mock_gmlc_provider(): - with unittest.mock.patch( - "prescient.data.providers.gmlc_data_provider.GmlcDataProvider" - ) as MockProvider: - instance = MockProvider.return_value - instance._start_time = "2020-01-01 00:00" - # Mock get_initial_actuals_model to return a model with .data attribute - model_mock = unittest.mock.Mock() - model_mock.data = { - "elements": { - "generator": { - "1": {"fuel": "C", "in_service": True}, - "2": {"fuel": "None", "in_service": True}, - "2-c": {"fuel": "None", "in_service": True}, - }, - "branch": {"branch1": {}, "branch2-c": {}}, - "storage": {"stor1": {}}, - }, - "system": { - "time_keys": [ - "2020-01-28 00:00", - "2020-01-28 01:00", - "2020-01-28 02:00", - "2020-04-23 00:00", - "2020-07-05 00:00", - "2020-10-14 00:00", - "2020-12-14 00:00", - ] - }, - } - model_mock.clone_at_time_keys.side_effect = lambda keys: f"clone_{keys}" - instance.get_initial_actuals_model.return_value = model_mock - instance.populate_with_actuals.return_value = None - yield instance - - -@pytest.mark.usefixtures("mock_prescient_config", "mock_gmlc_provider") -class TestExpansionPlanningData: +class TestExpansionPlanningData(unittest.TestCase): def test_data_init(self): # Test that the ExpansionPlanningData object initializes properly with default values testObject = ExpansionPlanningData() - assert isinstance(testObject, ExpansionPlanningData) - assert testObject.stages == 2 - assert testObject.num_reps == 4 - assert testObject.len_reps == 1 - assert testObject.num_commit == 24 - assert testObject.num_dispatch == 1 - assert testObject.duration_dispatch == 60 + self.assertIsInstance(testObject, ExpansionPlanningData) + self.assertEqual(testObject.stages, 2) + self.assertEqual(testObject.num_reps, 4) + self.assertEqual(testObject.len_reps, 1) + self.assertEqual(testObject.num_commit, 24) + self.assertEqual(testObject.num_dispatch, 1) + self.assertEqual(testObject.duration_dispatch, 60) # Test that the ExpansionPlanningData object initializes properly with input values testObject = ExpansionPlanningData(1, 2, 2, 2, 2, 15) - assert isinstance(testObject, ExpansionPlanningData) - assert testObject.stages == 1 - assert testObject.num_reps == 2 - assert testObject.len_reps == 2 - assert testObject.num_commit == 2 - assert testObject.num_dispatch == 2 - assert testObject.duration_dispatch == 15 + self.assertEqual(testObject.stages, 1) + self.assertEqual(testObject.num_reps, 2) + self.assertEqual(testObject.len_reps, 2) + self.assertEqual(testObject.num_commit, 2) + self.assertEqual(testObject.num_dispatch, 2) + self.assertEqual(testObject.duration_dispatch, 15) # Test that the ExpansionPlanningData object initializes properly with partial input values testObject = ExpansionPlanningData(duration_dispatch=15) - assert isinstance(testObject, ExpansionPlanningData) - assert testObject.stages == 2 - assert testObject.num_reps == 4 - assert testObject.len_reps == 1 - assert testObject.num_commit == 24 - assert testObject.num_dispatch == 1 - assert testObject.duration_dispatch == 15 + self.assertEqual(testObject.stages, 2) + self.assertEqual(testObject.num_reps, 4) + self.assertEqual(testObject.len_reps, 1) + self.assertEqual(testObject.num_commit, 24) + self.assertEqual(testObject.num_dispatch, 1) + self.assertEqual(testObject.duration_dispatch, 15) # -------------------------------------------------LOAD_PRESCIENT------------------------------------------------------------ # - def test_options_dict( - self, mock_prescient_config, mock_gmlc_provider, test_data_path - ): + def test_options_dict(self, mock_prescient_config): # Test passing in an options dictionary testObject = ExpansionPlanningData() # set options that are not the defaults @@ -249,33 +93,30 @@ def test_options_dict( "ruc_horizon": 12, } - testObject.load_prescient(data_path=test_data_path, options_dict=options) + testObject.load_prescient(data_path=input_data_source, options_dict=options) + mock_prescient_config.set_value.assert_called_once() passed_dict = mock_prescient_config.set_value.call_args[0][0] - assert passed_dict["data_path"] == str(test_data_path) - assert passed_dict["num_days"] == 100 - assert passed_dict["ruc_horizon"] == 12 + self.assertEqual(passed_dict["data_path"], str(input_data_source)) + self.assertEqual(passed_dict["num_days"], 100) + self.assertEqual(passed_dict["ruc_horizon"], 12) - def test_no_options_dict( - self, mock_prescient_config, mock_gmlc_provider, test_data_path - ): + def test_no_options_dict(self, mock_prescient_config): # Test not passing in an options dictionary testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=test_data_path) + testObject.load_prescient(data_path=input_data_source) mock_prescient_config.set_value.assert_called_once() passed_dict = mock_prescient_config.set_value.call_args[0][0] # Default options should be set - assert passed_dict["data_path"] == str(test_data_path) - assert passed_dict["num_days"] == 365 - assert passed_dict["ruc_horizon"] == 36 + self.assertEqual(passed_dict["data_path"], str(input_data_source)) + self.assertEqual(passed_dict["num_days"], 365) + self.assertEqual(passed_dict["ruc_horizon"], 36) - def test_default_representative_dates( - self, mock_prescient_config, mock_gmlc_provider, test_data_path - ): + def test_default_representative_dates(self): # Test no representative dates passed in, initializing with defaults testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=test_data_path) + testObject.load_prescient(data_path=input_data_source) # default dates: expected_dates = [ "2020-01-28 00:00", @@ -283,34 +124,26 @@ def test_default_representative_dates( "2020-07-05 00:00", "2020-10-14 00:00", ] - assert testObject.representative_dates == expected_dates + self.assertEqual(testObject.representative_dates, expected_dates) - def test_passed_representative_dates( - self, mock_prescient_config, mock_gmlc_provider, test_data_path - ): + def test_passed_representative_dates(self): # Test new representative dates passed in, replacing defaults - testObject = ExpansionPlanningData() - testObject.load_prescient( - data_path=test_data_path, - representative_dates=[ - "2020-01-28 00:00", - "2020-04-23 00:00", - "2020-07-05 00:00", - "2020-12-14 00:00", - ], - ) - expected_dates = [ "2020-01-28 00:00", "2020-04-23 00:00", "2020-07-05 00:00", "2020-12-14 00:00", ] - assert testObject.representative_dates == expected_dates - def test_representative_date_not_in_time_keys( - self, mock_prescient_config, mock_gmlc_provider, test_data_path - ): + testObject = ExpansionPlanningData() + testObject.load_prescient( + data_path=input_data_source, + representative_dates=expected_dates, + ) + + self.assertEqual(testObject.representative_dates, expected_dates) + + def test_representative_date_not_in_time_keys(self): # Test passing invalid/not covered dates testObject = ExpansionPlanningData() bad_dates = [ @@ -319,37 +152,31 @@ def test_representative_date_not_in_time_keys( "2020-07-05 00:00", "2099-01-01 00:00", # Not in time_keys ] - with pytest.raises(ValueError): + with self.assertRaises(ValueError): testObject.load_prescient( - data_path=test_data_path, representative_dates=bad_dates + data_path=input_data_source, representative_dates=bad_dates ) - def test_empty_representative_dates( - self, mock_prescient_config, mock_gmlc_provider, test_data_path - ): + def test_empty_representative_dates(self): # Test an empty list passed to overwrite defaults without setting new dates testObject = ExpansionPlanningData() - with pytest.raises(ZeroDivisionError): - testObject.load_prescient(data_path=test_data_path, representative_dates=[]) + with self.assertRaises(ZeroDivisionError): + testObject.load_prescient( + data_path=input_data_source, representative_dates=[] + ) - def test_default_representative_weights( - self, mock_prescient_config, mock_gmlc_provider, test_data_path - ): + def test_default_representative_weights(self): # Test no representative weights passed in, so the function calculates them testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=test_data_path) + testObject.load_prescient(data_path=input_data_source) - total_weight = mock_prescient_config.num_days * testObject.stages - assert total_weight == (365 * 2) + total_weight = 365 * testObject.stages expected_weight = int(total_weight / len(testObject.representative_dates)) - assert expected_weight == int((365 * 2) / 4) for w in testObject.representative_weights.values(): - assert w == expected_weight + self.assertEqual(w, expected_weight) - def test_no_representative_weights_passed_5_dates( - self, mock_prescient_config, mock_gmlc_provider, test_data_path - ): + def test_no_representative_weights_passed_5_dates(self): # Test no representative weights passed in and custom dates passed in, so the function calculates them dates = [ "2020-01-28 00:00", @@ -360,123 +187,56 @@ def test_no_representative_weights_passed_5_dates( ] testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=test_data_path, representative_dates=dates) + testObject.load_prescient( + data_path=input_data_source, representative_dates=dates + ) - total_weight = mock_prescient_config.num_days * testObject.stages - assert total_weight == (365 * 2) + total_weight = 365 * testObject.stages expected_weight = int(total_weight / len(testObject.representative_dates)) - assert expected_weight == int((365 * 2) / 5) for w in testObject.representative_weights.values(): - assert w == expected_weight + self.assertEqual(w, expected_weight) - def test_passed_representative_weights( - self, mock_prescient_config, mock_gmlc_provider, test_data_path - ): + def test_passed_representative_weights(self): # Test representative weights passed in weights = {1: 91, 2: 91, 3: 91, 4: 91} testObject = ExpansionPlanningData() testObject.load_prescient( - data_path=test_data_path, representative_weights=weights + data_path=input_data_source, representative_weights=weights ) - assert testObject.representative_weights == weights - - def test_key_function_calls( - self, mock_prescient_config, mock_gmlc_provider, test_data_path - ): - # Test that each function is called within the load_prescient function - testObject = ExpansionPlanningData() - - # Patch internal methods to track calls - with ( - unittest.mock.patch.object( - testObject, "load_default_data_settings" - ) as mock_load_defaults, - unittest.mock.patch.object( - testObject, "load_storage_csv" - ) as mock_load_storage, - ): - - testObject.load_prescient(data_path=test_data_path) - - # External calls - mock_prescient_config.set_value.assert_called_once() - mock_gmlc_provider.get_initial_actuals_model.assert_called_once() - mock_gmlc_provider.populate_with_actuals.assert_called_once() - - # Internal calls - mock_load_defaults.assert_called_once() - mock_load_storage.assert_called_once_with(test_data_path) - - def test_total_num_steps_calculation( - self, mock_prescient_config, mock_gmlc_provider, test_data_path - ): - # Check that the total number of time steps is calculated based in the number of days in the config - mock_prescient_config.num_days = 365 - - testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=test_data_path) - - # period_per_step is 24 - expected_total_steps = 365 * 24 - - mock_gmlc_provider.get_initial_actuals_model.assert_called_once() - call_args = mock_gmlc_provider.get_initial_actuals_model.call_args[1] - assert call_args["num_time_steps"] == expected_total_steps + self.assertEqual(testObject.representative_weights, weights) - def test_total_num_steps_calculation_2_years( - self, mock_prescient_config, mock_gmlc_provider, test_data_path - ): - # Check that the total number of time steps is calculated based in the number of days in the config - mock_prescient_config.num_days = 730 - - testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=test_data_path) - - # period_per_step is 24 - expected_total_steps = 730 * 24 - - mock_gmlc_provider.get_initial_actuals_model.assert_called_once() - call_args = mock_gmlc_provider.get_initial_actuals_model.call_args[1] - assert call_args["num_time_steps"] == expected_total_steps - - def test_in_service_flags_set( - self, mock_prescient_config, mock_gmlc_provider, test_data_path - ): + # FIXME + def test_in_service_flags_set(self): # Test in service flags are being set properly testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=test_data_path) + testObject.load_prescient(data_path=input_data_source) assert testObject.md.data["elements"]["generator"]["2-c"]["in_service"] == False assert ( testObject.md.data["elements"]["branch"]["branch2-c"]["in_service"] == False ) - def test_missing_simulation_objects_csv( - self, mock_prescient_config, mock_gmlc_provider, tmp_path - ): + def test_missing_simulation_objects_csv(self, tmp_path): # test if a data path is missing the simulation objects, it should throw an error testObject = ExpansionPlanningData() # tmp_path is empty, no simulation_objects.csv - with pytest.raises(FileNotFoundError): + with self.assertRaises(FileNotFoundError): testObject.load_prescient(data_path=tmp_path) - def test_incorrect_simulation_objects_csv( - self, mock_prescient_config, mock_gmlc_provider, tmp_path - ): + def test_incorrect_simulation_objects_csv(self, tmp_path): # test if simulations_objects is missing key details, it should throw an error csv_file = tmp_path / "simulation_objects.csv" csv_file.write_text("index,DAY_AHEAD\nWrongKey,24\n") testObject = ExpansionPlanningData() - with pytest.raises(KeyError): + with self.assertRaises(KeyError): testObject.load_prescient(data_path=tmp_path) - def test_clone_at_time_keys_called_correctly( - self, mock_prescient_config, mock_gmlc_provider, test_data_path - ): + # FIXME + def test_clone_at_time_keys_called_correctly(self): testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=test_data_path) + testObject.load_prescient(data_path=input_data_source) # Patch clone_at_time_keys on the existing model instance with unittest.mock.patch.object( @@ -484,17 +244,18 @@ def test_clone_at_time_keys_called_correctly( ) as mock_clone: # Call load_prescient again to trigger cloning with patched method - testObject.load_prescient(data_path=test_data_path) + testObject.load_prescient(data_path=input_data_source) # Check that clone_at_time_keys was called once per representative date expected_calls = len(testObject.representative_dates) assert mock_clone.call_count == expected_calls # -------------------------------------------------IMPORT_LOAD_SCALING------------------------------------------------------------ # - def test_import_load_scaling_normal(self, load_scaling_data_path): + # FIXME + def test_import_load_scaling_normal(self): # test successful passthrough of load scaling function testObject = ExpansionPlanningData() - testObject.import_load_scaling(load_scaling_data_path) + testObject.import_load_scaling(load_scaling_file) df = testObject.load_scaling assert isinstance(df, pd.DataFrame) @@ -505,70 +266,38 @@ def test_import_load_scaling_normal(self, load_scaling_data_path): assert col in df.columns assert not df.empty - def test_import_load_scaling_incorrect_num_years(self, load_scaling_data_path): + def test_import_load_scaling_incorrect_num_years(self): # Test value error raised if the length of forecast years is incorrect testObject = ExpansionPlanningData(stages=3) forecast_years = [2025, 2030] - with pytest.raises(ValueError): - testObject.import_load_scaling(load_scaling_data_path, forecast_years) + with self.assertRaises(ValueError): + testObject.import_load_scaling(load_scaling_file, forecast_years) - def test_import_load_scaling_incorrect_years_too_early( - self, load_scaling_data_path - ): + def test_import_load_scaling_incorrect_years_too_early(self): # Test value error raised if the forecast years are outside the supported ranges testObject = ExpansionPlanningData(stages=3) forecast_years = [2019, 2030, 2055] - with pytest.raises(ValueError): - testObject.import_load_scaling(load_scaling_data_path, forecast_years) + with self.assertRaises(ValueError): + testObject.import_load_scaling(load_scaling_file, forecast_years) # -------------------------------------------------IMPORT_OUTAGE_DATA------------------------------------------------------------ # - def test_import_outage_data( - self, outage_data, county_fips_csv, bus_data_csv, monkeypatch - ): + def test_import_outage_data(self): testObject = ExpansionPlanningData() - # Load DataFrames from temp CSV files for patching - df_outage = pd.read_csv(outage_data) - df_county_fips = pd.read_csv(county_fips_csv) - df_bus_data = pd.read_csv(bus_data_csv) - - # Patch pd.read_csv to return appropriate DataFrame based on input path - def mock_read_csv(filepath, *args, **kwargs): - if not isinstance(filepath, Path): - filepath = Path(filepath) - if filepath.name.endswith("outage.csv"): - return df_outage - elif filepath.name.endswith("county_fips_match.csv"): - return df_county_fips - elif filepath.name.endswith("Bus_data_gen_weights_mappings.csv"): - return df_bus_data - else: - raise FileNotFoundError(f"Unexpected file path: {filepath}") - - monkeypatch.setattr(pd, "read_csv", mock_read_csv) - - # Patch to_csv to avoid actual file writes during test - monkeypatch.setattr(pd.DataFrame, "to_csv", lambda self, *args, **kwargs: None) + testObject.import_outage_data(outage_data_path) - testObject.import_outage_data(outage_data) - - assert hasattr(testObject, "bus_hours") df = testObject.bus_hours - assert isinstance(df, pd.DataFrame) - assert set(df.columns) == {"hour", "Bus Number"} - assert pd.api.types.is_integer_dtype(df["hour"]) - assert pd.api.types.is_integer_dtype(df["Bus Number"]) - assert (df["hour"] == 20).any() - assert (df["Bus Number"] == 1004).any() + + self.assertHasAttr(testObject, "bus_hours") + self.assertIsInstance(df, pd.DataFrame) + self.assertIn(["hour", "Bus Number"], df.columns) # -------------------------------------------------LOAD_DEFAULT_DATA_SETTINGS------------------------------------------------------------ # - def test_load_default_data_settings( - self, mock_prescient_config, mock_gmlc_provider, test_data_path - ): + def test_load_default_data_settings(self): testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=test_data_path) + testObject.load_prescient(data_path=input_data_source) testObject.load_default_data_settings() @@ -576,53 +305,47 @@ def test_load_default_data_settings( for gen_name, gen in testObject.md.data["elements"]["generator"].items(): if gen.get("fuel") == "C": if gen.get("in_service") is False: - assert gen["lifetime"] == 1 + self.assertEqual(gen["lifetime"], 1) else: - assert gen["lifetime"] == 2 + self.assertEqual(gen["lifetime"], 2) else: - assert gen["lifetime"] == 3 + self.assertEqual(gen["lifetime"], 3) # Check other fixed attributes - assert gen["spinning_reserve_frac"] == 0.1 - assert gen["quickstart_reserve_frac"] == 0.1 - assert gen["capital_multiplier"] == 1 - assert gen["extension_multiplier"] == 0 - assert gen["max_operating_reserve"] == 1 - assert gen["max_spinning_reserve"] == 1 - assert gen["max_quickstart_reserve"] == 1 - assert gen["ramp_up_rate"] == 0.1 - assert gen["ramp_down_rate"] == 0.1 - assert gen["emissions_factor"] == 1 - assert gen["start_fuel"] == 1 - assert gen["investment_cost"] == 1 + self.assertEqual(gen["spinning_reserve_frac"], 0.1) + self.assertEqual(gen["quickstart_reserve_frac"], 0.1) + self.assertEqual(gen["capital_multiplier"], 1) + self.assertEqual(gen["extension_multiplier"], 0) + self.assertEqual(gen["max_operating_reserve"], 1) + self.assertEqual(gen["max_spinning_reserve"], 1) + self.assertEqual(gen["max_quickstart_reserve"], 1) + self.assertEqual(gen["ramp_up_rate"], 0.1) + self.assertEqual(gen["ramp_down_rate"], 0.1) + self.assertEqual(gen["emissions_factor"], 1) + self.assertEqual(gen["start_fuel"], 1) + self.assertEqual(gen["investment_cost"], 1) # Check branches for branch in testObject.md.data["elements"]["branch"].values(): - assert branch["loss_rate"] == 0 - assert branch["distance"] == 1 - assert branch["capital_cost"] == 10000000 + self.assertEqual(branch["loss_rate"], 0) + self.assertEqual(branch["distance"], 1) + self.assertEqual(branch["capital_cost"], 10000000) # Check system system = testObject.md.data["system"] - assert system["min_operating_reserve"] == 0.1 - assert system["min_spinning_reserve"] == 0.1 + self.assertEqual(system["min_operating_reserve"], 0.1) + self.assertEqual(system["min_spinning_reserve"], 0.1) # -------------------------------------------------LOAD_STORAGE_CSV------------------------------------------------------------ # - def test_load_storage_csv_success( - self, test_data_path, mock_gmlc_provider, mock_prescient_config - ): + def test_load_storage_csv_success(self): testObject = ExpansionPlanningData() - # Setup md with empty data structure to avoid errors - testObject.md = mock_gmlc_provider.get_initial_actuals_model.return_value - testObject.load_storage_csv(test_data_path) + testObject.load_prescient(data_path=input_data_source) + testObject.load_storage_csv(storage_file) # Check that storage data was loaded into md.data["elements"]["storage"] storage = testObject.md.data["elements"].get("storage", None) - assert storage is not None - assert isinstance(storage, dict) - - # Check that storage keys include the name from your CSV fixture - assert "100MW_400MWh_1" in storage + self.assertIsNotNone(storage) + self.assertIsInstance(storage, dict) # Check some expected keys in storage data expected_keys = { @@ -635,52 +358,33 @@ def test_load_storage_csv_success( "investment_cost_kwh", } for key in expected_keys: - assert key in storage["100MW_400MWh_1"] + assert key in storage["100MW_400MWh_1"].keys() - def test_load_storage_csv_file_not_found( - self, tmp_path, mock_gmlc_provider, mock_prescient_config - ): + def test_load_storage_string_path(self): testObject = ExpansionPlanningData() - testObject.md = mock_gmlc_provider.get_initial_actuals_model.return_value - - # Use a directory without storage.csv - missing_path = tmp_path + testObject.load_prescient(data_path=input_data_source) + testObject.load_storage_csv(str(storage_file)) # should not throw an error - with unittest.mock.patch("builtins.print") as mock_print: - testObject.load_storage_csv(missing_path) - - # Should print warning about missing file - mock_print.assert_called_once() - args = mock_print.call_args[0][0] - assert "does not exist" in args + def test_load_storage_csv_file_not_found(self): + testObject = ExpansionPlanningData() + testObject.load_prescient(input_data_source) + testObject.load_storage_csv(input_data_source) # Storage should be set to empty dict storage = testObject.md.data["elements"].get("storage", None) - assert storage == {} + self.assertIsInstance(storage, dict) + self.assertEqual(storage, {}) # -------------------------------------------------TEXAS_CASE_STUDY_UPDATES----------------------------------------------------------- # - def test_texas_case_study( - self, mock_gmlc_provider, mock_prescient_config, test_data_path, texas_data_path - ): + def test_texas_case_study(self): testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=test_data_path) - - generators = testObject.md.data["elements"].get("generator", {}) - fixed_generators = {} - for gen_key, gen_val in generators.items(): - fixed_generators[str(gen_key)] = gen_val - testObject.md.data["elements"]["generator"] = fixed_generators - - # Setup representative_data with the expected structure - mock_data_point = unittest.mock.Mock() - mock_data_point.data = testObject.md.data - testObject.representative_data = [mock_data_point] + testObject.load_prescient(data_path=input_data_source) # Call the method under test testObject.texas_case_study_updates(texas_data_path) generator = testObject.md.data["elements"].get("generator", None) - assert generator is not None + self.assertIsNotNone(generator) expected_columns = [ "capex1", @@ -700,18 +404,13 @@ def test_texas_case_study( # Check that each expected column is added to each generator for gen_name, gen_data in generator.items(): for col in expected_columns: - assert col in gen_data, f"Column {col} missing in generator {gen_name}" - - # check example values - gen1 = generator.get("1") - assert gen1 is not None - assert abs(gen1["capex1"] - 7040.489322) < 1e-6 - assert abs(gen1["fuel_cost1"] - 7.210946358) < 1e-6 + self.assertIn( + col, gen_data, f"Column {col} missing in generator {gen_name}" + ) def test_texas_case_study_invalid_data_path(self): # Test that an error is raised if not a Texas case Study testObject = ExpansionPlanningData() - invalid_path = "/some/invalid/path" - with pytest.raises(ValueError, match="not a Texas case study"): - testObject.texas_case_study_updates(invalid_path) + with self.assertRaises(ValueError, match="not a Texas case study"): + testObject.texas_case_study_updates(input_data_source) From eb334e7f76468c5213fdd50049b5f623b55fd870 Mon Sep 17 00:00:00 2001 From: bstorm Date: Wed, 6 May 2026 15:40:55 -0600 Subject: [PATCH 16/24] add check for empty dataframe --- gtep/gtep_data.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 0e05dbff..34ac55dc 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -401,6 +401,8 @@ def texas_case_study_updates(self, data_path): for col in bonus_feature_list: for gen in data_point.data["elements"]["generator"]: if not data_point.data["elements"]["generator"][gen].get(col): - data_point.data["elements"]["generator"][gen][col] = float( - generator_df[generator_df["GEN UID"] == gen][col].iloc[0] - ) + matching_rows = generator_df[generator_df["GEN UID"] == gen] + if not matching_rows.empty: + data_point.data["elements"]["generator"][gen][col] = float( + matching_rows[col].iloc[0] + ) From c9cd080c54ee143e5afe8b0da870a6f843690c0b Mon Sep 17 00:00:00 2001 From: bstorm Date: Wed, 6 May 2026 16:01:54 -0600 Subject: [PATCH 17/24] add directory call for outage data --- gtep/gtep_data.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 34ac55dc..640565d7 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -265,12 +265,15 @@ def import_outage_data(self, load_file_name): r" (\d+):" ) filtered_outages = filtered_outages[["fips_code", "hour"]] - county_to_fips = pd.read_csv( - "./gtep/data/123_Bus_Resil_Week/county_fips_match.csv" - ) - bus_to_county = pd.read_csv( - "./gtep/data/123_Bus_Resil_Week/Bus_data_gen_weights_mappings.csv" - ) + + base_dir = Path(load_file_name).parent + + county_fips_path = base_dir / "county_fips_match.csv" + bus_to_county_path = base_dir / "Bus_data_gen_weights_mappings.csv" + + county_to_fips = pd.read_csv(county_fips_path) + bus_to_county = pd.read_csv(bus_to_county_path) + county_to_fips = county_to_fips[["County", "FIPS"]] bus_to_county = bus_to_county[["Bus Number", "County"]] bus_to_county = bus_to_county.merge(county_to_fips, how="inner", on="County") From c7645ac486afcb7b7a9fb093b29962d3db8b5031 Mon Sep 17 00:00:00 2001 From: bstorm Date: Wed, 6 May 2026 17:15:26 -0600 Subject: [PATCH 18/24] add resil week to texas check and base directory to csv path --- gtep/gtep_data.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 640565d7..10e7ef65 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -285,7 +285,8 @@ def import_outage_data(self, load_file_name): how="left", ) bus_hours = bus_hours[bus_hours["Bus Number"].notna()] - bus_hours.to_csv("./gtep/data/123_Bus_Resil_Week/not_right.csv") + csv_path = base_dir / "not_right.csv" + bus_hours.to_csv(csv_path) self.bus_hours = bus_hours[["hour", "Bus Number"]] self.bus_hours = self.bus_hours.astype(int) @@ -377,7 +378,11 @@ def texas_case_study_updates(self, data_path): :param data_path: filepath for generator data csv file """ # check that datapath is coming from a texas case study directory - if ("Texas" not in str(data_path)) and ("Coal" not in str(data_path)): + if ( + ("Texas" not in str(data_path)) + and ("Coal" not in str(data_path)) + and ("Resil_Week" not in str(data_path)) + ): raise ValueError("The data path provided is not a Texas case study") # enforce pathlib object From b82794c0bd94471ec5a24f3b0cf744a473b1dc14 Mon Sep 17 00:00:00 2001 From: bstorm Date: Wed, 6 May 2026 17:36:21 -0600 Subject: [PATCH 19/24] convert to unittest assert and use real data files --- gtep/tests/unit/test_gtep_data.py | 107 ++++++++++-------------------- 1 file changed, 35 insertions(+), 72 deletions(-) diff --git a/gtep/tests/unit/test_gtep_data.py b/gtep/tests/unit/test_gtep_data.py index 464badad..9296ba9b 100644 --- a/gtep/tests/unit/test_gtep_data.py +++ b/gtep/tests/unit/test_gtep_data.py @@ -13,12 +13,13 @@ import pyomo.common.unittest as unittest -from unittest.mock import create_autospec +from unittest.mock import create_autospec, patch import pytest from gtep.gtep_data import ExpansionPlanningData import prescient.simulator.config import pandas as pd from pathlib import Path +import tempfile curr_dir = Path(__file__).resolve().parent input_data_source = (curr_dir / ".." / ".." / "data" / "5bus").resolve() @@ -29,27 +30,24 @@ storage_file = (curr_dir / ".." / ".." / "data" / "9_bus_GTEP_dir").resolve() -texas_data_path = (curr_dir / ".." / ".." / "data" / "Texas_2000").resolve +texas_data_path = (curr_dir / ".." / ".." / "data" / "123_Bus_Coal").resolve() outage_data_path = ( curr_dir / ".." / ".." / "data" / "123_Bus_Resil_Week" / "may_20.csv" ).resolve() -# Function Mocks (Load Prescient) +# creates a face simulations object file @pytest.fixture -def mock_prescient_config(): - mock_instance = create_autospec( - prescient.simulator.config.PrescientConfig, instance=True - ) - mock_instance.set_value.return_value = None - mock_instance.num_days = 365 - mock_instance.sced_frequency_minutes = 60 - - with unittest.mock.patch( - "gtep.gtep_data.PrescientConfig", return_value=mock_instance - ): - yield mock_instance +def inaccurate_simulation_objects_file(tmp_path): + """ + Creates a simulation_objects.csv file with incorrect content + inside a temporary directory and returns the directory path. + """ + csv_file = tmp_path / "simulation_objects.csv" + # Write inaccurate content (missing required keys) + csv_file.write_text("index,DAY_AHEAD\nWrongKey,24\n") + return tmp_path class TestExpansionPlanningData(unittest.TestCase): @@ -84,35 +82,6 @@ def test_data_init(self): self.assertEqual(testObject.duration_dispatch, 15) # -------------------------------------------------LOAD_PRESCIENT------------------------------------------------------------ # - def test_options_dict(self, mock_prescient_config): - # Test passing in an options dictionary - testObject = ExpansionPlanningData() - # set options that are not the defaults - options = { - "num_days": 100, - "ruc_horizon": 12, - } - - testObject.load_prescient(data_path=input_data_source, options_dict=options) - - mock_prescient_config.set_value.assert_called_once() - passed_dict = mock_prescient_config.set_value.call_args[0][0] - self.assertEqual(passed_dict["data_path"], str(input_data_source)) - self.assertEqual(passed_dict["num_days"], 100) - self.assertEqual(passed_dict["ruc_horizon"], 12) - - def test_no_options_dict(self, mock_prescient_config): - - # Test not passing in an options dictionary - testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=input_data_source) - mock_prescient_config.set_value.assert_called_once() - passed_dict = mock_prescient_config.set_value.call_args[0][0] - # Default options should be set - self.assertEqual(passed_dict["data_path"], str(input_data_source)) - self.assertEqual(passed_dict["num_days"], 365) - self.assertEqual(passed_dict["ruc_horizon"], 36) - def test_default_representative_dates(self): # Test no representative dates passed in, initializing with defaults testObject = ExpansionPlanningData() @@ -206,41 +175,35 @@ def test_passed_representative_weights(self): ) self.assertEqual(testObject.representative_weights, weights) - # FIXME def test_in_service_flags_set(self): # Test in service flags are being set properly testObject = ExpansionPlanningData() testObject.load_prescient(data_path=input_data_source) - assert testObject.md.data["elements"]["generator"]["2-c"]["in_service"] == False - assert ( - testObject.md.data["elements"]["branch"]["branch2-c"]["in_service"] == False + self.assertEqual( + testObject.md.data["elements"]["generator"]["3_CT"]["in_service"], True + ) + self.assertEqual( + testObject.md.data["elements"]["branch"]["branch_3_4_1"]["in_service"], + True, ) - def test_missing_simulation_objects_csv(self, tmp_path): + def test_missing_simulation_objects_csv(self): # test if a data path is missing the simulation objects, it should throw an error - testObject = ExpansionPlanningData() - # tmp_path is empty, no simulation_objects.csv - with self.assertRaises(FileNotFoundError): - testObject.load_prescient(data_path=tmp_path) - - def test_incorrect_simulation_objects_csv(self, tmp_path): - # test if simulations_objects is missing key details, it should throw an error - csv_file = tmp_path / "simulation_objects.csv" - csv_file.write_text("index,DAY_AHEAD\nWrongKey,24\n") - - testObject = ExpansionPlanningData() - with self.assertRaises(KeyError): - testObject.load_prescient(data_path=tmp_path) + with tempfile.TemporaryDirectory() as tmpdirname: + testObject = ExpansionPlanningData() + with self.assertRaises(FileNotFoundError): + testObject.load_prescient(data_path=tmpdirname) - # FIXME def test_clone_at_time_keys_called_correctly(self): testObject = ExpansionPlanningData() testObject.load_prescient(data_path=input_data_source) + model_class = type(testObject.md) + # Patch clone_at_time_keys on the existing model instance with unittest.mock.patch.object( - testObject.md, "clone_at_time_keys", wraps=testObject.md.clone_at_time_keys + model_class, "clone_at_time_keys", wraps=testObject.md.clone_at_time_keys ) as mock_clone: # Call load_prescient again to trigger cloning with patched method @@ -248,23 +211,22 @@ def test_clone_at_time_keys_called_correctly(self): # Check that clone_at_time_keys was called once per representative date expected_calls = len(testObject.representative_dates) - assert mock_clone.call_count == expected_calls + self.assertEqual(mock_clone.call_count, expected_calls) # -------------------------------------------------IMPORT_LOAD_SCALING------------------------------------------------------------ # - # FIXME def test_import_load_scaling_normal(self): # test successful passthrough of load scaling function testObject = ExpansionPlanningData() testObject.import_load_scaling(load_scaling_file) df = testObject.load_scaling - assert isinstance(df, pd.DataFrame) + self.assertIsInstance(df, pd.DataFrame) expected_columns = ["year", "month", "day", "hour"] + [ str(i) for i in range(1, 9) ] for col in expected_columns: - assert col in df.columns - assert not df.empty + self.assertIn(col, df.columns) + self.assertFalse(df.empty) def test_import_load_scaling_incorrect_num_years(self): # Test value error raised if the length of forecast years is incorrect @@ -292,7 +254,8 @@ def test_import_outage_data(self): self.assertHasAttr(testObject, "bus_hours") self.assertIsInstance(df, pd.DataFrame) - self.assertIn(["hour", "Bus Number"], df.columns) + self.assertIn("hour", df.columns) + self.assertIn("Bus Number", df.columns) # -------------------------------------------------LOAD_DEFAULT_DATA_SETTINGS------------------------------------------------------------ # def test_load_default_data_settings(self): @@ -378,7 +341,7 @@ def test_load_storage_csv_file_not_found(self): # -------------------------------------------------TEXAS_CASE_STUDY_UPDATES----------------------------------------------------------- # def test_texas_case_study(self): testObject = ExpansionPlanningData() - testObject.load_prescient(data_path=input_data_source) + testObject.load_prescient(data_path=texas_data_path) # Call the method under test testObject.texas_case_study_updates(texas_data_path) @@ -412,5 +375,5 @@ def test_texas_case_study_invalid_data_path(self): # Test that an error is raised if not a Texas case Study testObject = ExpansionPlanningData() - with self.assertRaises(ValueError, match="not a Texas case study"): + with self.assertRaises(ValueError): testObject.texas_case_study_updates(input_data_source) From 60b9c44c77106b0ec5b4bbae5711139c36e600ac Mon Sep 17 00:00:00 2001 From: bstorm Date: Mon, 11 May 2026 08:55:12 -0600 Subject: [PATCH 20/24] add testing guards --- gtep/tests/unit/test_gtep_data.py | 45 ++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/gtep/tests/unit/test_gtep_data.py b/gtep/tests/unit/test_gtep_data.py index 9296ba9b..48cabc40 100644 --- a/gtep/tests/unit/test_gtep_data.py +++ b/gtep/tests/unit/test_gtep_data.py @@ -13,13 +13,13 @@ import pyomo.common.unittest as unittest -from unittest.mock import create_autospec, patch +from unittest.mock import patch import pytest from gtep.gtep_data import ExpansionPlanningData -import prescient.simulator.config import pandas as pd from pathlib import Path import tempfile +import os curr_dir = Path(__file__).resolve().parent input_data_source = (curr_dir / ".." / ".." / "data" / "5bus").resolve() @@ -37,19 +37,6 @@ ).resolve() -# creates a face simulations object file -@pytest.fixture -def inaccurate_simulation_objects_file(tmp_path): - """ - Creates a simulation_objects.csv file with incorrect content - inside a temporary directory and returns the directory path. - """ - csv_file = tmp_path / "simulation_objects.csv" - # Write inaccurate content (missing required keys) - csv_file.write_text("index,DAY_AHEAD\nWrongKey,24\n") - return tmp_path - - class TestExpansionPlanningData(unittest.TestCase): def test_data_init(self): @@ -214,6 +201,10 @@ def test_clone_at_time_keys_called_correctly(self): self.assertEqual(mock_clone.call_count, expected_calls) # -------------------------------------------------IMPORT_LOAD_SCALING------------------------------------------------------------ # + @pytest.mark.skipif( + not os.path.exists(load_scaling_file), + reason=f"Data file {load_scaling_file} not found", + ) def test_import_load_scaling_normal(self): # test successful passthrough of load scaling function testObject = ExpansionPlanningData() @@ -228,6 +219,10 @@ def test_import_load_scaling_normal(self): self.assertIn(col, df.columns) self.assertFalse(df.empty) + @pytest.mark.skipif( + not os.path.exists(load_scaling_file), + reason=f"Data file {load_scaling_file} not found", + ) def test_import_load_scaling_incorrect_num_years(self): # Test value error raised if the length of forecast years is incorrect testObject = ExpansionPlanningData(stages=3) @@ -236,6 +231,10 @@ def test_import_load_scaling_incorrect_num_years(self): with self.assertRaises(ValueError): testObject.import_load_scaling(load_scaling_file, forecast_years) + @pytest.mark.skipif( + not os.path.exists(load_scaling_file), + reason=f"Data file {load_scaling_file} not found", + ) def test_import_load_scaling_incorrect_years_too_early(self): # Test value error raised if the forecast years are outside the supported ranges testObject = ExpansionPlanningData(stages=3) @@ -245,6 +244,10 @@ def test_import_load_scaling_incorrect_years_too_early(self): testObject.import_load_scaling(load_scaling_file, forecast_years) # -------------------------------------------------IMPORT_OUTAGE_DATA------------------------------------------------------------ # + @pytest.mark.skipif( + not os.path.exists(outage_data_path), + reason=f"Data file {outage_data_path} not found", + ) def test_import_outage_data(self): testObject = ExpansionPlanningData() @@ -300,6 +303,10 @@ def test_load_default_data_settings(self): self.assertEqual(system["min_spinning_reserve"], 0.1) # -------------------------------------------------LOAD_STORAGE_CSV------------------------------------------------------------ # + @pytest.mark.skipif( + not os.path.exists(storage_file), + reason=f"Data file {storage_file} not found", + ) def test_load_storage_csv_success(self): testObject = ExpansionPlanningData() testObject.load_prescient(data_path=input_data_source) @@ -323,6 +330,10 @@ def test_load_storage_csv_success(self): for key in expected_keys: assert key in storage["100MW_400MWh_1"].keys() + @pytest.mark.skipif( + not os.path.exists(storage_file), + reason=f"Data file {storage_file} not found", + ) def test_load_storage_string_path(self): testObject = ExpansionPlanningData() testObject.load_prescient(data_path=input_data_source) @@ -339,6 +350,10 @@ def test_load_storage_csv_file_not_found(self): self.assertEqual(storage, {}) # -------------------------------------------------TEXAS_CASE_STUDY_UPDATES----------------------------------------------------------- # + @pytest.mark.skipif( + not os.path.exists(texas_data_path), + reason=f"Data file {texas_data_path} not found", + ) def test_texas_case_study(self): testObject = ExpansionPlanningData() testObject.load_prescient(data_path=texas_data_path) From 8ed540c205f7ed8efd62f1714b58689d95aeb1ef Mon Sep 17 00:00:00 2001 From: blnicho Date: Mon, 11 May 2026 14:43:29 -0600 Subject: [PATCH 21/24] Add missing dependency --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 6515abb0..d1f2d6b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,7 @@ dependencies = [ "matplotlib", "ipython", "openpyxl", + "pyxlsb", ] [project.urls] From 11e9f8a7e863c645d88c05753fdd898842a52d79 Mon Sep 17 00:00:00 2001 From: bstorm Date: Mon, 11 May 2026 15:18:04 -0600 Subject: [PATCH 22/24] fix attribute failure for outage --- gtep/tests/unit/test_gtep_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gtep/tests/unit/test_gtep_data.py b/gtep/tests/unit/test_gtep_data.py index 48cabc40..dcc78b82 100644 --- a/gtep/tests/unit/test_gtep_data.py +++ b/gtep/tests/unit/test_gtep_data.py @@ -255,7 +255,7 @@ def test_import_outage_data(self): df = testObject.bus_hours - self.assertHasAttr(testObject, "bus_hours") + self.assertTrue(hasattr(testObject, "bus_hours")) self.assertIsInstance(df, pd.DataFrame) self.assertIn("hour", df.columns) self.assertIn("Bus Number", df.columns) From fd77c9fb7c015e02eabfdf25032615dd1cfe7c46 Mon Sep 17 00:00:00 2001 From: bstorm Date: Mon, 11 May 2026 15:19:32 -0600 Subject: [PATCH 23/24] remove print statement --- gtep/gtep_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 10e7ef65..454bc81f 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -182,7 +182,6 @@ def import_load_scaling(self, load_file_name, forecast_years=None): """ adjusted_forecast = pd.read_excel(load_file_name) - print((adjusted_forecast["year"]).unique) if forecast_years is None: forecast_years = [2025, 2030, 2035] From 330d5a005fdd57bf19674d661c4cd88cf8e1c4e4 Mon Sep 17 00:00:00 2001 From: bstorm Date: Thu, 21 May 2026 11:50:19 -0600 Subject: [PATCH 24/24] update based on PR notes --- gtep/gtep_data.py | 12 +++++++++--- gtep/tests/unit/test_gtep_data.py | 17 ++++++++++------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 454bc81f..8c48ad68 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -57,13 +57,13 @@ def load_prescient( self, data_path, representative_dates=None, - representative_weights={}, + representative_weights=None, options_dict=None, ): """Loads data structured via Prescient data loader. :param data_path: Folder containing the data to be loaded - :param representative_dates: List of time points to include. Note: Change the last date for whatever extreme day is needed based on the given run(s) + :param representative_dates: List of time points to include. :param representative_weights: dictionary of weights for each representative date, defaults to empty Dict :param options_dict: Options dictionary to pass to the Prescient data loader, defaults to None @@ -153,10 +153,16 @@ def load_prescient( "2020-07-05 00:00", "2020-10-14 00:00", ## Change the last date for whatever extreme day is needed based on the given run(s) ] + #enforce representative dates as a list + if not isinstance(representative_dates, list): + representative_dates = [representative_dates] + #check that the representative dates has at least one value + if not len(representative_dates) >= 1: + raise ValueError('Invalid input for representative_dates. representative_dates should be a list of date strings') self.representative_dates = representative_dates self.representative_weights = representative_weights - if not representative_weights: + if representative_weights is None: # set the weight for each day to the total weight divided by number of days total_weight = prescient_options.num_days * self.stages weight_per_date = int(total_weight / (len(representative_dates))) diff --git a/gtep/tests/unit/test_gtep_data.py b/gtep/tests/unit/test_gtep_data.py index dcc78b82..1eaab4cd 100644 --- a/gtep/tests/unit/test_gtep_data.py +++ b/gtep/tests/unit/test_gtep_data.py @@ -68,7 +68,7 @@ def test_data_init(self): self.assertEqual(testObject.num_dispatch, 1) self.assertEqual(testObject.duration_dispatch, 15) - # -------------------------------------------------LOAD_PRESCIENT------------------------------------------------------------ # + ### LOAD_PRESCIENT ### def test_default_representative_dates(self): # Test no representative dates passed in, initializing with defaults testObject = ExpansionPlanningData() @@ -200,7 +200,7 @@ def test_clone_at_time_keys_called_correctly(self): expected_calls = len(testObject.representative_dates) self.assertEqual(mock_clone.call_count, expected_calls) - # -------------------------------------------------IMPORT_LOAD_SCALING------------------------------------------------------------ # + ### IMPORT_LOAD_SCALING ### @pytest.mark.skipif( not os.path.exists(load_scaling_file), reason=f"Data file {load_scaling_file} not found", @@ -243,7 +243,7 @@ def test_import_load_scaling_incorrect_years_too_early(self): with self.assertRaises(ValueError): testObject.import_load_scaling(load_scaling_file, forecast_years) - # -------------------------------------------------IMPORT_OUTAGE_DATA------------------------------------------------------------ # + ### IMPORT_OUTAGE_DATA ### @pytest.mark.skipif( not os.path.exists(outage_data_path), reason=f"Data file {outage_data_path} not found", @@ -260,7 +260,7 @@ def test_import_outage_data(self): self.assertIn("hour", df.columns) self.assertIn("Bus Number", df.columns) - # -------------------------------------------------LOAD_DEFAULT_DATA_SETTINGS------------------------------------------------------------ # + ### LOAD_DEFAULT_DATA_SETTINGS ### def test_load_default_data_settings(self): testObject = ExpansionPlanningData() testObject.load_prescient(data_path=input_data_source) @@ -302,7 +302,7 @@ def test_load_default_data_settings(self): self.assertEqual(system["min_operating_reserve"], 0.1) self.assertEqual(system["min_spinning_reserve"], 0.1) - # -------------------------------------------------LOAD_STORAGE_CSV------------------------------------------------------------ # + ### LOAD_STORAGE_CSV ### @pytest.mark.skipif( not os.path.exists(storage_file), reason=f"Data file {storage_file} not found", @@ -328,7 +328,7 @@ def test_load_storage_csv_success(self): "investment_cost_kwh", } for key in expected_keys: - assert key in storage["100MW_400MWh_1"].keys() + self.assertin(key ,storage["100MW_400MWh_1"].keys()) @pytest.mark.skipif( not os.path.exists(storage_file), @@ -338,6 +338,9 @@ def test_load_storage_string_path(self): testObject = ExpansionPlanningData() testObject.load_prescient(data_path=input_data_source) testObject.load_storage_csv(str(storage_file)) # should not throw an error + self.assertin('storage' ,testObject.md.data["elements"].keys()) + #check that the storage data is not an empty dict + self.assertTrue(testObject.md.data["elements"]['storage']) def test_load_storage_csv_file_not_found(self): testObject = ExpansionPlanningData() @@ -349,7 +352,7 @@ def test_load_storage_csv_file_not_found(self): self.assertIsInstance(storage, dict) self.assertEqual(storage, {}) - # -------------------------------------------------TEXAS_CASE_STUDY_UPDATES----------------------------------------------------------- # + ### TEXAS_CASE_STUDY_UPDATES ### @pytest.mark.skipif( not os.path.exists(texas_data_path), reason=f"Data file {texas_data_path} not found",