From 5b968e464a3382029768a1b5c605df6652d17ea3 Mon Sep 17 00:00:00 2001 From: Meyers-Im Date: Mon, 28 Aug 2023 15:29:52 -0700 Subject: [PATCH 1/9] switch default to clarabel solver --- requirements.txt | 9 ++++----- solardatatools/data_handler.py | 26 +++++++++++++------------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/requirements.txt b/requirements.txt index f06e043b..1c48ac49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,8 +12,7 @@ cmake pykml haversine boto3 -# I suggest using the MOSEK convex solver with cvxpy, if possible. For additional information about setting up MOSEK, -# see here: https://docs.mosek.com/8.1/pythonapi/install-interface.html. A license is required to use this software. -# Academic license info: https://www.mosek.com/products/academic-licenses/ -# Trial commercial license info: https://www.mosek.com/products/trial/ -Mosek +sig-decomp +osqp +clarabel +qss diff --git a/solardatatools/data_handler.py b/solardatatools/data_handler.py index c38d4af6..9d47ae54 100644 --- a/solardatatools/data_handler.py +++ b/solardatatools/data_handler.py @@ -188,7 +188,7 @@ def run_pipeline( daytime_threshold=0.005, units="W", solver="QSS", - solver_convex="OSQP", + solver_convex="CLARABEL", reset=True, ): if solver == "MOSEK": @@ -470,7 +470,9 @@ def run_pipeline( ) # Update daytime detection based on cleaned up data - self.daytime_analysis.calculate_times(self.filled_data_matrix, solver=solver_convex) + self.daytime_analysis.calculate_times( + self.filled_data_matrix, solver=solver_convex + ) self.boolean_masks.daytime = self.daytime_analysis.sunup_mask_estimated ###################################################################### # Process Extra columns @@ -892,7 +894,7 @@ def capacity_clustering( self.filled_data_matrix, filter=self.daily_flags.no_errors, quantile=1.00, - w1=40e-6, # scaled weights for QSS + w1=40e-6, # scaled weights for QSS w2=6561e-6, solver=solver, ) @@ -919,7 +921,7 @@ def capacity_clustering( gridspec_kw={"height_ratios": [4, 1]}, ) ax[0].plot(xs, s1, label="capacity change detector") - ax[0].plot(xs, s1+s2+s3, label="signal model") + ax[0].plot(xs, s1 + s2 + s3, label="signal model") ax[0].plot(xs, metric, alpha=0.3, label="measured signal") ax[0].legend() ax[0].set_title("Detection of system capacity changes") @@ -930,7 +932,7 @@ def capacity_clustering( else: fig, ax = plt.subplots(nrows=1, figsize=figsize) ax.plot(xs, s1, label="capacity change detector") - ax.plot(xs, s1+s2+s3, label="signal model") + ax.plot(xs, s1 + s2 + s3, label="signal model") ax.plot(xs, metric, alpha=0.3, label="measured signal") ax.legend() ax.set_title("Detection of system capacity changes") @@ -948,8 +950,8 @@ def auto_fix_time_shifts( solver=None, ): def max_min_scale(signal): - maximum = np.nanquantile(signal, .95) - minimum = np.nanquantile(signal, .05) + maximum = np.nanquantile(signal, 0.95) + minimum = np.nanquantile(signal, 0.05) return (signal - minimum) / (maximum - minimum), minimum, maximum def rescale_signal(signal, minimum, maximum): @@ -980,7 +982,7 @@ def rescale_signal(signal, minimum, maximum): threshold=threshold, periodic_detector=periodic_detector, solver=solver, - sum_card=False + sum_card=False, ) if solver == "QSS": @@ -995,7 +997,7 @@ def rescale_signal(signal, minimum, maximum): threshold=threshold, periodic_detector=periodic_detector, solver=solver, - sum_card=True + sum_card=True, ) else: # If solver is QSS, run with nonconvex formulation again @@ -1009,14 +1011,12 @@ def rescale_signal(signal, minimum, maximum): threshold=threshold, periodic_detector=periodic_detector, solver=solver, - sum_card=True + sum_card=True, ) # Scale data back self.filled_data_matrix = rescale_signal( - self.time_shift_analysis.corrected_data, - min_metric, - max_metric + self.time_shift_analysis.corrected_data, min_metric, max_metric ) if len(self.time_shift_analysis.index_set) == 0: self.time_shifts = False From 5ae2839b3eb148d9b6c89d8b194b4db726cc0be2 Mon Sep 17 00:00:00 2001 From: Meyers-Im Date: Mon, 25 Sep 2023 11:06:48 -0700 Subject: [PATCH 2/9] modify tests to use clarabel --- .../test_cvx_signal_decompositions.py | 305 ++++++++++++------ 1 file changed, 210 insertions(+), 95 deletions(-) diff --git a/tests/solardatatools/test_cvx_signal_decompositions.py b/tests/solardatatools/test_cvx_signal_decompositions.py index b54a31d0..482701b1 100644 --- a/tests/solardatatools/test_cvx_signal_decompositions.py +++ b/tests/solardatatools/test_cvx_signal_decompositions.py @@ -57,9 +57,8 @@ class TestSignalDecompositions(unittest.TestCase): - def setUp(self): - self.cvxpy_solver = "MOSEK" # all tests are using MOSEK + self.cvxpy_solver = "CLARABEL" # all tests are using MOSEK self.mae_threshold = 0.001 self.obj_tolerance = 1 @@ -72,12 +71,21 @@ def test_cxv_l2_l1d1_l2d2p365_default(self): # Load input and output data filepath = Path(__file__).parent.parent - data_file_path = (filepath / "fixtures" / "signal_decompositions" / "_cvx_signal_decompositions") + data_file_path = ( + filepath + / "fixtures" + / "signal_decompositions" + / "_cvx_signal_decompositions" + ) - input_path = str(data_file_path) + "/" + "test_cvx_l2_l1d1_l2d2p365_default_input.json" - output_path = str(data_file_path) + "/" + "test_cvx_l2_l1d1_l2d2p365_default_output.json" + input_path = ( + str(data_file_path) + "/" + "test_cvx_l2_l1d1_l2d2p365_default_input.json" + ) + output_path = ( + str(data_file_path) + "/" + "test_cvx_l2_l1d1_l2d2p365_default_output.json" + ) - # Load input + # Load input with open(input_path) as f: input = json.load(f) @@ -95,11 +103,7 @@ def test_cxv_l2_l1d1_l2d2p365_default(self): # Run test actual_s_hat, actual_s_seas, _, actual_obj_val = sd._cvx_l2_l1d1_l2d2p365( - signal, - w1=10, - w2=1e5, - solver=self.cvxpy_solver, - return_all=True + signal, w1=10, w2=1e5, solver=self.cvxpy_solver, return_all=True ) mae_s_hat = mae(actual_s_hat, expected_s_hat) @@ -109,16 +113,28 @@ def test_cxv_l2_l1d1_l2d2p365_default(self): self.assertLess(mae_s_seas, self.mae_threshold) self.assertAlmostEqual(expected_obj_val, actual_obj_val, self.obj_tolerance) - def test_cvx_l2_l1d1_l2d2p365_transition(self): """Test with piecewise fn transition location""" # Load input and output data filepath = Path(__file__).parent.parent - data_file_path = (filepath / "fixtures" / "signal_decompositions" / "_cvx_signal_decompositions") + data_file_path = ( + filepath + / "fixtures" + / "signal_decompositions" + / "_cvx_signal_decompositions" + ) - input_path = str(data_file_path) + "/" + "test_cvx_l2_l1d1_l2d2p365_transition_input.json" - output_path = str(data_file_path) + "/" + "test_cvx_l2_l1d1_l2d2p365_transition_output.json" + input_path = ( + str(data_file_path) + + "/" + + "test_cvx_l2_l1d1_l2d2p365_transition_input.json" + ) + output_path = ( + str(data_file_path) + + "/" + + "test_cvx_l2_l1d1_l2d2p365_transition_output.json" + ) # Load input with open(input_path) as f: @@ -139,11 +155,7 @@ def test_cvx_l2_l1d1_l2d2p365_transition(self): # Run test actual_s_hat, actual_s_seas, _, actual_obj_val = sd._cvx_l2_l1d1_l2d2p365( - signal, - w1=10, - w2=1e5, - transition_locs=indices, - return_all=True + signal, w1=10, w2=1e5, transition_locs=indices, return_all=True ) mae_s_hat = mae(actual_s_hat, expected_s_hat) @@ -158,10 +170,23 @@ def test_cvx_l2_l1d1_l2d2p365_transition_wrong(self): # Load input and output data filepath = Path(__file__).parent.parent - data_file_path = (filepath / "fixtures" / "signal_decompositions" / "_cvx_signal_decompositions") + data_file_path = ( + filepath + / "fixtures" + / "signal_decompositions" + / "_cvx_signal_decompositions" + ) - input_path = str(data_file_path) + "/" + "test_cvx_l2_l1d1_l2d2p365_transition_wrong_input.json" - output_path = str(data_file_path) + "/" + "test_cvx_l2_l1d1_l2d2p365_transition_wrong_output.json" + input_path = ( + str(data_file_path) + + "/" + + "test_cvx_l2_l1d1_l2d2p365_transition_wrong_input.json" + ) + output_path = ( + str(data_file_path) + + "/" + + "test_cvx_l2_l1d1_l2d2p365_transition_wrong_output.json" + ) # Load input with open(input_path) as f: @@ -173,7 +198,7 @@ def test_cvx_l2_l1d1_l2d2p365_transition_wrong(self): # Input signal = np.array(input["test_signal"]) - transition = input["indices"] + transition = input["indices"] # Expected output expected_s_hat = output["expected_s_hat_mosek_transition_wrong_365"] @@ -182,11 +207,7 @@ def test_cvx_l2_l1d1_l2d2p365_transition_wrong(self): # Run test actual_s_hat, actual_s_seas, _, actual_obj_val = sd._cvx_l2_l1d1_l2d2p365( - signal, - w1=10, - w2=1e5, - transition_locs=transition, - return_all=True + signal, w1=10, w2=1e5, transition_locs=transition, return_all=True ) mae_s_hat = mae(actual_s_hat, expected_s_hat) @@ -201,10 +222,23 @@ def test_cvx_l2_l1d1_l2d2p365_default_long(self): # Load input and output data filepath = Path(__file__).parent.parent - data_file_path = (filepath / "fixtures" / "signal_decompositions" / "_cvx_signal_decompositions") + data_file_path = ( + filepath + / "fixtures" + / "signal_decompositions" + / "_cvx_signal_decompositions" + ) - input_path = str(data_file_path) + "/" + "test_cvx_l2_l1d1_l2d2p365_default_long_input.json" - output_path = str(data_file_path) + "/" + "test_cvx_l2_l1d1_l2d2p365_default_long_output.json" + input_path = ( + str(data_file_path) + + "/" + + "test_cvx_l2_l1d1_l2d2p365_default_long_input.json" + ) + output_path = ( + str(data_file_path) + + "/" + + "test_cvx_l2_l1d1_l2d2p365_default_long_output.json" + ) # Load input with open(input_path) as f: @@ -224,11 +258,7 @@ def test_cvx_l2_l1d1_l2d2p365_default_long(self): # Run test actual_s_hat, actual_s_seas, _, actual_obj_val = sd._cvx_l2_l1d1_l2d2p365( - signal, - w1=10, - w2=1e5, - solver=self.cvxpy_solver, - return_all=True + signal, w1=10, w2=1e5, solver=self.cvxpy_solver, return_all=True ) mae_s_hat = mae(actual_s_hat, expected_s_hat) @@ -243,10 +273,23 @@ def test_cvx_l2_l1d1_l2d2p365_idx_select(self): # Load input and output data filepath = Path(__file__).parent.parent - data_file_path = (filepath / "fixtures" / "signal_decompositions" / "_cvx_signal_decompositions") + data_file_path = ( + filepath + / "fixtures" + / "signal_decompositions" + / "_cvx_signal_decompositions" + ) - input_path = str(data_file_path) + "/" + "test_cvx_l2_l1d1_l2d2p365_idx_select_input.json" - output_path = str(data_file_path) + "/" + "test_cvx_l2_l1d1_l2d2p365_idx_select_output.json" + input_path = ( + str(data_file_path) + + "/" + + "test_cvx_l2_l1d1_l2d2p365_idx_select_input.json" + ) + output_path = ( + str(data_file_path) + + "/" + + "test_cvx_l2_l1d1_l2d2p365_idx_select_output.json" + ) # Load input with open(input_path) as f: @@ -272,7 +315,7 @@ def test_cvx_l2_l1d1_l2d2p365_idx_select(self): w2=1e5, solver=self.cvxpy_solver, use_ixs=indices, - return_all=True + return_all=True, ) mae_s_hat = mae(actual_s_hat, expected_s_hat) @@ -287,10 +330,23 @@ def test_cvx_l2_l1d1_l2d2p365_yearly_periodic(self): # Load input and output data filepath = Path(__file__).parent.parent - data_file_path = (filepath / "fixtures" / "signal_decompositions" / "_cvx_signal_decompositions") + data_file_path = ( + filepath + / "fixtures" + / "signal_decompositions" + / "_cvx_signal_decompositions" + ) - input_path = str(data_file_path) + "/" + "test_cvx_l2_l1d1_l2d2p365_yearly_periodic_input.json" - output_path = str(data_file_path) + "/" + "test_cvx_l2_l1d1_l2d2p365_yearly_periodic_output.json" + input_path = ( + str(data_file_path) + + "/" + + "test_cvx_l2_l1d1_l2d2p365_yearly_periodic_input.json" + ) + output_path = ( + str(data_file_path) + + "/" + + "test_cvx_l2_l1d1_l2d2p365_yearly_periodic_output.json" + ) # Load input with open(input_path) as f: @@ -315,7 +371,7 @@ def test_cvx_l2_l1d1_l2d2p365_yearly_periodic(self): w2=1e5, solver=self.cvxpy_solver, yearly_periodic=True, - return_all=True + return_all=True, ) mae_s_hat = mae(actual_s_hat, expected_s_hat) @@ -325,7 +381,6 @@ def test_cvx_l2_l1d1_l2d2p365_yearly_periodic(self): self.assertLess(mae_s_seas, self.mae_threshold) self.assertAlmostEqual(expected_obj_val, actual_obj_val, self.obj_tolerance) - ################### # _cvx_tl1_l2d2p365 ################### @@ -335,10 +390,19 @@ def test_cvx_tl1_l2d2p365_default(self): # Load input and output data filepath = Path(__file__).parent.parent - data_file_path = (filepath / "fixtures" / "signal_decompositions"/ "_cvx_signal_decompositions") + data_file_path = ( + filepath + / "fixtures" + / "signal_decompositions" + / "_cvx_signal_decompositions" + ) - input_path = str(data_file_path) + "/" + "test_cvx_tl1_l2d2p365_default_input.json" - output_path = str(data_file_path) + "/" + "test_cvx_tl1_l2d2p365_default_output.json" + input_path = ( + str(data_file_path) + "/" + "test_cvx_tl1_l2d2p365_default_input.json" + ) + output_path = ( + str(data_file_path) + "/" + "test_cvx_tl1_l2d2p365_default_output.json" + ) # Load input with open(input_path) as f: @@ -356,11 +420,9 @@ def test_cvx_tl1_l2d2p365_default(self): expected_obj_val = output["expected_obj_val_mosek_365"] # Run test with default args - actual_s_seas, actual_obj_val = sd._cvx_tl1_l2d2p365(signal, - tau=0.8, - w1=1e5, - solver=self.cvxpy_solver, - return_all=True) + actual_s_seas, actual_obj_val = sd._cvx_tl1_l2d2p365( + signal, tau=0.8, w1=1e5, solver=self.cvxpy_solver, return_all=True + ) mae_s_seas = mae(actual_s_seas, expected_s_seas) @@ -372,10 +434,19 @@ def test_cvx_tl1_l2d2p365_idx_select(self): # Load input and output data filepath = Path(__file__).parent.parent - data_file_path = (filepath / "fixtures" / "signal_decompositions" / "_cvx_signal_decompositions") + data_file_path = ( + filepath + / "fixtures" + / "signal_decompositions" + / "_cvx_signal_decompositions" + ) - input_path = str(data_file_path) + "/" + "test_cvx_tl1_l2d2p365_idx_select_input.json" - output_path = str(data_file_path) + "/" + "test_cvx_tl1_l2d2p365_idx_select_output.json" + input_path = ( + str(data_file_path) + "/" + "test_cvx_tl1_l2d2p365_idx_select_input.json" + ) + output_path = ( + str(data_file_path) + "/" + "test_cvx_tl1_l2d2p365_idx_select_output.json" + ) # Load input with open(input_path) as f: @@ -394,12 +465,14 @@ def test_cvx_tl1_l2d2p365_idx_select(self): expected_obj_val = output["expected_obj_val_mosek_ixs"] # Run test - actual_s_seas, actual_obj_val = sd._cvx_tl1_l2d2p365(signal, - tau=0.8, - w1=1e5, - solver=self.cvxpy_solver, - use_ixs=indices, - return_all=True) + actual_s_seas, actual_obj_val = sd._cvx_tl1_l2d2p365( + signal, + tau=0.8, + w1=1e5, + solver=self.cvxpy_solver, + use_ixs=indices, + return_all=True, + ) mae_s_seas = mae(actual_s_seas, expected_s_seas) @@ -411,10 +484,23 @@ def test_cxv_tl1_l2d2p365_long_not_yearly_periodic(self): # Load input and output data filepath = Path(__file__).parent.parent - data_file_path = (filepath / "fixtures" / "signal_decompositions" / "_cvx_signal_decompositions") + data_file_path = ( + filepath + / "fixtures" + / "signal_decompositions" + / "_cvx_signal_decompositions" + ) - input_path = str(data_file_path) + "/" + "test_cvx_tl1_l2d2p365_long_not_yearly_periodic_input.json" - output_path = str(data_file_path) + "/" + "test_cvx_tl1_l2d2p365_long_not_yearly_periodic_output.json" + input_path = ( + str(data_file_path) + + "/" + + "test_cvx_tl1_l2d2p365_long_not_yearly_periodic_input.json" + ) + output_path = ( + str(data_file_path) + + "/" + + "test_cvx_tl1_l2d2p365_long_not_yearly_periodic_output.json" + ) # Load input with open(input_path) as f: @@ -432,19 +518,20 @@ def test_cxv_tl1_l2d2p365_long_not_yearly_periodic(self): expected_obj_val = output["expected_obj_val_mosek_yearly_periodic"] # Run test with default args - actual_s_seas, actual_obj_val = sd._cvx_tl1_l2d2p365(signal, - tau=0.8, - solver=self.cvxpy_solver, - w1=1e5, - yearly_periodic=False, - return_all=True) + actual_s_seas, actual_obj_val = sd._cvx_tl1_l2d2p365( + signal, + tau=0.8, + solver=self.cvxpy_solver, + w1=1e5, + yearly_periodic=False, + return_all=True, + ) mae_s_seas = mae(actual_s_seas, expected_s_seas) self.assertLess(mae_s_seas, self.mae_threshold) self.assertAlmostEqual(expected_obj_val, actual_obj_val, self.obj_tolerance) - ####################### # _cvx_l1_l1d1_l2d2p365 ####################### @@ -454,10 +541,19 @@ def test_cvx_l1_l1d1_l2d2p365_default(self): # Load input and output data filepath = Path(__file__).parent.parent - data_file_path = (filepath / "fixtures" / "signal_decompositions" / "_cvx_signal_decompositions") + data_file_path = ( + filepath + / "fixtures" + / "signal_decompositions" + / "_cvx_signal_decompositions" + ) - input_path = str(data_file_path) + "/" + "test_cvx_l1_l1d1_l2d2p365_default_input.json" - output_path = str(data_file_path) + "/" + "test_cvx_l1_l1d1_l2d2p365_default_output.json" + input_path = ( + str(data_file_path) + "/" + "test_cvx_l1_l1d1_l2d2p365_default_input.json" + ) + output_path = ( + str(data_file_path) + "/" + "test_cvx_l1_l1d1_l2d2p365_default_output.json" + ) # Load input with open(input_path) as f: @@ -477,11 +573,7 @@ def test_cvx_l1_l1d1_l2d2p365_default(self): # Run test with default args actual_s_hat, actual_s_seas, _, actual_obj_val = sd._cvx_l1_l1d1_l2d2p365( - signal, - w1=5, - w2=1e5, - solver=self.cvxpy_solver, - return_all=True + signal, w1=5, w2=1e5, solver=self.cvxpy_solver, return_all=True ) mae_s_hat = mae(actual_s_hat, expected_s_hat) @@ -496,10 +588,23 @@ def test_cvx_l1_l1d1_l2d2p365_idx_select(self): # Load input and output data filepath = Path(__file__).parent.parent - data_file_path = (filepath / "fixtures" / "signal_decompositions" / "_cvx_signal_decompositions") + data_file_path = ( + filepath + / "fixtures" + / "signal_decompositions" + / "_cvx_signal_decompositions" + ) - input_path = str(data_file_path) + "/" + "test_cvx_l1_l1d1_l2d2p365_idx_select_input.json" - output_path = str(data_file_path) + "/" + "test_cvx_l1_l1d1_l2d2p365_idx_select_output.json" + input_path = ( + str(data_file_path) + + "/" + + "test_cvx_l1_l1d1_l2d2p365_idx_select_input.json" + ) + output_path = ( + str(data_file_path) + + "/" + + "test_cvx_l1_l1d1_l2d2p365_idx_select_output.json" + ) # Load input with open(input_path) as f: @@ -525,7 +630,7 @@ def test_cvx_l1_l1d1_l2d2p365_idx_select(self): w2=1e5, solver=self.cvxpy_solver, use_ixs=indices, - return_all = True + return_all=True, ) mae_s_hat = mae(actual_s_hat, expected_s_hat) @@ -535,7 +640,6 @@ def test_cvx_l1_l1d1_l2d2p365_idx_select(self): self.assertLess(mae_s_seas, self.mae_threshold) self.assertAlmostEqual(expected_obj_val, actual_obj_val, self.obj_tolerance) - ############################### # _cvx_l2_l1d2_constrained ############################### @@ -545,10 +649,23 @@ def test_cvx_l2_l1d2_constrained_default(self): # Load input and output data filepath = Path(__file__).parent.parent - data_file_path = (filepath / "fixtures" / "signal_decompositions" / "_cvx_signal_decompositions") + data_file_path = ( + filepath + / "fixtures" + / "signal_decompositions" + / "_cvx_signal_decompositions" + ) - input_path = str(data_file_path) + "/" + "test_cvx_l2_l1d2_constrained_default_input.json" - output_path = str(data_file_path) + "/" + "test_cvx_l2_l1d2_constrained_default_output.json" + input_path = ( + str(data_file_path) + + "/" + + "test_cvx_l2_l1d2_constrained_default_input.json" + ) + output_path = ( + str(data_file_path) + + "/" + + "test_cvx_l2_l1d2_constrained_default_output.json" + ) # Load input with open(input_path) as f: @@ -567,10 +684,7 @@ def test_cvx_l2_l1d2_constrained_default(self): # Run test with default args actual_y_hat, actual_obj_val = sd._cvx_l2_l1d2_constrained( - signal, - w1=1e1, - solver=self.cvxpy_solver, - return_all=True + signal, w1=1e1, solver=self.cvxpy_solver, return_all=True ) mae_y_hat = mae(actual_y_hat, expected_y_hat) @@ -578,5 +692,6 @@ def test_cvx_l2_l1d2_constrained_default(self): self.assertLess(mae_y_hat, self.mae_threshold) self.assertAlmostEqual(expected_obj_val, actual_obj_val, self.obj_tolerance) -if __name__ == '__main__': - unittest.main() \ No newline at end of file + +if __name__ == "__main__": + unittest.main() From 1edaa9b36d6cdce913ac3e8a3d9643c91f8732e7 Mon Sep 17 00:00:00 2001 From: Meyers-Im Date: Mon, 25 Sep 2023 11:16:04 -0700 Subject: [PATCH 3/9] updating tests --- .../test_cvx_signal_decompositions.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/solardatatools/test_cvx_signal_decompositions.py b/tests/solardatatools/test_cvx_signal_decompositions.py index 482701b1..1607599f 100644 --- a/tests/solardatatools/test_cvx_signal_decompositions.py +++ b/tests/solardatatools/test_cvx_signal_decompositions.py @@ -59,7 +59,7 @@ class TestSignalDecompositions(unittest.TestCase): def setUp(self): self.cvxpy_solver = "CLARABEL" # all tests are using MOSEK - self.mae_threshold = 0.001 + self.mae_threshold = 0.003 self.obj_tolerance = 1 ####################### @@ -155,7 +155,12 @@ def test_cvx_l2_l1d1_l2d2p365_transition(self): # Run test actual_s_hat, actual_s_seas, _, actual_obj_val = sd._cvx_l2_l1d1_l2d2p365( - signal, w1=10, w2=1e5, transition_locs=indices, return_all=True + signal, + w1=10, + w2=1e5, + transition_locs=indices, + solver=self.cvxpy_solver, + return_all=True, ) mae_s_hat = mae(actual_s_hat, expected_s_hat) @@ -207,7 +212,12 @@ def test_cvx_l2_l1d1_l2d2p365_transition_wrong(self): # Run test actual_s_hat, actual_s_seas, _, actual_obj_val = sd._cvx_l2_l1d1_l2d2p365( - signal, w1=10, w2=1e5, transition_locs=transition, return_all=True + signal, + w1=10, + w2=1e5, + transition_locs=transition, + solver=self.cvxpy_solver, + return_all=True, ) mae_s_hat = mae(actual_s_hat, expected_s_hat) From 45518a5ef0b4609354d3271ab64435806a52cffd Mon Sep 17 00:00:00 2001 From: Meyers-Im Date: Mon, 25 Sep 2023 16:15:57 -0700 Subject: [PATCH 4/9] fixing imports of test fixtures --- tests/solardatatools/test_data_handler.py | 2 +- tests/solardatatools/test_system_profiler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/solardatatools/test_data_handler.py b/tests/solardatatools/test_data_handler.py index 77311d70..e8258a99 100644 --- a/tests/solardatatools/test_data_handler.py +++ b/tests/solardatatools/test_data_handler.py @@ -10,7 +10,7 @@ class TestDataHandler(unittest.TestCase): def test_load_and_run(self): filepath = Path(__file__).parent.parent data_file_path = filepath / "fixtures" / "data_transforms" / "timeseries.csv" - df = pd.read_csv(data_file_path, index_col=0, parse_dates=True) + df = pd.read_csv(data_file_path, parse_dates=[1], index_col=1) dh = DataHandler(df) dh.run_pipeline(verbose=False) # dh.report() diff --git a/tests/solardatatools/test_system_profiler.py b/tests/solardatatools/test_system_profiler.py index fa3efd7c..39f8a8df 100644 --- a/tests/solardatatools/test_system_profiler.py +++ b/tests/solardatatools/test_system_profiler.py @@ -11,7 +11,7 @@ def test_system_profiler(self): data_file_path = ( filepath / "fixtures" / "system_profiler" / "data_handler_input.csv" ) - data = pd.read_csv(data_file_path, index_col=0, parse_dates=True) + data = pd.read_csv(data_file_path, parse_dates=[1], index_col=1) dh = DataHandler(data, datetime_col="Date-Time") dh.fix_dst() dh.run_pipeline( From 208a5262b6e2abbc409691b1179fae8fb7f62617 Mon Sep 17 00:00:00 2001 From: Meyers-Im Date: Mon, 25 Sep 2023 16:28:34 -0700 Subject: [PATCH 5/9] test fixing... --- tests/solardatatools/test_data_handler.py | 3 ++- tests/solardatatools/test_system_profiler.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/solardatatools/test_data_handler.py b/tests/solardatatools/test_data_handler.py index e8258a99..3af06e75 100644 --- a/tests/solardatatools/test_data_handler.py +++ b/tests/solardatatools/test_data_handler.py @@ -12,7 +12,8 @@ def test_load_and_run(self): data_file_path = filepath / "fixtures" / "data_transforms" / "timeseries.csv" df = pd.read_csv(data_file_path, parse_dates=[1], index_col=1) dh = DataHandler(df) - dh.run_pipeline(verbose=False) + dh.fix_dst() + dh.run_pipeline(power_col="ac_power", fix_shifts=True, verbose=False) # dh.report() self.assertAlmostEqual(dh.capacity_estimate, 6.7453649044036865, places=2) self.assertAlmostEqual(dh.data_quality_score, 0.9948186528497409, places=3) diff --git a/tests/solardatatools/test_system_profiler.py b/tests/solardatatools/test_system_profiler.py index 39f8a8df..ef2dad32 100644 --- a/tests/solardatatools/test_system_profiler.py +++ b/tests/solardatatools/test_system_profiler.py @@ -12,7 +12,7 @@ def test_system_profiler(self): filepath / "fixtures" / "system_profiler" / "data_handler_input.csv" ) data = pd.read_csv(data_file_path, parse_dates=[1], index_col=1) - dh = DataHandler(data, datetime_col="Date-Time") + dh = DataHandler(data) dh.fix_dst() dh.run_pipeline( power_col="ac_power", fix_shifts=False, correct_tz=False, verbose=False From 9c9b66b2477b35009b2f7cbe3c4e8abfb157c48d Mon Sep 17 00:00:00 2001 From: Meyers-Im Date: Mon, 25 Sep 2023 16:36:13 -0700 Subject: [PATCH 6/9] bugs --- tests/solardatatools/test_data_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/solardatatools/test_data_handler.py b/tests/solardatatools/test_data_handler.py index 3af06e75..f390579c 100644 --- a/tests/solardatatools/test_data_handler.py +++ b/tests/solardatatools/test_data_handler.py @@ -10,10 +10,10 @@ class TestDataHandler(unittest.TestCase): def test_load_and_run(self): filepath = Path(__file__).parent.parent data_file_path = filepath / "fixtures" / "data_transforms" / "timeseries.csv" - df = pd.read_csv(data_file_path, parse_dates=[1], index_col=1) + df = pd.read_csv(data_file_path, parse_dates=[0], index_col=0) dh = DataHandler(df) dh.fix_dst() - dh.run_pipeline(power_col="ac_power", fix_shifts=True, verbose=False) + dh.run_pipeline(power_col="ac_power_01", fix_shifts=True, verbose=False) # dh.report() self.assertAlmostEqual(dh.capacity_estimate, 6.7453649044036865, places=2) self.assertAlmostEqual(dh.data_quality_score, 0.9948186528497409, places=3) From 416309bc4b574c003d1725f806dc2a77cf256b00 Mon Sep 17 00:00:00 2001 From: Meyers-Im Date: Tue, 26 Sep 2023 10:33:03 -0700 Subject: [PATCH 7/9] pinning cvxpy --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1c48ac49..19922abf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ matplotlib seaborn requests pvlib -cvxpy>=1.1.0 +cvxpy>=1.3.2 cmake pykml haversine From 5cdbb4c98dd4c7a6cbc6829888c6527f7205d21a Mon Sep 17 00:00:00 2001 From: Meyers-Im Date: Tue, 26 Sep 2023 10:54:03 -0700 Subject: [PATCH 8/9] adding report statements for troubleshooting --- tests/solardatatools/test_data_handler.py | 2 +- tests/solardatatools/test_system_profiler.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/solardatatools/test_data_handler.py b/tests/solardatatools/test_data_handler.py index f390579c..0ae9c342 100644 --- a/tests/solardatatools/test_data_handler.py +++ b/tests/solardatatools/test_data_handler.py @@ -14,7 +14,7 @@ def test_load_and_run(self): dh = DataHandler(df) dh.fix_dst() dh.run_pipeline(power_col="ac_power_01", fix_shifts=True, verbose=False) - # dh.report() + dh.report() self.assertAlmostEqual(dh.capacity_estimate, 6.7453649044036865, places=2) self.assertAlmostEqual(dh.data_quality_score, 0.9948186528497409, places=3) self.assertAlmostEqual(dh.data_clearness_score, 0.49222797927461137, places=3) diff --git a/tests/solardatatools/test_system_profiler.py b/tests/solardatatools/test_system_profiler.py index ef2dad32..19798d91 100644 --- a/tests/solardatatools/test_system_profiler.py +++ b/tests/solardatatools/test_system_profiler.py @@ -15,8 +15,9 @@ def test_system_profiler(self): dh = DataHandler(data) dh.fix_dst() dh.run_pipeline( - power_col="ac_power", fix_shifts=False, correct_tz=False, verbose=False + power_col="ac_power", fix_shifts=False, correct_tz=False, verbose=True ) + dh.report() dh.setup_location_and_orientation_estimation(-5) estimate_latitude = dh.estimate_latitude() From 8c66babe829dfdfbf61a8d848016776c3c9034c9 Mon Sep 17 00:00:00 2001 From: Meyers-Im Date: Tue, 26 Sep 2023 11:57:18 -0700 Subject: [PATCH 9/9] update final test and remove testing print statements --- tests/solardatatools/test_data_handler.py | 2 +- tests/solardatatools/test_system_profiler.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/solardatatools/test_data_handler.py b/tests/solardatatools/test_data_handler.py index 0ae9c342..f390579c 100644 --- a/tests/solardatatools/test_data_handler.py +++ b/tests/solardatatools/test_data_handler.py @@ -14,7 +14,7 @@ def test_load_and_run(self): dh = DataHandler(df) dh.fix_dst() dh.run_pipeline(power_col="ac_power_01", fix_shifts=True, verbose=False) - dh.report() + # dh.report() self.assertAlmostEqual(dh.capacity_estimate, 6.7453649044036865, places=2) self.assertAlmostEqual(dh.data_quality_score, 0.9948186528497409, places=3) self.assertAlmostEqual(dh.data_clearness_score, 0.49222797927461137, places=3) diff --git a/tests/solardatatools/test_system_profiler.py b/tests/solardatatools/test_system_profiler.py index 19798d91..1c61e217 100644 --- a/tests/solardatatools/test_system_profiler.py +++ b/tests/solardatatools/test_system_profiler.py @@ -15,9 +15,9 @@ def test_system_profiler(self): dh = DataHandler(data) dh.fix_dst() dh.run_pipeline( - power_col="ac_power", fix_shifts=False, correct_tz=False, verbose=True + power_col="ac_power", fix_shifts=False, correct_tz=False, verbose=False ) - dh.report() + # dh.report() dh.setup_location_and_orientation_estimation(-5) estimate_latitude = dh.estimate_latitude() @@ -39,7 +39,7 @@ def test_system_profiler(self): # to pick some hyperparameters. ref_latitude = 37.5 # +/- 0.8 ref_longitude = -77.0 # +/- 0.03 - ref_tilt_real_loc = 22.45 # +/- 0.015 + ref_tilt_real_loc = 22.5 # +/- 0.015 ref_az_real_loc = 0.28 # +/- 0.015 # Updated tolerances based on new updates for sunset/sunrise SDs