diff --git a/docs/cods_example.ipynb b/docs/cods_example.ipynb index f738e941..771663e0 100644 --- a/docs/cods_example.ipynb +++ b/docs/cods_example.ipynb @@ -336,29 +336,8 @@ "15 False \n", "\n", "Bootstrapping for uncertainty analysis (16 realizations):\n", - "# 16 | Used: 0.4 min | Left: 0.0 min | Progress: [----------------------------->] 100 %" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/mdecegli/opt/anaconda3/envs/cods_test/lib/python3.10/site-packages/rdtools/soiling.py:1875: FutureWarning: In a future version of pandas all arguments of concat except for the argument 'objs' will be keyword-only\n", - " concat_tot_mod = pd.concat([kdf.total_model for kdf in bt_kdfs], 1)\n", - "/Users/mdecegli/opt/anaconda3/envs/cods_test/lib/python3.10/site-packages/rdtools/soiling.py:1876: FutureWarning: In a future version of pandas all arguments of concat except for the argument 'objs' will be keyword-only\n", - " concat_SR = pd.concat([kdf.soiling_ratio for kdf in bt_kdfs], 1)\n", - "/Users/mdecegli/opt/anaconda3/envs/cods_test/lib/python3.10/site-packages/rdtools/soiling.py:1877: FutureWarning: In a future version of pandas all arguments of concat except for the argument 'objs' will be keyword-only\n", - " concat_r_s = pd.concat([kdf.soiling_rates for kdf in bt_kdfs], 1)\n", - "/Users/mdecegli/opt/anaconda3/envs/cods_test/lib/python3.10/site-packages/rdtools/soiling.py:1878: FutureWarning: In a future version of pandas all arguments of concat except for the argument 'objs' will be keyword-only\n", - " concat_ce = pd.concat([kdf.cleaning_events for kdf in bt_kdfs], 1)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Final RMSE: 0.01723\n" + "# 16 | Used: 0.5 min | Left: 0.0 min | Progress: [----------------------------->] 100 %\n", + "Final RMSE: 0.01722\n" ] } ], diff --git a/docs/sphinx/source/changelog/pending.rst b/docs/sphinx/source/changelog/pending.rst index 5aa04b95..58c57447 100644 --- a/docs/sphinx/source/changelog/pending.rst +++ b/docs/sphinx/source/changelog/pending.rst @@ -8,7 +8,7 @@ Enhancements now shows the number of year-on-year slopes in addition to color coding points (:issue:`298`, :pull:`324`) * The Combined estimation Of Degradation and Soiling (CODS) algorithm is implemented - in the soiling module and illustrated in an example notebook (:pull:`150`) + in the soiling module and illustrated in an example notebook (:pull:`150`, :pull:`333`) * Circular block bootstrapping added as a method for calculating uncertainty in ``degradation_year_on_year()`` via the ``Uncertainty_method`` argument (:pull:`150`) diff --git a/rdtools/soiling.py b/rdtools/soiling.py index 864fb3b5..7b35efc1 100644 --- a/rdtools/soiling.py +++ b/rdtools/soiling.py @@ -1154,9 +1154,6 @@ class CODSAnalysis(): adf_res : list The results of an Augmented Dickey-Fuller test (telling whether the residuals are stationary or not) - _parameters_n_weights : pandas.DataFrame - Contains information about the parameters used in each bootstrap model - fit, and the resultant weight. Raises ------ @@ -1183,8 +1180,8 @@ def __init__(self, energy_normalized_daily): def iterative_signal_decomposition( self, order=('SR', 'SC', 'Rd'), degradation_method='YoY', - max_iterations=18, detection_tuner=.5, convergence_criterion=5e-3, - pruning_iterations=1, pruning_tuner=.6, soiling_significance=.75, + max_iterations=18, cleaning_sensitivity=.5, convergence_criterion=5e-3, + pruning_iterations=1, clean_pruning_sensitivity=.6, soiling_significance=.75, process_noise=1e-4, renormalize_SR=None, ffill=True, clip_soiling=True, verbose=False): ''' @@ -1226,7 +1223,7 @@ def iterative_signal_decomposition( max_iterations : int, default 18 Max number of iterations to perform. Each iteration fits only one of the components, so three iterations are needed to fit all three components. - detection_tuner : float, default .5 + cleaning_sensitivity : float, default .5 Higher value gives lower cleaning event detection sensitivity. Should be between 0.1 and 2 convergence_criterion : float, default 5e-3 @@ -1234,7 +1231,7 @@ def iterative_signal_decomposition( convergence pruning_iterations : int, default 1 Number of iterations when pruning (removing) cleaning events - pruning_tuner : float, default .6 + clean_pruning_sensitivity : float, default .6 Sensitivity tuner that decides how easily a cleaning event is pruned (removed). Larger values means a smaller chance of pruning a given event. Should be between 0.1 and 2 @@ -1293,13 +1290,11 @@ def iterative_signal_decomposition( +------------------------+----------------------------------------------+ | Key | Description | +========================+==============================================+ - | 'degradation' | List of linear degradation rate of system in | - | | %/year, lower and upper bound of 95% | - | | confidence interval (list) | + | 'degradation' | Linear degradation rate of system in %/year | + | | (float) | +------------------------+----------------------------------------------+ - | 'soiling_loss' | List of average soiling losses over the time | - | | series in %, lower and upper bound of 95% | - | | confidence interval (list) | + | 'soiling_loss' | Average soiling losses over the time series | + | | in % (float) | +------------------------+----------------------------------------------+ | 'residual_shift' | Mean value of residuals. Multiply total | | | model by this number for complete overlap | @@ -1349,7 +1344,7 @@ def iterative_signal_decomposition( # Find possible cleaning events based on the performance index ce, rm9 = _rolling_median_ce_detection(pi.index, pi, ffill=ffill, - tuner=detection_tuner) + tuner=cleaning_sensitivity) pce = _collapse_cleaning_events(ce, rm9.diff().values, 5) small_soiling_signal, perfect_cleaning = False, True @@ -1368,13 +1363,13 @@ def iterative_signal_decomposition( if ic > 2: # Add possible cleaning events found by considering # the residuals pce = soiling_dfs[-1].cleaning_events.copy() - detection_tuner *= 1.2 # Increase value of detection tuner + cleaning_sensitivity *= 1.2 # decrease sensitivity ce, rm9 = _rolling_median_ce_detection( pi.index, residuals, ffill=ffill, - tuner=detection_tuner) + tuner=cleaning_sensitivity) ce = _collapse_cleaning_events(ce, rm9.diff().values, 5) pce[ce] = True - pruning_tuner /= 1.1 # Decrease value of pruning tuner + clean_pruning_sensitivity /= 1.1 # increase pruning sensitivity # Decompose input signal soiling_dummy = (pi / @@ -1388,7 +1383,7 @@ def iterative_signal_decomposition( clip_soiling=clip_soiling, prescient_cleaning_events=pce, pruning_iterations=pruning_iterations, - pruning_tuner=pruning_tuner, + clean_pruning_sensitivity=clean_pruning_sensitivity, perfect_cleaning=perfect_cleaning, process_noise=process_noise, renormalize_SR=renormalize_SR) @@ -1542,8 +1537,8 @@ def run_bootstrap(self, process_noise=1e-4, order_alternatives=(('SR', 'SC', 'Rd'), ('SC', 'SR', 'Rd')), - detection_tuner_alternatives=(.25, .75), - pruning_tuner_alternatives=(1/1.5, 1.5), + cleaning_sensitivity_alternatives=(.25, .75), + clean_pruning_sensitivity_alternatives=(1/1.5, 1.5), forward_fill_alternatives=(True, False), verbose=False, **kwargs): @@ -1571,7 +1566,7 @@ def run_bootstrap(self, Number of bootstrap realizations to be run minimum N, where N is the possible combinations of model alternatives defined by possible combination of order_alternatives, - detection_tuner_alternatives, pruning_tuner_alternatives and + cleaning_sensitivity_alternatives, clean_pruning_sensitivity_alternatives and forward_fill_alternatives. confidence_level : float, default 68.2 The size of the confidence intervals to return, in percent @@ -1583,11 +1578,11 @@ def run_bootstrap(self, order_alternatives : tuple of tuples, default (('SR', 'SC', 'Rd'), ('SC', 'SR', 'Rd')) Component estimation orders that will be tested during initial model fitting. - detection_tuner_alternatives : tuple, default (.25, .75) + cleaning_sensitivity_alternatives : tuple, default (.25, .75) Detection tuner values that will be tested during initial fitting. Length must be >= 1. First and last values define limits of values that will be used during bootstrapping. - pruning_tuner_alternatives : tuple, default (1/1.5, 1.5) + clean_pruning_sensitivity_alternatives : tuple, default (1/1.5, 1.5) Pruning tuner values that will be tested during initial fitting. Length must be >= 1. First and last values define limits of values that will be used during bootstrapping. @@ -1675,8 +1670,8 @@ def run_bootstrap(self, # Generate combinations of model parameter alternatives parameter_alternatives = [order_alternatives, - detection_tuner_alternatives, - pruning_tuner_alternatives, + cleaning_sensitivity_alternatives, + clean_pruning_sensitivity_alternatives, forward_fill_alternatives] index_list = list(itertools.product([0, 1], repeat=len(parameter_alternatives))) combination_of_parameters = [[parameter_alternatives[j][indexes[j]] @@ -1697,8 +1692,8 @@ def run_bootstrap(self, try: df_out, result_dict = self.iterative_signal_decomposition( max_iterations=18, order=order, clip_soiling=True, - detection_tuner=dt, pruning_iterations=1, - pruning_tuner=pt, process_noise=process_noise, ffill=ff, + cleaning_sensitivity=dt, pruning_iterations=1, + clean_pruning_sensitivity=pt, process_noise=process_noise, ffill=ff, degradation_method=degradation_method, **kwargs) # Save results @@ -1824,8 +1819,8 @@ def run_bootstrap(self, # Do Signal decomposition for soiling and degradation component kdf, results_dict = temporary_cods_instance.iterative_signal_decomposition( max_iterations=4, order=order, clip_soiling=True, - detection_tuner=dt, pruning_iterations=1, - pruning_tuner=pt, process_noise=pn, + cleaning_sensitivity=dt, pruning_iterations=1, + clean_pruning_sensitivity=pt, process_noise=pn, renormalize_SR=renormalize_SR, ffill=ffill, degradation_method=degradation_method, **kwargs) @@ -1872,10 +1867,10 @@ def run_bootstrap(self, ci_high_edge = (50 + confidence_level / 2) / 100 # Concatenate boostrap model fits - concat_tot_mod = pd.concat([kdf.total_model for kdf in bt_kdfs], 1) - concat_SR = pd.concat([kdf.soiling_ratio for kdf in bt_kdfs], 1) - concat_r_s = pd.concat([kdf.soiling_rates for kdf in bt_kdfs], 1) - concat_ce = pd.concat([kdf.cleaning_events for kdf in bt_kdfs], 1) + concat_tot_mod = pd.concat([kdf.total_model for kdf in bt_kdfs], axis=1) + concat_SR = pd.concat([kdf.soiling_ratio for kdf in bt_kdfs], axis=1) + concat_r_s = pd.concat([kdf.soiling_rates for kdf in bt_kdfs], axis=1) + concat_ce = pd.concat([kdf.cleaning_events for kdf in bt_kdfs], axis=1) # Find confidence intervals for SR and soiling rates df_out['SR_low'] = concat_SR.quantile(ci_low_edge, 1) @@ -1935,7 +1930,7 @@ def run_bootstrap(self, def _Kalman_filter_for_SR(self, zs_series, process_noise=1e-4, zs_std=.05, rate_std=.005, max_soiling_rates=.0005, - pruning_iterations=1, pruning_tuner=.6, + pruning_iterations=1, clean_pruning_sensitivity=.6, renormalize_SR=None, perfect_cleaning=False, prescient_cleaning_events=None, clip_soiling=True, ffill=True): @@ -1960,7 +1955,7 @@ def _Kalman_filter_for_SR(self, zs_series, process_noise=1e-4, zs_std=.05, Represents the maximum allowed positive soiling rate (when soiling is removed) pruning_iterations : int, default 1 Number of iterations when pruning (removing) cleaning events - pruning_tuner : float, default 0.6 + clean_pruning_sensitivity : float, default 0.6 Sensitivity tuner that decides how easily a cleaning event is pruned (removed). Larger values means a smaller chance of pruning a given event. renormalize_SR : float or None, default None @@ -2093,7 +2088,7 @@ def _Kalman_filter_for_SR(self, zs_series, process_noise=1e-4, zs_std=.05, pi_after_cleaning = rm_smooth_pi.loc[cleaning_events] # Detect outiers/false positives false_positives = _find_numeric_outliers(pi_after_cleaning, - pruning_tuner, 'lower') + clean_pruning_sensitivity, 'lower') cleaning_events = \ false_positives[~false_positives].index.tolist() @@ -2280,8 +2275,8 @@ def soiling_cods(energy_normalized_daily, process_noise=1e-4, order_alternatives=(('SR', 'SC', 'Rd'), ('SC', 'SR', 'Rd')), - detection_tuner_alternatives=(.25, .75), - pruning_tuner_alternatives=(1/1.5, 1.5), + cleaning_sensitivity_alternatives=(.25, .75), + clean_pruning_sensitivity_alternatives=(1/1.5, 1.5), forward_fill_alternatives=(True, False), verbose=False, **kwargs): @@ -2314,11 +2309,11 @@ def soiling_cods(energy_normalized_daily, order_alternatives : tuple of tuples, default (('SR', 'SC', 'Rd'), ('SC', 'SR', 'Rd')) Component estimation orders that will be tested during initial model fitting. - detection_tuner_alternatives : tuple, default (.25, .75) + cleaning_sensitivity_alternatives : tuple, default (.25, .75) Detection tuner values that will be tested during initial fitting. Length must be >= 1. First and last values define limits of values that will be used during bootstrapping. - pruning_tuner_alternatives : tuple, default (1/1.5, 1.5) + clean_pruning_sensitivity_alternatives : tuple, default (1/1.5, 1.5) Pruning tuner values that will be tested during initial fitting. Length must be >= 1. First and last values define limits of values that will be used during bootstrapping. @@ -2409,8 +2404,8 @@ def soiling_cods(energy_normalized_daily, degradation_method=degradation_method, process_noise=process_noise, order_alternatives=order_alternatives, - detection_tuner_alternatives=detection_tuner_alternatives, - pruning_tuner_alternatives=pruning_tuner_alternatives, + cleaning_sensitivity_alternatives=cleaning_sensitivity_alternatives, + clean_pruning_sensitivity_alternatives=clean_pruning_sensitivity_alternatives, forward_fill_alternatives=forward_fill_alternatives, **kwargs)