diff --git a/applications/train_classifier_ptype.py b/applications/train_classifier_ptype.py index c5391dd..5ec356a 100644 --- a/applications/train_classifier_ptype.py +++ b/applications/train_classifier_ptype.py @@ -83,7 +83,7 @@ def trainer(conf, evaluate=True, data_split=0, mc_forward_passes=0): output_features = conf["output_features"] metric = conf["metric"] # flag for using the evidential model - if conf["model"]["loss"] == "dirichlet": + if conf["model"]["loss"] == "evidential": use_uncertainty = True else: use_uncertainty = False @@ -177,6 +177,10 @@ def trainer(conf, evaluate=True, data_split=0, mc_forward_passes=0): x = scaled_data[f"{name}_x"] if use_uncertainty: pred_probs, u, ale, epi = mlp.predict(x, return_uncertainties=True) + pred_probs = pred_probs.numpy() + u = u.numpy() + ale = ale.numpy() + epi = epi.numpy() entropy = np.zeros(pred_probs.shape) mutual_info = np.zeros(pred_probs.shape) elif mc_forward_passes > 0: # Compute epistemic uncertainty with MC dropout @@ -185,7 +189,7 @@ def trainer(conf, evaluate=True, data_split=0, mc_forward_passes=0): x, mc_forward_passes=mc_forward_passes) u = np.zeros(pred_probs.shape) else: - pred_probs = mlp.predict(x) + pred_probs = mlp.predict(x, return_uncertainties=False) ale = np.zeros(pred_probs.shape) u = np.zeros(pred_probs.shape) epi = np.zeros(pred_probs.shape) diff --git a/config/ptype/evidential.yml b/config/ptype/evidential.yml index 7018159..7c52f64 100644 --- a/config/ptype/evidential.yml +++ b/config/ptype/evidential.yml @@ -135,7 +135,7 @@ case_studies: - '2021-02-17' - '2021-02-18' - '2021-02-19' -data_path: /glade/p/cisl/aiml/ai2es/winter_ptypes/ptype_qc/mPING_interpolated_QC2.parquet +data_path: /glade/campaign/cisl/aiml/ai2es/winter_ptypes/ptype_qc/mPING_hourafter_interpolated_QC3.parquet direction: max ensemble: mc_steps: 0 @@ -147,24 +147,23 @@ input_features: - VGRD_m/s metric: val_ave_acc model: - activation: leaky - annealing_coeff: 34.593686950910275 - balanced_classes: 1 - batch_size: 100 - dropout_alpha: 0.20146936081973893 - epochs: 1000 - hidden_layers: 2 - hidden_neurons: 6461 + activation: leaky_relu + annealing_coeff: 34 + batch_size: 1130 + dropout_alpha: 0.11676011477923032 + epochs: 100 + evidential: true + n_inputs: 84 + hidden_layers: 4 + hidden_neurons: 212 + l2_weight: 0.000881889591229087 loss: evidential - loss_weights: - - 58.64242174310205 - - 94.59680461256323 - - 124.5896569779261 - - 227.38800030539545 - lr: 0.0027750619126744817 + lr: 0.004800502096767794 + n_classes: 4 optimizer: adam + output_activation: linear use_dropout: 1 - verbose: 0 + verbose: 1 mping_path: /glade/p/cisl/aiml/ai2es/winter_ptypes/precip_rap/mPING_mixture/ output_features: - ra_percent diff --git a/mlguess/keras/callbacks.py b/mlguess/keras/callbacks.py index 8be92ae..55cc0db 100644 --- a/mlguess/keras/callbacks.py +++ b/mlguess/keras/callbacks.py @@ -13,9 +13,12 @@ import os import keras import numpy as np +from hagelslag.evaluation.ProbabilityMetrics import DistributedROC +from sklearn.metrics import precision_recall_fscore_support, roc_auc_score logger = logging.getLogger(__name__) + def get_callbacks(config: Dict[str, str], path_extend=False) -> List[Callback]: callbacks = [] @@ -85,7 +88,6 @@ def get_config(self): return {} - class MetricsCallback(keras.callbacks.Callback): def __init__(self, x, y, name="val", n_bins=10, use_uncertainty=False, **kwargs): super().__init__() @@ -99,10 +101,11 @@ def __init__(self, x, y, name="val", n_bins=10, use_uncertainty=False, **kwargs) self.bin_uppers = bin_boundaries[1:] def on_epoch_end(self, epoch, logs={}): - pred_probs = np.asarray(self.model.predict(self.x)) if self.use_uncertainty: - pred_probs, _, _, _ = calc_prob_uncertainty(pred_probs) + pred_probs, _, _, _ = self.model.predict(self.x, return_uncertainties=True) pred_probs = pred_probs.numpy() + else: + pred_probs = np.asarray(self.model.predict(self.x, return_uncertainties=False)) logs[f"{self.name}_csi"] = self.mean_csi(pred_probs) true_labels = np.argmax(self.y, 1) pred_labels = np.argmax(pred_probs, 1) @@ -192,4 +195,3 @@ def ece(self, true_labels, pred_probs): pass mean = np.mean(ece) if np.isfinite(np.mean(ece)) else self.bin_lowers.shape[0] return mean - diff --git a/mlguess/keras/models.py b/mlguess/keras/models.py index cdade79..b923264 100644 --- a/mlguess/keras/models.py +++ b/mlguess/keras/models.py @@ -85,7 +85,7 @@ def __init__(self, hidden_layers=2, hidden_neurons=64, evidential=False, activat annealing_coeff=1.0, use_noise=False, noise_sd=0.0, lr=0.001, use_dropout=False, dropout_alpha=0.2, batch_size=128, epochs=2, kernel_reg=None, l1_weight=0.0, l2_weight=0.0, sgd_momentum=0.9, adam_beta_1=0.9, adam_beta_2=0.999, epsilon=1e-7, decay=0, verbose=0, random_state=1000, n_classes=2, - n_inputs=42, callbacks=None, **kwargs): + n_inputs=42, callbacks=[], **kwargs): super().__init__(**kwargs) self.hidden_layers = hidden_layers @@ -184,7 +184,7 @@ def fit(self, x=None, y=None, **kwargs): report_epoch_callback = ReportEpoch(e) self.loss = evidential_cat_loss(evi_coef=self.annealing_coeff, epoch_callback=report_epoch_callback) - self.callbacks = [report_epoch_callback] + self.callbacks.append(report_epoch_callback) super().compile(loss=self.loss, optimizer=self.optimizer_obj, @@ -257,6 +257,7 @@ def predict_dropout(self, x, mc_forward_passes=10, batch_size=None): def get_config(self): base_config = super().get_config() parameter_config = {hp: getattr(self, hp) for hp in self.hyperparameters} + parameter_config['callbacks'] = [] return {**base_config, **parameter_config} @@ -427,7 +428,7 @@ def fit(self, x=None, y=None, **kwargs): self.loss = gaussian_nll super().compile(optimizer=self.optimizer_obj, loss=self.loss) hist = super().fit(x, y, epochs=self.epochs, batch_size=self.batch_size, **kwargs) - self.training_var = np.var(x, axis=-1) + self.training_var = np.var(y, axis=-1) return hist