From 9813fa0a03d6d5b31f215356172736ab6be31889 Mon Sep 17 00:00:00 2001 From: Max Mauermann Date: Mon, 8 Jan 2024 16:12:31 +0100 Subject: [PATCH 1/2] non-event-samples get included in training set, disable some feature when using a single event-class --- config.py | 3 +++ model.py | 8 ++++---- train.py | 10 +++++++++- utils.py | 20 +++++++++++++++++++- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/config.py b/config.py index 6c162eec..9a0f7799 100644 --- a/config.py +++ b/config.py @@ -163,6 +163,9 @@ # Mutliple executions will be averaged, so the evaluation is more consistent AUTOTUNE_EXECUTIONS_PER_TRIAL: int = 1 +# If a binary classification model is trained, this value will be detected automatically in the training script +BINARY_CLASSIFICATION: bool = False + ##################### # Misc runtime vars # ##################### diff --git a/model.py b/model.py index a4908138..bc6d4ff6 100644 --- a/model.py +++ b/model.py @@ -225,12 +225,12 @@ def on_epoch_end(self, epoch, logs=None): x_train, y_train = utils.upsampling(x_train, y_train, upsampling_ratio, upsampling_mode) print(f"Upsampled training data to {x_train.shape[0]} samples.", flush=True) - # Apply mixup to training data - if train_with_mixup: + # Apply mixup to training data + if train_with_mixup and not cfg.BINARY_CLASSIFICATION: x_train, y_train = utils.mixup(x_train, y_train) # Apply label smoothing - if train_with_label_smoothing: + if train_with_label_smoothing and not cfg.BINARY_CLASSIFICATION: y_train = utils.label_smoothing(y_train) # Early stopping @@ -252,7 +252,7 @@ def on_epoch_end(self, epoch, logs=None): classifier.compile( optimizer=keras.optimizers.Adam(learning_rate=lr_schedule), loss=custom_loss, - metrics=[keras.metrics.AUC(curve="PR", multi_label=False, name="AUPRC")], + metrics=[keras.metrics.AUC(curve="PR", multi_label=False, name="AUPRC")], # TODO: Use AUROCC ) # Train model diff --git a/train.py b/train.py index 8de370cc..c5ea1ddc 100644 --- a/train.py +++ b/train.py @@ -41,7 +41,15 @@ def _loadTrainingData(cache_mode="none", cache_file=""): labels = list(sorted(utils.list_subdirectories(cfg.TRAIN_DATA_PATH))) # Get valid labels - valid_labels = [l for l in labels if not l.lower() in cfg.NON_EVENT_CLASSES and not l.startswith("-")] + valid_labels = [l for l in labels if not l.lower() in cfg.NON_EVENT_CLASSES and not l.startswith("-")] + + cfg.BINARY_CLASSIFICATION = len(valid_labels) == 1 + + if cfg.BINARY_CLASSIFICATION: + if len([l for l in labels if l.startswith("-")]) > 0: + raise Exception("no negative labels with binary classification") + if len([l for l in labels if l in cfg.NON_EVENT_CLASSES]) == 0: + raise Exception("need non event class for binary classification") # Load training data x_train = [] diff --git a/utils.py b/utils.py index 278b9921..5769f5b4 100644 --- a/utils.py +++ b/utils.py @@ -98,7 +98,6 @@ def random_split(x, y, val_ratio=0.2): train_indices = positive_indices[:num_samples_train] val_indices = positive_indices[num_samples_train:num_samples_train + num_samples_val] - # Append samples to training and validation data x_train.append(x[train_indices]) y_train.append(y[train_indices]) @@ -109,6 +108,25 @@ def random_split(x, y, val_ratio=0.2): x_train.append(x[negative_indices]) y_train.append(y[negative_indices]) + # Add samples for non-event classes to train (or split for validation when binary classification) + non_event_indices = np.where(y[:,:] == 0)[0] + if cfg.BINARY_CLASSIFICATION: + num_samples = len(non_event_indices) + num_samples_train = max(1, int(num_samples * (1 - val_ratio))) + num_samples_val = max(0, num_samples - num_samples_train) + np.random.shuffle(non_event_indices) + train_indices = non_event_indices[:num_samples_train] + val_indices = non_event_indices[num_samples_train:num_samples_train + num_samples_val] + x_train.append(x[train_indices]) + y_train.append(y[train_indices]) + x_val.append(x[val_indices]) + y_val.append(y[val_indices]) + else: + x_train.append(x[non_event_indices]) + y_train.append(y[non_event_indices]) + + + # Concatenate data x_train = np.concatenate(x_train) y_train = np.concatenate(y_train) From 4eca0d39300289ac60932e3c1ccf9da6f4d5b462 Mon Sep 17 00:00:00 2001 From: Max Mauermann Date: Mon, 8 Jan 2024 17:17:10 +0100 Subject: [PATCH 2/2] use noise samples for validation also when using more classes --- train.py | 4 ++-- utils.py | 27 +++++++++++---------------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/train.py b/train.py index c5ea1ddc..d7a40398 100644 --- a/train.py +++ b/train.py @@ -47,9 +47,9 @@ def _loadTrainingData(cache_mode="none", cache_file=""): if cfg.BINARY_CLASSIFICATION: if len([l for l in labels if l.startswith("-")]) > 0: - raise Exception("no negative labels with binary classification") + raise Exception("negative labels cant be used with binary classification") if len([l for l in labels if l in cfg.NON_EVENT_CLASSES]) == 0: - raise Exception("need non event class for binary classification") + raise Exception("non-event samples are required for binary classification") # Load training data x_train = [] diff --git a/utils.py b/utils.py index 5769f5b4..83cef468 100644 --- a/utils.py +++ b/utils.py @@ -108,23 +108,18 @@ def random_split(x, y, val_ratio=0.2): x_train.append(x[negative_indices]) y_train.append(y[negative_indices]) - # Add samples for non-event classes to train (or split for validation when binary classification) + # Add samples for non-event classes to training and validation data non_event_indices = np.where(y[:,:] == 0)[0] - if cfg.BINARY_CLASSIFICATION: - num_samples = len(non_event_indices) - num_samples_train = max(1, int(num_samples * (1 - val_ratio))) - num_samples_val = max(0, num_samples - num_samples_train) - np.random.shuffle(non_event_indices) - train_indices = non_event_indices[:num_samples_train] - val_indices = non_event_indices[num_samples_train:num_samples_train + num_samples_val] - x_train.append(x[train_indices]) - y_train.append(y[train_indices]) - x_val.append(x[val_indices]) - y_val.append(y[val_indices]) - else: - x_train.append(x[non_event_indices]) - y_train.append(y[non_event_indices]) - + num_samples = len(non_event_indices) + num_samples_train = max(1, int(num_samples * (1 - val_ratio))) + num_samples_val = max(0, num_samples - num_samples_train) + np.random.shuffle(non_event_indices) + train_indices = non_event_indices[:num_samples_train] + val_indices = non_event_indices[num_samples_train:num_samples_train + num_samples_val] + x_train.append(x[train_indices]) + y_train.append(y[train_indices]) + x_val.append(x[val_indices]) + y_val.append(y[val_indices]) # Concatenate data