params.yaml

settings:
  n_jobs: 26
  backend: joblib
  mlflow_tracking_uri: http://localhost:5000
  plot_format: pdf

  # only valid for `scripts/run_pipeline.py`
  stages:
    fetch_data: false
    preprocess_data: false
    sample_data: false
    calculate_values: false
    calculate_threshold_characteristics: false
    evaluate_curves: false
    evaluate_metrics: false
    render_plots: true
    shutdown: false

  time:
    active: true

  threshold_characteristics:
    active: false
    valuation_method: banzhaf_shapley  # Method used to calculate the threshold characteristics.
    model: logistic_regression  # Default model to use for determining the values
    max_plotting_percentage: 1e-4  # Threshold for stopping plotting in direction of x-axis.

active:
  experiments:
  - point_removal
  - noise_removal
  models:
  - logistic_regression
  - knn
  - svm
  - gradient_boosting_classifier
  datasets:
  - cifar10
  - click
  - covertype
  - cpu
  - diabetes
  - fmnist_binary
  - mnist_binary
  - mnist_multi
  - phoneme
  valuation_methods:
  - random
  - classwise_shapley
  - tmc_shapley
  - beta_shapley
  - banzhaf_shapley
#  - msr_banzhaf_shapley_500
#  - msr_banzhaf_shapley_5000
  - least_core
  - loo
  repetitions:
  - 1
  - 2
  - 3
  - 4
  - 5
  - 6
  - 7
  - 8
  - 9
  - 10
  - 11
  - 12
  - 13
  - 14
  - 15
  - 16
  - 17
  - 18
  - 19
  - 20


experiments:
  point_removal:
    sampler: default
    curves:

      accuracy_logistic_regression:
        fn: metric
        metric: accuracy
        eval_model: logistic_regression
        plots:
          - accuracy
      accuracy_knn:
        fn: metric
        metric: accuracy
        eval_model: knn
        plots:
          - accuracy
      accuracy_gradient_boosting_classifier:
        fn: metric
        metric: accuracy
        eval_model: gradient_boosting_classifier
        plots:
          - accuracy
      accuracy_svm:
        fn: metric
        metric: accuracy
        eval_model: svm
        plots:
          - accuracy
      accuracy_mlp:
        fn: metric
        metric: accuracy
        eval_model: mlp
        plots:
          - accuracy

      value_decay:
        fn: value_decay
        fraction: 1.0
        plots:
          - value_decay

      top_fraction:
        fn: top_fraction
        alpha_range:
          from: 0.01
          to: 0.5
          step: 0.01
        plots:
          - rank_stability

    metrics:
      weighted_relative_accuracy_difference_random:
        curve:
          - accuracy_logistic_regression
          - accuracy_knn
          - accuracy_gradient_boosting_classifier
          - accuracy_svm
          - accuracy_mlp
        lamb: 0.1
        fn: weighted_relative_accuracy_difference_random
        plots:
          - table
          - box_wrad

      geometric_weighted_drop_half:
        curve:
          - accuracy_logistic_regression
          - accuracy_knn
          - accuracy_gradient_boosting_classifier
          - accuracy_svm
          - accuracy_mlp
        fn: geometric_weighted_drop
        input_perc: 0.5
        plots:
          - table
          - box_wad

      geometric_weighted_drop:
        curve:
          - accuracy_logistic_regression
          - accuracy_knn
          - accuracy_gradient_boosting_classifier
          - accuracy_svm
          - accuracy_mlp
        fn: geometric_weighted_drop
        input_perc: 1.0
        plots:
          - table
          - box_wad

  noise_removal:
    sampler: default
    preprocessors:
      flip_labels:
        perc: 0.2
    curves:
      precision_recall:
        fn: precision_recall
        flipped_labels: preprocessor.flip_labels.idx
        plots:
          - precision_recall
    metrics:
      roc_auc:
        curve:
          - precision_recall
        fn: roc_auc
        plots:
          - table
          - box_auc

plots:

  accuracy:
    type: line
    mean_agg: mean
    std_agg: bootstrap
    plot_perc: 0.5
    x_label: "n"
    y_label: "Accuracy"

  precision_recall:
    type: line
    mean_agg: mean
    std_agg: bootstrap
    x_label: "Recall"
    y_label: "Precision"

  table:
    type: table
    format: ".3f"

  box_wrad:
    type: boxplot
    x_label: "WRAD"

  box_wad:
    type: boxplot
    x_label: "WAD"

  box_auc:
    type: boxplot
    x_label: "AUC"

  rank_stability:
    type: line
    mean_agg: intersect
    x_label: "%"
    y_label: "%"

  value_decay:
    type: line
    mean_agg: mean
    std_agg: bootstrap
    x_label: "n"
    y_label: "%"

samplers:
  default:
    train: 0.1667
    val: 0.1667
    test: 0.6667
    max_samples: 3000

datasets:
  diabetes:
    openml_id: 37

  cpu:
    openml_id: 197
    preprocessor:
      threshold_y:
        threshold: 89

  click:
    openml_id: 1216

  covertype:
    openml_id: 1596

  phoneme:
    openml_id: 1489

  fmnist_binary:
    openml_id: 40996
    filters:
      binarization:
        label_zero: '0'
        label_one: '1'
    preprocessor:
      principal_resnet_components:
        n_components: 32
        grayscale: true
        seed: 101

  cifar10:
    openml_id: 40927
    filters:
      binarization:
        label_zero: '1'
        label_one: '9'
    preprocessor:
      principal_resnet_components:
        n_components: 32
        grayscale: false
        seed: 102

  mnist_binary:
    openml_id: 554
    filters:
      binarization:
        label_zero: '1'
        label_one: '7'
    preprocessor:
      principal_resnet_components:
        n_components: 32
        grayscale: true
        seed: 103

  mnist_multi:
    openml_id: 554
    preprocessor:
      principal_resnet_components:
        n_components: 32
        grayscale: true
        seed: 104

models:
  logistic_regression:
    model: logistic_regression
    solver: liblinear

  gradient_boosting_classifier:
    model: gradient_boosting_classifier
    n_estimators: 40
    min_samples_split: 6
    max_depth: 2

  knn:
    model: knn
    n_neighbors: 5
    weights: uniform

  svm:
    model: svm
    kernel: rbf

  mlp:
    model: mlp

valuation_methods:
  random:
    algorithm: random

  loo:
    algorithm: loo
    progress: true
    cache_group: acc

  classwise_shapley:
    algorithm: classwise_shapley
    cache_group: disc_acc
    normalize_values: true
    n_resample_complement_sets: 1
    n_updates: 500
    rtol: 1e-4
    progress: true
    use_default_scorer_value: false

  beta_shapley:
    algorithm: beta_shapley
    alpha: 16.0
    beta: 1.0
    n_updates: 500
    progress: true
    cache_group: acc

  banzhaf_shapley:
    algorithm: banzhaf_shapley
    n_updates: 500
    progress: true
    cache_group: acc

  msr_banzhaf_shapley_500:
    algorithm: msr_banzhaf_shapley
    n_updates: 500
    progress: true
    cache_group: acc

  msr_banzhaf_shapley_5000:
    algorithm: msr_banzhaf_shapley
    n_updates: 5000
    progress: true
    cache_group: acc

  tmc_shapley:
    algorithm: tmc_shapley
    rtol: 1e-4
    n_updates: 500
    progress: true
    cache_group: acc

  least_core:
    algorithm: least_core
    n_updates: 5000
    progress: false
    cache_group: acc