merge

configs/new_mi.json

@@ -0,0 +1,34 @@
+{
+    "base_model": "EleutherAI/gpt-neo-125m",
+    "dataset_member": "the_pile",
+    "dataset_nonmember": "the_pile",
+    "min_words": 100,
+    "max_words": 200,
+    "max_tokens": 512,
+    "max_data": 100000,
+    "output_name": "unified_mia",
+    "specific_source": "Github",
+    "n_samples": 1000,
+    "blackbox_attacks": ["loss", "ref", "min_k"],
+    "ref_config": {
+        "models": [
+            "stabilityai/stablelm-base-alpha-3b-v2"
+        ]
+    },
+    "neighborhood_config": {
+        "model": "bert",
+        "n_perturbation_list": [
+            25
+        ],
+        "pct_words_masked": 0.3,
+        "span_length": 2,
+        "dump_cache": false,
+        "load_from_cache": true
+    },
+    "env_config": {
+        "results": "results_new",
+        "device_map": "balanced_low_0"
+    },
+    "dump_cache": false,
+    "load_from_cache": true
+}

new_mi_experiment.py

@@ -75,8 +75,9 @@ def run_blackbox_attacks(
         runnable_attacks.append(a)
     attacks = runnable_attacks
 
-    neighborhood_attacker = NeighborhoodAttack(config, target_model)
-    neighborhood_attacker.prepare()
+    if BlackBoxAttacks.NEIGHBOR in attacks:
+        neighborhood_attacker = NeighborhoodAttack(config, target_model)
+        neighborhood_attacker.prepare()
 
     results = defaultdict(list)
     for classification in keys_care_about:
@@ -239,7 +240,10 @@ def run_blackbox_attacks(
 
                 # Update collected scores for each sample with ref-based attack scores
                 for classification, result in results.items():
-                    for r in tqdm(result, desc="Ref scores"):
+                    itr = result
+                    if verbose:
+                        itr = tqdm(itr, desc="Ref scores")
+                    for r in itr:
                         ref_model_scores = []
                         for i, s in enumerate(r["sample"]):
                             if config.pretokenized:
@@ -631,7 +635,7 @@ def edit(x, n: int):
             )
             pbar.update(1)
 
-            for attack in other_blackbox_predictions.keys():
+            for attack in config.blackbox_attacks:
                 score_dict[attack][n][i] = other_blackbox_predictions[attack]["member"]
 
     pbar.close()

run.py

@@ -73,8 +73,9 @@ def run_blackbox_attacks(
         runnable_attacks.append(a)
     attacks = runnable_attacks
 
-    neighborhood_attacker = NeighborhoodAttack(config, target_model)
-    neighborhood_attacker.prepare()
+    if BlackBoxAttacks.NEIGHBOR in attacks:
+        neighborhood_attacker = NeighborhoodAttack(config, target_model)
+        neighborhood_attacker.prepare()
 
     results = defaultdict(list)
     for classification in keys_care_about: