Merge branch 'main' of https://github.com/iamgroot42/mimir

docs/attacks/index.html

@@ -45,7 +45,7 @@ <h2 class="section-title" id="header-submodules">Sub-modules</h2>
 </dd>
 <dt><code class="name"><a title="mimir.attacks.loss" href="loss.html">mimir.attacks.loss</a></code></dt>
 <dd>
-<div class="desc"><p>Straight-forward LOSS attack</p></div>
+<div class="desc"><p>Straight-forward LOSS attack, as described in <a href="https://ieeexplore.ieee.org/abstract/document/8429311">https://ieeexplore.ieee.org/abstract/document/8429311</a></p></div>
 </dd>
 <dt><code class="name"><a title="mimir.attacks.min_k" href="min_k.html">mimir.attacks.min_k</a></code></dt>
 <dd>

docs/attacks/loss.html

@@ -5,7 +5,7 @@
 <meta name="viewport" content="width=device-width, initial-scale=1, minimum-scale=1" />
 <meta name="generator" content="pdoc 0.10.0" />
 <title>mimir.attacks.loss API documentation</title>
-<meta name="description" content="Straight-forward LOSS attack" />
+<meta name="description" content="Straight-forward LOSS attack, as described in https://ieeexplore.ieee.org/abstract/document/8429311" />
 <link rel="preload stylesheet" as="style" href="https://cdnjs.cloudflare.com/ajax/libs/10up-sanitize.css/11.0.1/sanitize.min.css" integrity="sha256-PK9q560IAAa6WVRRh76LtCaI8pjTJ2z11v0miyNNjrs=" crossorigin>
 <link rel="preload stylesheet" as="style" href="https://cdnjs.cloudflare.com/ajax/libs/10up-sanitize.css/11.0.1/typography.min.css" integrity="sha256-7l/o7C8jubJiy74VsKTidCy1yBkRtiUGbVkYBylBqUg=" crossorigin>
 <link rel="stylesheet preload" as="style" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.1.1/styles/github.min.css" crossorigin>
@@ -22,22 +22,27 @@
 <h1 class="title">Module <code>mimir.attacks.loss</code></h1>
 </header>
 <section id="section-intro">
-<p>Straight-forward LOSS attack</p>
+<p>Straight-forward LOSS attack, as described in <a href="https://ieeexplore.ieee.org/abstract/document/8429311">https://ieeexplore.ieee.org/abstract/document/8429311</a></p>
 <details class="source">
 <summary>
 <span>Expand source code</span>
 </summary>
 <pre><code class="python">&#34;&#34;&#34;
-    Straight-forward LOSS attack
+    Straight-forward LOSS attack, as described in https://ieeexplore.ieee.org/abstract/document/8429311
 &#34;&#34;&#34;
+import torch as ch
 from mimir.attacks.blackbox_attacks import Attack
 
 
 class LOSSAttack(Attack):
     def __init__(self, config, model):
         super().__init__(config, model, ref_model=None)
 
+    @ch.no_grad()
     def _attack(self, document, probs, tokens=None, **kwargs):
+        &#34;&#34;&#34;
+            LOSS-score. Use log-likelihood from model.
+        &#34;&#34;&#34;
         return self.model.get_ll(document, probs=probs, tokens=tokens)</code></pre>
 </details>
 </section>
@@ -64,7 +69,11 @@ <h2 class="section-title" id="header-classes">Classes</h2>
     def __init__(self, config, model):
         super().__init__(config, model, ref_model=None)
 
+    @ch.no_grad()
     def _attack(self, document, probs, tokens=None, **kwargs):
+        &#34;&#34;&#34;
+            LOSS-score. Use log-likelihood from model.
+        &#34;&#34;&#34;
         return self.model.get_ll(document, probs=probs, tokens=tokens)</code></pre>
 </details>
 <h3>Ancestors</h3>

docs/attacks/min_k.html

@@ -32,7 +32,6 @@ <h1 class="title">Module <code>mimir.attacks.min_k</code></h1>
 &#34;&#34;&#34;
 import torch as ch
 import numpy as np
-
 from mimir.attacks.blackbox_attacks import Attack
 
 
@@ -42,6 +41,9 @@ <h1 class="title">Module <code>mimir.attacks.min_k</code></h1>
 
     @ch.no_grad()
     def _attack(self, document, probs, tokens=None, **kwargs):
+        &#34;&#34;&#34;
+        Min-k % Prob Attack. Gets model praobbilities and returns likelihood when computed over top k% of ngrams.
+        &#34;&#34;&#34;
         # Hyper-params specific to min-k attack
         k: float = kwargs.get(&#34;k&#34;, 0.2)
         window: int = kwargs.get(&#34;window&#34;, 1)
@@ -87,6 +89,9 @@ <h2 class="section-title" id="header-classes">Classes</h2>
 
     @ch.no_grad()
     def _attack(self, document, probs, tokens=None, **kwargs):
+        &#34;&#34;&#34;
+        Min-k % Prob Attack. Gets model praobbilities and returns likelihood when computed over top k% of ngrams.
+        &#34;&#34;&#34;
         # Hyper-params specific to min-k attack
         k: float = kwargs.get(&#34;k&#34;, 0.2)
         window: int = kwargs.get(&#34;window&#34;, 1)

docs/attacks/neighborhood.html

@@ -46,6 +46,17 @@ <h1 class="title">Module <code>mimir.attacks.neighborhood</code></h1>
 from mimir.attacks.blackbox_attacks import Attack
 
 
+# def get_mask_model(config: ExperimentConfig, **kwargs):
+#     if &#34;t5&#34; in config.neighborhood_config.model:
+#         mask_model = T5Model(
+#                 config, model_kwargs=model_kwargs, tokenizer_kwargs=tokenizer_kwargs
+#             )
+#         elif &#34;bert&#34; in config.neighborhood_config.model:
+#             mask_model = BertModel(config)
+#         else:
+#             raise ValueError(f&#34;Unknown model {config.neighborhood_config.model}&#34;)
+
+
 class NeighborhoodAttack(Attack):
     def __init__(
         self,
@@ -58,43 +69,56 @@ <h1 class="title">Module <code>mimir.attacks.neighborhood</code></h1>
         self.ref_model = self._pick_neighbor_model()
         assert issubclass(type(self.ref_model), MaskFillingModel), &#34;ref_model must be MaskFillingModel for neighborhood attack&#34;
 
+    def get_mask_model(self):
+        return self.ref_model
+
+    def create_fill_dictionary(self, data):
+        neigh_config = self.config.neighborhood_config
+        if &#34;t5&#34; in neigh_config.model and neigh_config.random_fills:
+            if not self.config.pretokenized:
+                # TODO: maybe can be done if detokenized, but currently not supported
+                self.ref_model.create_fill_dictionary(data)
+
     def _pick_neighbor_model(self):
         # mask filling t5 model
         mask_model = None
         neigh_config = self.config.neighborhood_config
         env_config = self.config.env_config
-        if neigh_config:
-            model_kwargs = dict()
-            if not neigh_config.random_fills:
-                if env_config.int8:
-                    model_kwargs = dict(
-                        load_in_8bit=True, device_map=&#34;auto&#34;, torch_dtype=torch.bfloat16
-                    )
-                elif env_config.half:
-                    model_kwargs = dict(torch_dtype=torch.bfloat16)
-                try:
-                    n_positions = (
-                        512  # Should fix later, but for T-5 this is 512 indeed
-                    )
-                    # mask_model.config.n_positions
-                except AttributeError:
-                    n_positions = self.config.max_tokens
-            else:
+
+        model_kwargs = dict()
+        if not neigh_config.random_fills:
+            if env_config.int8:
+                model_kwargs = dict(
+                    load_in_8bit=True, device_map=&#34;auto&#34;, torch_dtype=torch.bfloat16
+                )
+            elif env_config.half:
+                model_kwargs = dict(torch_dtype=torch.bfloat16)
+            try:
+                n_positions = (
+                    512  # Should fix later, but for T-5 this is 512 indeed
+                )
+                # mask_model.config.n_positions
+            except AttributeError:
                 n_positions = self.config.max_tokens
-            tokenizer_kwargs = {
-                &#34;model_max_length&#34;: n_positions,
-            }
+        else:
+            n_positions = self.config.max_tokens
+        tokenizer_kwargs = {
+            &#34;model_max_length&#34;: n_positions,
+        }
 
-        if &#34;t5&#34; in self.config.neighborhood_config.model:
+        print(f&#34;Loading mask filling model {neigh_config.model}...&#34;)
+        if &#34;t5&#34; in neigh_config.model:
             mask_model = T5Model(
                 self.config,
                 model_kwargs=model_kwargs,
                 tokenizer_kwargs=tokenizer_kwargs,
             )
-        elif &#34;bert&#34; in self.config.neighborhood_config.model:
+        elif &#34;bert&#34; in neigh_config.model:
             mask_model = BertModel(self.config)
         else:
-            raise ValueError(f&#34;Unknown model {self.config.neighborhood_config.model}&#34;)
+            raise ValueError(f&#34;Unknown model {neigh_config.model}&#34;)
+        # if config.dataset_member in [&#39;english&#39;, &#39;german&#39;]:
+        #     preproc_tokenizer = mask_tokenizer
         return mask_model
 
     def load(self):
@@ -124,6 +148,9 @@ <h1 class="title">Module <code>mimir.attacks.neighborhood</code></h1>
         return neighbors
 
     def _attack(self, document, probs, tokens=None, **kwargs):
+        &#34;&#34;&#34;
+        Neighborhood attack score. Looks at difference in likelihood for given document and average likelihood of its neighbors
+        &#34;&#34;&#34;
         # documents here are actually neighbors
         batch_size = kwargs.get(&#34;batch_size&#34;, 4)
         substr_neighbors = kwargs.get(&#34;substr_neighbors&#34;, None)
@@ -1138,43 +1165,56 @@ <h3>Inherited members</h3>
         self.ref_model = self._pick_neighbor_model()
         assert issubclass(type(self.ref_model), MaskFillingModel), &#34;ref_model must be MaskFillingModel for neighborhood attack&#34;
 
+    def get_mask_model(self):
+        return self.ref_model
+
+    def create_fill_dictionary(self, data):
+        neigh_config = self.config.neighborhood_config
+        if &#34;t5&#34; in neigh_config.model and neigh_config.random_fills:
+            if not self.config.pretokenized:
+                # TODO: maybe can be done if detokenized, but currently not supported
+                self.ref_model.create_fill_dictionary(data)
+
     def _pick_neighbor_model(self):
         # mask filling t5 model
         mask_model = None
         neigh_config = self.config.neighborhood_config
         env_config = self.config.env_config
-        if neigh_config:
-            model_kwargs = dict()
-            if not neigh_config.random_fills:
-                if env_config.int8:
-                    model_kwargs = dict(
-                        load_in_8bit=True, device_map=&#34;auto&#34;, torch_dtype=torch.bfloat16
-                    )
-                elif env_config.half:
-                    model_kwargs = dict(torch_dtype=torch.bfloat16)
-                try:
-                    n_positions = (
-                        512  # Should fix later, but for T-5 this is 512 indeed
-                    )
-                    # mask_model.config.n_positions
-                except AttributeError:
-                    n_positions = self.config.max_tokens
-            else:
+
+        model_kwargs = dict()
+        if not neigh_config.random_fills:
+            if env_config.int8:
+                model_kwargs = dict(
+                    load_in_8bit=True, device_map=&#34;auto&#34;, torch_dtype=torch.bfloat16
+                )
+            elif env_config.half:
+                model_kwargs = dict(torch_dtype=torch.bfloat16)
+            try:
+                n_positions = (
+                    512  # Should fix later, but for T-5 this is 512 indeed
+                )
+                # mask_model.config.n_positions
+            except AttributeError:
                 n_positions = self.config.max_tokens
-            tokenizer_kwargs = {
-                &#34;model_max_length&#34;: n_positions,
-            }
+        else:
+            n_positions = self.config.max_tokens
+        tokenizer_kwargs = {
+            &#34;model_max_length&#34;: n_positions,
+        }
 
-        if &#34;t5&#34; in self.config.neighborhood_config.model:
+        print(f&#34;Loading mask filling model {neigh_config.model}...&#34;)
+        if &#34;t5&#34; in neigh_config.model:
             mask_model = T5Model(
                 self.config,
                 model_kwargs=model_kwargs,
                 tokenizer_kwargs=tokenizer_kwargs,
             )
-        elif &#34;bert&#34; in self.config.neighborhood_config.model:
+        elif &#34;bert&#34; in neigh_config.model:
             mask_model = BertModel(self.config)
         else:
-            raise ValueError(f&#34;Unknown model {self.config.neighborhood_config.model}&#34;)
+            raise ValueError(f&#34;Unknown model {neigh_config.model}&#34;)
+        # if config.dataset_member in [&#39;english&#39;, &#39;german&#39;]:
+        #     preproc_tokenizer = mask_tokenizer
         return mask_model
 
     def load(self):
@@ -1204,6 +1244,9 @@ <h3>Inherited members</h3>
         return neighbors
 
     def _attack(self, document, probs, tokens=None, **kwargs):
+        &#34;&#34;&#34;
+        Neighborhood attack score. Looks at difference in likelihood for given document and average likelihood of its neighbors
+        &#34;&#34;&#34;
         # documents here are actually neighbors
         batch_size = kwargs.get(&#34;batch_size&#34;, 4)
         substr_neighbors = kwargs.get(&#34;substr_neighbors&#34;, None)
@@ -1224,6 +1267,36 @@ <h3>Ancestors</h3>
 </ul>
 <h3>Methods</h3>
 <dl>
+<dt id="mimir.attacks.neighborhood.NeighborhoodAttack.create_fill_dictionary"><code class="name flex">
+<span>def <span class="ident">create_fill_dictionary</span></span>(<span>self, data)</span>
+</code></dt>
+<dd>
+<div class="desc"></div>
+<details class="source">
+<summary>
+<span>Expand source code</span>
+</summary>
+<pre><code class="python">def create_fill_dictionary(self, data):
+    neigh_config = self.config.neighborhood_config
+    if &#34;t5&#34; in neigh_config.model and neigh_config.random_fills:
+        if not self.config.pretokenized:
+            # TODO: maybe can be done if detokenized, but currently not supported
+            self.ref_model.create_fill_dictionary(data)</code></pre>
+</details>
+</dd>
+<dt id="mimir.attacks.neighborhood.NeighborhoodAttack.get_mask_model"><code class="name flex">
+<span>def <span class="ident">get_mask_model</span></span>(<span>self)</span>
+</code></dt>
+<dd>
+<div class="desc"></div>
+<details class="source">
+<summary>
+<span>Expand source code</span>
+</summary>
+<pre><code class="python">def get_mask_model(self):
+    return self.ref_model</code></pre>
+</details>
+</dd>
 <dt id="mimir.attacks.neighborhood.NeighborhoodAttack.get_neighbors"><code class="name flex">
 <span>def <span class="ident">get_neighbors</span></span>(<span>self, documents, **kwargs)</span>
 </code></dt>
@@ -1833,6 +1906,8 @@ <h4><code><a title="mimir.attacks.neighborhood.MaskFillingModel" href="#mimir.at
 <li>
 <h4><code><a title="mimir.attacks.neighborhood.NeighborhoodAttack" href="#mimir.attacks.neighborhood.NeighborhoodAttack">NeighborhoodAttack</a></code></h4>
 <ul class="">
+<li><code><a title="mimir.attacks.neighborhood.NeighborhoodAttack.create_fill_dictionary" href="#mimir.attacks.neighborhood.NeighborhoodAttack.create_fill_dictionary">create_fill_dictionary</a></code></li>
+<li><code><a title="mimir.attacks.neighborhood.NeighborhoodAttack.get_mask_model" href="#mimir.attacks.neighborhood.NeighborhoodAttack.get_mask_model">get_mask_model</a></code></li>
 <li><code><a title="mimir.attacks.neighborhood.NeighborhoodAttack.get_neighbors" href="#mimir.attacks.neighborhood.NeighborhoodAttack.get_neighbors">get_neighbors</a></code></li>
 </ul>
 </li>

docs/attacks/reference.html

@@ -41,6 +41,9 @@ <h1 class="title">Module <code>mimir.attacks.reference</code></h1>
         self.ref_model.load()
 
     def _attack(self, document, probs, tokens=None, **kwargs):
+        &#34;&#34;&#34;
+        Reference-based attack score. Performs difficulty calibration in model likelihood using a reference model.
+        &#34;&#34;&#34;
         loss = kwargs.get(&#39;loss&#39;, None)
         if loss is None:
             loss = self.model.get_ll(document, probs=probs, tokens=tokens)
@@ -75,6 +78,9 @@ <h2 class="section-title" id="header-classes">Classes</h2>
         self.ref_model.load()
 
     def _attack(self, document, probs, tokens=None, **kwargs):
+        &#34;&#34;&#34;
+        Reference-based attack score. Performs difficulty calibration in model likelihood using a reference model.
+        &#34;&#34;&#34;
         loss = kwargs.get(&#39;loss&#39;, None)
         if loss is None:
             loss = self.model.get_ll(document, probs=probs, tokens=tokens)