Update documentation

iamgroot42 · Feb 7, 2024 · ed510b9 · ed510b9
1 parent aece367
commit ed510b9
Show file tree

Hide file tree

Showing 14 changed files with 780 additions and 360 deletions.
diff --git a/docs/attacks/attack_utils.html b/docs/attacks/attack_utils.html
@@ -64,18 +64,10 @@ <h1 class="title">Module <code>mimir.attacks.attack_utils</code></h1>
     return texts
 
 
-def get_likelihood(logits, labels):
-    assert logits.shape[0] == 1
-    assert labels.shape[0] == 1
-
-    logits = logits.view(-1, logits.shape[-1])[:-1]
-    labels = labels.view(-1)[1:]
-    log_probs = torch.nn.functional.log_softmax(logits, dim=-1)
-    log_likelihood = log_probs.gather(dim=-1, index=labels.unsqueeze(-1)).squeeze(-1)
-    return log_likelihood.mean()
-
-
 def f1_score(prediction, ground_truth):
+    &#34;&#34;&#34;
+        Compute F1 score for given prediction and ground truth.
+    &#34;&#34;&#34;
     common = Counter(prediction) &amp; Counter(ground_truth)
     num_same = sum(common.values())
     if num_same == 0:
@@ -240,12 +232,15 @@ <h2 class="section-title" id="header-functions">Functions</h2>
 <span>def <span class="ident">f1_score</span></span>(<span>prediction, ground_truth)</span>
 </code></dt>
 <dd>
-<div class="desc"></div>
+<div class="desc"><p>Compute F1 score for given prediction and ground truth.</p></div>
 <details class="source">
 <summary>
 <span>Expand source code</span>
 </summary>
 <pre><code class="python">def f1_score(prediction, ground_truth):
+    &#34;&#34;&#34;
+        Compute F1 score for given prediction and ground truth.
+    &#34;&#34;&#34;
     common = Counter(prediction) &amp; Counter(ground_truth)
     num_same = sum(common.values())
     if num_same == 0:
@@ -299,26 +294,6 @@ <h2 class="section-title" id="header-functions">Functions</h2>
     return roc_auc</code></pre>
 </details>
 </dd>
-<dt id="mimir.attacks.attack_utils.get_likelihood"><code class="name flex">
-<span>def <span class="ident">get_likelihood</span></span>(<span>logits, labels)</span>
-</code></dt>
-<dd>
-<div class="desc"></div>
-<details class="source">
-<summary>
-<span>Expand source code</span>
-</summary>
-<pre><code class="python">def get_likelihood(logits, labels):
-    assert logits.shape[0] == 1
-    assert labels.shape[0] == 1
-
-    logits = logits.view(-1, logits.shape[-1])[:-1]
-    labels = labels.view(-1)[1:]
-    log_probs = torch.nn.functional.log_softmax(logits, dim=-1)
-    log_likelihood = log_probs.gather(dim=-1, index=labels.unsqueeze(-1)).squeeze(-1)
-    return log_likelihood.mean()</code></pre>
-</details>
-</dd>
 <dt id="mimir.attacks.attack_utils.get_precision_recall_metrics"><code class="name flex">
 <span>def <span class="ident">get_precision_recall_metrics</span></span>(<span>preds_member, preds_nonmember)</span>
 </code></dt>
@@ -435,7 +410,6 @@ <h1>Index</h1>
 <li><code><a title="mimir.attacks.attack_utils.f1_score" href="#mimir.attacks.attack_utils.f1_score">f1_score</a></code></li>
 <li><code><a title="mimir.attacks.attack_utils.filter_out_nan" href="#mimir.attacks.attack_utils.filter_out_nan">filter_out_nan</a></code></li>
 <li><code><a title="mimir.attacks.attack_utils.get_auc_from_thresholds" href="#mimir.attacks.attack_utils.get_auc_from_thresholds">get_auc_from_thresholds</a></code></li>
-<li><code><a title="mimir.attacks.attack_utils.get_likelihood" href="#mimir.attacks.attack_utils.get_likelihood">get_likelihood</a></code></li>
 <li><code><a title="mimir.attacks.attack_utils.get_precision_recall_metrics" href="#mimir.attacks.attack_utils.get_precision_recall_metrics">get_precision_recall_metrics</a></code></li>
 <li><code><a title="mimir.attacks.attack_utils.get_roc_metrics" href="#mimir.attacks.attack_utils.get_roc_metrics">get_roc_metrics</a></code></li>
 </ul>

diff --git a/docs/attacks/blackbox_attacks.html b/docs/attacks/blackbox_attacks.html
@@ -37,35 +37,61 @@ <h1 class="title">Module <code>mimir.attacks.blackbox_attacks</code></h1>
 
 # Attack definitions
 class BlackBoxAttacks(str, Enum):
-    LOSS = &#34;loss&#34;
-    REFERENCE_BASED = &#34;ref&#34;
-    ZLIB = &#34;zlib&#34;
-    MIN_K = &#34;min_k&#34;
-    NEIGHBOR = &#34;ne&#34;
+    LOSS = &#34;loss&#34; # Done
+    REFERENCE_BASED = &#34;ref&#34; # Done
+    ZLIB = &#34;zlib&#34; # Done
+    MIN_K = &#34;min_k&#34; # Done
+    NEIGHBOR = &#34;ne&#34; # Done
     QUANTILE = &#34;quantile&#34;
 
 
-# TODO: Move attacks in models into this file as functions
-# TODO Use decorators to link attack implementations with enum above
-
 # Base attack class
 class Attack:
     def __init__(self, config, target_model: Model, ref_model: Model = None):
         self.config = config
         self.target_model = target_model
         self.ref_model = ref_model
+        self.is_loaded = False
 
-    def prepare(self, **kwargs):
+    def load(self):
         &#34;&#34;&#34;
         Any attack-specific steps (one-time) preparation
         &#34;&#34;&#34;
         pass
 
-    def attack(self, document, **kwargs):
+    def unload(self):
+        if self.ref_model is not None:
+            self.ref_model.unload()
+            self.is_loaded = False
+
+    def _attack(self, document, probs, tokens=None, **kwargs):
         &#34;&#34;&#34;
-        Score a document using the attack&#39;s scoring function
+        Actual logic for attack. 
+        &#34;&#34;&#34;
+        raise NotImplementedError(&#34;Attack must implement attack()&#34;)
+
+    def attack(self, document, probs, **kwargs):
         &#34;&#34;&#34;
-        raise NotImplementedError(&#34;Attack must implement attack()&#34;)</code></pre>
+        Score a document using the attack&#39;s scoring function. Calls self._attack
+        &#34;&#34;&#34;
+        # Load attack if not loaded yet
+        if not self.is_loaded:
+            self.load()
+            self.is_loaded = True
+
+        detokenized_sample = kwargs.get(&#34;detokenized_sample&#34;, None)
+        if self.config.pretokenized and detokenized_sample is None:
+            raise ValueError(&#34;detokenized_sample must be provided&#34;)
+
+        score = (
+            self._attack(document, probs=probs, **kwargs)
+            if not self.config.pretokenized
+            else self._attack(
+                detokenized_sample, tokens=document, probs=probs, **kwargs
+            )
+        )
+
+        return score</code></pre>
 </details>
 </section>
 <section>
@@ -92,60 +118,123 @@ <h2 class="section-title" id="header-classes">Classes</h2>
         self.config = config
         self.target_model = target_model
         self.ref_model = ref_model
+        self.is_loaded = False
 
-    def prepare(self, **kwargs):
+    def load(self):
         &#34;&#34;&#34;
         Any attack-specific steps (one-time) preparation
         &#34;&#34;&#34;
         pass
 
-    def attack(self, document, **kwargs):
+    def unload(self):
+        if self.ref_model is not None:
+            self.ref_model.unload()
+            self.is_loaded = False
+
+    def _attack(self, document, probs, tokens=None, **kwargs):
         &#34;&#34;&#34;
-        Score a document using the attack&#39;s scoring function
+        Actual logic for attack. 
+        &#34;&#34;&#34;
+        raise NotImplementedError(&#34;Attack must implement attack()&#34;)
+
+    def attack(self, document, probs, **kwargs):
         &#34;&#34;&#34;
-        raise NotImplementedError(&#34;Attack must implement attack()&#34;)</code></pre>
+        Score a document using the attack&#39;s scoring function. Calls self._attack
+        &#34;&#34;&#34;
+        # Load attack if not loaded yet
+        if not self.is_loaded:
+            self.load()
+            self.is_loaded = True
+
+        detokenized_sample = kwargs.get(&#34;detokenized_sample&#34;, None)
+        if self.config.pretokenized and detokenized_sample is None:
+            raise ValueError(&#34;detokenized_sample must be provided&#34;)
+
+        score = (
+            self._attack(document, probs=probs, **kwargs)
+            if not self.config.pretokenized
+            else self._attack(
+                detokenized_sample, tokens=document, probs=probs, **kwargs
+            )
+        )
+
+        return score</code></pre>
 </details>
 <h3>Subclasses</h3>
 <ul class="hlist">
 <li><a title="mimir.attacks.loss.LOSSAttack" href="loss.html#mimir.attacks.loss.LOSSAttack">LOSSAttack</a></li>
+<li><a title="mimir.attacks.min_k.MinKProbAttack" href="min_k.html#mimir.attacks.min_k.MinKProbAttack">MinKProbAttack</a></li>
 <li><a title="mimir.attacks.neighborhood.NeighborhoodAttack" href="neighborhood.html#mimir.attacks.neighborhood.NeighborhoodAttack">NeighborhoodAttack</a></li>
 <li><a title="mimir.attacks.quantile.QuantileAttack" href="quantile.html#mimir.attacks.quantile.QuantileAttack">QuantileAttack</a></li>
 <li><a title="mimir.attacks.reference.ReferenceAttack" href="reference.html#mimir.attacks.reference.ReferenceAttack">ReferenceAttack</a></li>
+<li><a title="mimir.attacks.zlib.ZLIBAttack" href="zlib.html#mimir.attacks.zlib.ZLIBAttack">ZLIBAttack</a></li>
 </ul>
 <h3>Methods</h3>
 <dl>
 <dt id="mimir.attacks.blackbox_attacks.Attack.attack"><code class="name flex">
-<span>def <span class="ident">attack</span></span>(<span>self, document, **kwargs)</span>
+<span>def <span class="ident">attack</span></span>(<span>self, document, probs, **kwargs)</span>
 </code></dt>
 <dd>
-<div class="desc"><p>Score a document using the attack's scoring function</p></div>
+<div class="desc"><p>Score a document using the attack's scoring function. Calls self._attack</p></div>
 <details class="source">
 <summary>
 <span>Expand source code</span>
 </summary>
-<pre><code class="python">def attack(self, document, **kwargs):
+<pre><code class="python">def attack(self, document, probs, **kwargs):
     &#34;&#34;&#34;
-    Score a document using the attack&#39;s scoring function
+    Score a document using the attack&#39;s scoring function. Calls self._attack
     &#34;&#34;&#34;
-    raise NotImplementedError(&#34;Attack must implement attack()&#34;)</code></pre>
+    # Load attack if not loaded yet
+    if not self.is_loaded:
+        self.load()
+        self.is_loaded = True
+
+    detokenized_sample = kwargs.get(&#34;detokenized_sample&#34;, None)
+    if self.config.pretokenized and detokenized_sample is None:
+        raise ValueError(&#34;detokenized_sample must be provided&#34;)
+
+    score = (
+        self._attack(document, probs=probs, **kwargs)
+        if not self.config.pretokenized
+        else self._attack(
+            detokenized_sample, tokens=document, probs=probs, **kwargs
+        )
+    )
+
+    return score</code></pre>
 </details>
 </dd>
-<dt id="mimir.attacks.blackbox_attacks.Attack.prepare"><code class="name flex">
-<span>def <span class="ident">prepare</span></span>(<span>self, **kwargs)</span>
+<dt id="mimir.attacks.blackbox_attacks.Attack.load"><code class="name flex">
+<span>def <span class="ident">load</span></span>(<span>self)</span>
 </code></dt>
 <dd>
 <div class="desc"><p>Any attack-specific steps (one-time) preparation</p></div>
 <details class="source">
 <summary>
 <span>Expand source code</span>
 </summary>
-<pre><code class="python">def prepare(self, **kwargs):
+<pre><code class="python">def load(self):
     &#34;&#34;&#34;
     Any attack-specific steps (one-time) preparation
     &#34;&#34;&#34;
     pass</code></pre>
 </details>
 </dd>
+<dt id="mimir.attacks.blackbox_attacks.Attack.unload"><code class="name flex">
+<span>def <span class="ident">unload</span></span>(<span>self)</span>
+</code></dt>
+<dd>
+<div class="desc"></div>
+<details class="source">
+<summary>
+<span>Expand source code</span>
+</summary>
+<pre><code class="python">def unload(self):
+    if self.ref_model is not None:
+        self.ref_model.unload()
+        self.is_loaded = False</code></pre>
+</details>
+</dd>
 </dl>
 </dd>
 <dt id="mimir.attacks.blackbox_attacks.BlackBoxAttacks"><code class="flex name class">
@@ -159,11 +248,11 @@ <h3>Methods</h3>
 <span>Expand source code</span>
 </summary>
 <pre><code class="python">class BlackBoxAttacks(str, Enum):
-    LOSS = &#34;loss&#34;
-    REFERENCE_BASED = &#34;ref&#34;
-    ZLIB = &#34;zlib&#34;
-    MIN_K = &#34;min_k&#34;
-    NEIGHBOR = &#34;ne&#34;
+    LOSS = &#34;loss&#34; # Done
+    REFERENCE_BASED = &#34;ref&#34; # Done
+    ZLIB = &#34;zlib&#34; # Done
+    MIN_K = &#34;min_k&#34; # Done
+    NEIGHBOR = &#34;ne&#34; # Done
     QUANTILE = &#34;quantile&#34;</code></pre>
 </details>
 <h3>Ancestors</h3>
@@ -219,7 +308,8 @@ <h1>Index</h1>
 <h4><code><a title="mimir.attacks.blackbox_attacks.Attack" href="#mimir.attacks.blackbox_attacks.Attack">Attack</a></code></h4>
 <ul class="">
 <li><code><a title="mimir.attacks.blackbox_attacks.Attack.attack" href="#mimir.attacks.blackbox_attacks.Attack.attack">attack</a></code></li>
-<li><code><a title="mimir.attacks.blackbox_attacks.Attack.prepare" href="#mimir.attacks.blackbox_attacks.Attack.prepare">prepare</a></code></li>
+<li><code><a title="mimir.attacks.blackbox_attacks.Attack.load" href="#mimir.attacks.blackbox_attacks.Attack.load">load</a></code></li>
+<li><code><a title="mimir.attacks.blackbox_attacks.Attack.unload" href="#mimir.attacks.blackbox_attacks.Attack.unload">unload</a></code></li>
 </ul>
 </li>
 <li>

diff --git a/docs/attacks/index.html b/docs/attacks/index.html
@@ -47,6 +47,10 @@ <h2 class="section-title" id="header-submodules">Sub-modules</h2>
 <dd>
 <div class="desc"><p>Straight-forward LOSS attack</p></div>
 </dd>
+<dt><code class="name"><a title="mimir.attacks.min_k" href="min_k.html">mimir.attacks.min_k</a></code></dt>
+<dd>
+<div class="desc"><p>Min-k % Prob Attack: <a href="https://arxiv.org/pdf/2310.16789.pdf">https://arxiv.org/pdf/2310.16789.pdf</a></p></div>
+</dd>
 <dt><code class="name"><a title="mimir.attacks.neighborhood" href="neighborhood.html">mimir.attacks.neighborhood</a></code></dt>
 <dd>
 <div class="desc"><p>Neighborhood-MIA attack <a href="https://arxiv.org/pdf/2305.18462.pdf">https://arxiv.org/pdf/2305.18462.pdf</a></p></div>
@@ -60,6 +64,14 @@ <h2 class="section-title" id="header-submodules">Sub-modules</h2>
 <dd>
 <div class="desc"><p>Reference-based attacks.</p></div>
 </dd>
+<dt><code class="name"><a title="mimir.attacks.utils" href="utils.html">mimir.attacks.utils</a></code></dt>
+<dd>
+<div class="desc"></div>
+</dd>
+<dt><code class="name"><a title="mimir.attacks.zlib" href="zlib.html">mimir.attacks.zlib</a></code></dt>
+<dd>
+<div class="desc"><p>zlib-normalization Attack: <a href="https://www.usenix.org/system/files/sec21-carlini-extracting.pdf">https://www.usenix.org/system/files/sec21-carlini-extracting.pdf</a></p></div>
+</dd>
 </dl>
 </section>
 <section>
@@ -85,9 +97,12 @@ <h1>Index</h1>
 <li><code><a title="mimir.attacks.attack_utils" href="attack_utils.html">mimir.attacks.attack_utils</a></code></li>
 <li><code><a title="mimir.attacks.blackbox_attacks" href="blackbox_attacks.html">mimir.attacks.blackbox_attacks</a></code></li>
 <li><code><a title="mimir.attacks.loss" href="loss.html">mimir.attacks.loss</a></code></li>
+<li><code><a title="mimir.attacks.min_k" href="min_k.html">mimir.attacks.min_k</a></code></li>
 <li><code><a title="mimir.attacks.neighborhood" href="neighborhood.html">mimir.attacks.neighborhood</a></code></li>
 <li><code><a title="mimir.attacks.quantile" href="quantile.html">mimir.attacks.quantile</a></code></li>
 <li><code><a title="mimir.attacks.reference" href="reference.html">mimir.attacks.reference</a></code></li>
+<li><code><a title="mimir.attacks.utils" href="utils.html">mimir.attacks.utils</a></code></li>
+<li><code><a title="mimir.attacks.zlib" href="zlib.html">mimir.attacks.zlib</a></code></li>
 </ul>
 </li>
 </ul>