Merge pull request #15 from iamgroot42/michael/gpt_em_analysis_fix

Fixed incorrect reference in gpt EM analysis
iamgroot42 · Mar 1, 2024 · 0285c77 · 0285c77
2 parents ed98d1e + 0054c70
commit 0285c77
Showing 1 changed file with 14 additions and 14 deletions.
diff --git a/inspect_mi.ipynb b/inspect_mi.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 1,
    "id": "7624f659",
    "metadata": {},
    "outputs": [],
@@ -28,7 +28,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 2,
    "id": "19c20da5",
    "metadata": {},
    "outputs": [],
@@ -53,7 +53,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 3,
    "id": "f18f952b",
    "metadata": {},
    "outputs": [],
@@ -66,7 +66,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 4,
    "id": "76de7545",
    "metadata": {},
    "outputs": [],
@@ -104,7 +104,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 64,
+   "execution_count": 5,
    "id": "040eeb79",
    "metadata": {},
    "outputs": [],
@@ -118,7 +118,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 65,
+   "execution_count": 6,
    "id": "6a089015",
    "metadata": {},
    "outputs": [],
@@ -129,7 +129,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 66,
+   "execution_count": 7,
    "id": "dd97043c",
    "metadata": {},
    "outputs": [],
@@ -926,7 +926,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 12,
    "id": "57635b7b",
    "metadata": {},
    "outputs": [
@@ -1011,23 +1011,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 18,
    "id": "dcb5894d",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "arxiv: 1.000\n",
       "arxiv: 0.002\n",
-      "hackernews: 0.998\n",
-      "hackernews: 0.000\n"
+      "arxiv: 0.007\n",
+      "hackernews: 0.011\n",
+      "hackernews: 0.001\n"
      ]
     }
    ],
    "source": [
-    "fpr_rate = 0.01\n",
+    "fpr_rate = 0.05\n",
     "attacks = ['loss', 'ref-stablelm-base-alpha-3b-v2']\n",
     "# atk = attacks[0]\n",
     "# FPR 0 for LOSS attack\n",
@@ -1044,7 +1044,7 @@
     "        path = f\"/mmfs1/gscratch/h2lab/micdun/mimir/edit_distance_members/scores/gpt4/gpt_paraphrase_results_{source}.json\"\n",
     "        with open(path, 'r') as f:\n",
     "            gpt_data = json.load(f)\n",
-    "            gpt_data = gpt_data[attack_focus]['gpt']\n",
+    "            gpt_data = gpt_data[atk]['gpt']\n",
     "            care = - np.array(list(gpt_data.values()))\n",
     "\n",
     "        # Compute FPR for this threshold\n",