Save analysis

Author: peterdemin

anal.ipynb

@@ -37,28 +37,48 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 4,
+   "id": "9cf09f71-6bda-4ce8-af77-56ceb6807f98",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Counter({'X:': 375, 'Y:': 369})"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "collections.Counter(result['answer'] for result in load_results(RESULTS_PATH))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
    "id": "aaea1b19-acce-4476-b7e2-686fb90139c1",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "Counter({('A', False): 154,\n",
-       "         ('A', True): 102,\n",
-       "         ('B', True): 98,\n",
-       "         ('B', False): 47,\n",
-       "         (None, True): 1})"
+       "Counter({('B', True): 341,\n",
+       "         ('A', False): 338,\n",
+       "         ('B', False): 34,\n",
+       "         ('A', True): 31})"
       ]
      },
-     "execution_count": 12,
+     "execution_count": 5,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "RE_ANSWER = re.compile(r'[AB12]')\n",
-    "COALESCE = {'1': 'A', '2': 'B', 'A': 'A', 'B': 'B'}\n",
+    "RE_ANSWER = re.compile(r'[ABXY12]')\n",
+    "COALESCE = {'1': 'A', '2': 'B', 'A': 'A', 'B': 'B', 'Y': 'A', 'X': 'B'}\n",
     "\n",
     "def parse_answer(answer):\n",
     "    matches = RE_ANSWER.findall(answer)\n",
@@ -75,24 +95,24 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 6,
    "id": "97593984-1e89-4b4e-833f-268e6a3ea8d1",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "Counter({'A': 256, 'B': 145, None: 1})"
+       "Counter({'B': 375, 'A': 369})"
       ]
      },
-     "execution_count": 5,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
     "collections.Counter(\n",
-    "    (parse_answer(result['answer']))\n",
+    "    parse_answer(result['answer'])\n",
     "    for result in load_results(RESULTS_PATH)\n",
     ")"
    ]
@@ -119,14 +139,11 @@
    "metadata": {},
    "outputs": [
     {
-     "data": {
-      "text/plain": [
-       "{'AB': 61, 'AA': 60, 'BA': 16, 'BB': 13, 'U': 0}"
-      ]
-     },
-     "execution_count": 8,
-     "metadata": {},
-     "output_type": "execute_result"
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Accuracy: 84% Breakdown: AB=179  AA=8  BA=11  BB=14  U=0\n"
+     ]
     }
    ],
    "source": [
@@ -161,44 +178,25 @@
     "    return score\n",
     "\n",
     "\n",
-    "stable_score(aggregate_swapped_answers(load_results(RESULTS_PATH)))"
+    "scores = stable_score(aggregate_swapped_answers(load_results(RESULTS_PATH)))\n",
+    "acc = int(100 * scores['AB'] / sum(scores.values()))\n",
+    "scores_fmt = '  '.join(f'{k}={v}' for k, v in scores.items())\n",
+    "print(f'Accuracy: {acc}% Breakdown: {scores_fmt}')"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 9,
-   "id": "78dba699-28e8-497f-bd59-77fb4da48278",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{\"'terrorist' and'd ** k' can also be included in messages.\": {True: None}}"
-      ]
-     },
-     "execution_count": 9,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "results = [{'article_id': '6aa21c738f5c5c2fa7b1cad4f3dba97cc9634878', 'article_sent': \"'terrorist' and'd ** k' can also be included in messages.\", 'correct_sent': \"** k' can also be included in messages.\", 'correct_sent_id': '82-2', 'incorrect_sent': \"** k'' can also be included in messages.\", 'incorrect_sent_id': '68-2', 'answer': ':', 'swap': True}]\n",
-    "aggregate_swapped_answers(results)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
    "id": "da842095-9d47-49cf-80d3-facb795f097b",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "Counter({True: 252, False: 149, None: 1})"
+       "Counter({True: 679, False: 65})"
       ]
      },
-     "execution_count": 10,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -209,14 +207,6 @@
     "    for result in load_results(RESULTS_PATH)\n",
     ")"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "5c1f0345-1a13-42ee-b545-68f56fb70770",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {