Add large 3 and 4 evaluations

f147f318 · Albin Henriksson · 23a6821e · f147f318 · f147f318 · f147f318
Commit f147f318 authored 2 years ago by Albin Henriksson
--- a/bart/sparql.ipynb
+++ b/bart/sparql.ipynb
--- a/data/__pycache__/eval_query.cpython-39.pyc
+++ b/data/__pycache__/eval_query.cpython-39.pyc
--- a/data/eval/lc-quad-requeried-linked-test-eval-append-1-large-3-epochs.txt
+++ b/data/eval/lc-quad-requeried-linked-test-eval-append-1-large-3-epochs.txt
+String evaluation
+
+Precision macro: 0.8030609762521919
+Recall macro: 0.7913620401992508
+F1 macro: 0.7947304198110867
+Precision micro: 0.7912972085385879
+Recall micro: 0.7802153323079414
+F1 micro: 0.7857171972445279
+Fully correct: 162
+
+Query evaluation
+
+Precision macro: 0.34168246759219223
+Recall macro: 0.34538766263173165
+F1 macro: 0.33773159161379057
+Precision micro: 0.2601919788161299
+Recall micro: 0.34827395237216047
+F1 micro: 0.29785756006408937
+Fully correct: 358
+
--- a/data/eval/lc-quad-requeried-linked-test-eval-append-1-large-4-epochs.txt
+++ b/data/eval/lc-quad-requeried-linked-test-eval-append-1-large-4-epochs.txt
+String evaluation
+
+Precision macro: 0.7982044656463279
+Recall macro: 0.7850570068270347
+F1 macro: 0.7890307366150056
+Precision micro: 0.787020260253665
+Recall micro: 0.7735772686796729
+F1 micro: 0.7802408654827515
+Fully correct: 152
+
+Query evaluation
+
+Precision macro: 0.3288445236328654
+Recall macro: 0.33148790322402794
+F1 macro: 0.3254230404437138
+Precision micro: 0.4538291509556613
+Recall micro: 0.6749951653451918
+F1 micro: 0.5427460736574067
+Fully correct: 345
+
--- a/data/pred_responses/lc-quad-requeried-linked-test-pred_responses-append-1-large-3-epochs.json
+++ b/data/pred_responses/lc-quad-requeried-linked-test-pred_responses-append-1-large-3-epochs.json
--- a/data/pred_responses/lc-quad-requeried-linked-test-pred_responses-append-1-large-4-epochs.json
+++ b/data/pred_responses/lc-quad-requeried-linked-test-pred_responses-append-1-large-4-epochs.json
--- a/data/predicted/lc-quad-requeried-linked-test-predicted-append-1-4-epochs.csv
+++ b/data/predicted/lc-quad-requeried-linked-test-predicted-append-1-4-epochs.csv
--- a/data/predicted/lc-quad-requeried-linked-test-predicted-append-1-large-4-epochs.csv
+++ b/data/predicted/lc-quad-requeried-linked-test-predicted-append-1-large-4-epochs.csv
--- a/evaluate.ipynb
+++ b/evaluate.ipynb
@@ -2,7 +2,7 @@
 "cells": [
  {
   "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -11,27 +11,34 @@
    "\n",
    "# Specify the correct and predicted query responses\n",
    "input_path= \"data/lc-quad-requeried-linked-test.json\"\n",
-    "pred_path = \"data/predicted/lc-quad-requeried-linked-test-predicted-append-1-large-10-epochs.csv\"\n",
+    "pred_path = \"data/predicted/lc-quad-requeried-linked-test-predicted-append-1-large-4-epochs.csv\"\n",
    "dump_path = pred_path.replace(\"predicted\", \"pred_responses\").replace(\".csv\", \".json\")\n"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
-      "1161it [03:22,  5.74it/s]\n"
+      "1161it [03:03,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "Errors: 137\n"
+      "Errors: 15\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
     ]
    }
   ],
@@ -49,18 +56,38 @@
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "Evaluation againts server results\n"
+      "Evaluation againts server results\n",
+      "Begin evaluation\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 1161/1161 [00:00<00:00, 67736.64it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Precision macro:  0.34168246759219223\n",
+      "Recall macro:  0.34538766263173165\n",
+      "F1 macro:  0.33773159161379057\n",
+      "Precision micro:  0.2601919788161299\n",
+      "Recall micro:  0.34827395237216047\n",
+      "F1 micro:  0.29785756006408937\n",
+      "TP micro:  9433\n",
+      "FP micro:  26821\n",
+      "FN micro:  17652\n",
+      "Fully correct:  358\n"
     ]
    },
    {
-     "ename": "NameError",
-     "evalue": "name 'dump_path' is not defined",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
-      "\u001b[1;32mc:\\Users\\Albin\\workspace\\codebase\\evaluate.ipynb Cell 3\u001b[0m in \u001b[0;36m<cell line: 3>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/Albin/workspace/codebase/evaluate.ipynb#W2sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39m# Evaluate expected query response, against the predicted query response\u001b[39;00m\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/Albin/workspace/codebase/evaluate.ipynb#W2sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mEvaluation againts server results\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/Albin/workspace/codebase/evaluate.ipynb#W2sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m precision_macro_query, recall_macro_query, f1_macro_query, precision_micro_query, recall_micro_query, f1_micro_query, fully_correct_query \u001b[39m=\u001b[39m eq\u001b[39m.\u001b[39meval_query_response(input_path, dump_path)\n",
-      "\u001b[1;31mNameError\u001b[0m: name 'dump_path' is not defined"
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
     ]
    }
   ],
@@ -73,25 +100,45 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "Evaluation of queries as strings\n"
+      "Evaluation of queries as strings\n",
+      "Begin evaluation\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 1161/1161 [00:00<00:00, 193337.31it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Precision macro:  0.8030609762521919\n",
+      "Recall macro:  0.7913620401992508\n",
+      "F1 macro:  0.7947304198110867\n",
+      "Precision micro:  0.7912972085385879\n",
+      "Recall micro:  0.7802153323079414\n",
+      "F1 micro:  0.7857171972445279\n",
+      "TP micro:  9638\n",
+      "FP micro:  2542\n",
+      "FN micro:  2715\n",
+      "Fully correct:  162\n"
     ]
    },
    {
-     "ename": "NameError",
-     "evalue": "name 'eq' is not defined",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
-      "\u001b[1;32mc:\\Users\\Albin\\workspace\\codebase\\evaluate.ipynb Cell 4\u001b[0m in \u001b[0;36m<cell line: 3>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/Albin/workspace/codebase/evaluate.ipynb#W3sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39m# Evaluate the expected query against the predicted query as string likness.\u001b[39;00m\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/Albin/workspace/codebase/evaluate.ipynb#W3sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mEvaluation of queries as strings\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/Albin/workspace/codebase/evaluate.ipynb#W3sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m precision_macro_string, recall_macro_string, f1_macro_string, precision_micro_string, recall_micro_string, f1_micro_string, fully_correct_string \u001b[39m=\u001b[39m eq\u001b[39m.\u001b[39meval_query_json(input_path, dump_path)\n",
-      "\u001b[1;31mNameError\u001b[0m: name 'eq' is not defined"
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
     ]
    }
   ],
@@ -103,7 +150,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [

 %% Cell type:code id: tags:

 ``` python
 from data import eval_query as eq
 from data import pred_query_responses as pqr

 # Specify the correct and predicted query responses
 input_path= "data/lc-quad-requeried-linked-test.json"
-pred_path = "data/predicted/lc-quad-requeried-linked-test-predicted-append-1-large-10-epochs.csv"
+pred_path = "data/predicted/lc-quad-requeried-linked-test-predicted-append-1-large-4-epochs.csv"
 dump_path = pred_path.replace("predicted", "pred_responses").replace(".csv", ".json")
 ```

 %% Cell type:code id: tags:

 ``` python
 # Runs the predicted queries against the qald-10 server, then generates a file of responses, and returns path to it
 pqr.build_responsefile(dump_path, input_path, pred_path)
 ```

 %% Output

-    1161it [03:22,  5.74it/s]
+    1161it [03:03,  6.33it/s]

-    Errors: 137
+    Errors: 15
+
+    

 %% Cell type:code id: tags:

 ``` python

 # Evaluate expected query response, against the predicted query response
 print("Evaluation againts server results")
 precision_macro_query, recall_macro_query, f1_macro_query, precision_micro_query, recall_micro_query, f1_micro_query, fully_correct_query = eq.eval_query_response(input_path, dump_path)
 ```

 %% Output

    Evaluation againts server results
+    Begin evaluation
+
+    100%|██████████| 1161/1161 [00:00<00:00, 67736.64it/s]
+
+    Precision macro:  0.34168246759219223
+    Recall macro:  0.34538766263173165
+    F1 macro:  0.33773159161379057
+    Precision micro:  0.2601919788161299
+    Recall micro:  0.34827395237216047
+    F1 micro:  0.29785756006408937
+    TP micro:  9433
+    FP micro:  26821
+    FN micro:  17652
+    Fully correct:  358

-    ---------------------------------------------------------------------------
-    NameError                                 Traceback (most recent call last)
-    c:\Users\Albin\workspace\codebase\evaluate.ipynb Cell 3 in <cell line: 3>()
-          <a href='vscode-notebook-cell:/c%3A/Users/Albin/workspace/codebase/evaluate.ipynb#W2sZmlsZQ%3D%3D?line=0'>1</a> # Evaluate expected query response, against the predicted query response
-          <a href='vscode-notebook-cell:/c%3A/Users/Albin/workspace/codebase/evaluate.ipynb#W2sZmlsZQ%3D%3D?line=1'>2</a> print("Evaluation againts server results")
-    ----> <a href='vscode-notebook-cell:/c%3A/Users/Albin/workspace/codebase/evaluate.ipynb#W2sZmlsZQ%3D%3D?line=2'>3</a> precision_macro_query, recall_macro_query, f1_macro_query, precision_micro_query, recall_micro_query, f1_micro_query, fully_correct_query = eq.eval_query_response(input_path, dump_path)
-    NameError: name 'dump_path' is not defined
+    

 %% Cell type:code id: tags:

 ``` python
 # Evaluate the expected query against the predicted query as string likness.
 print("Evaluation of queries as strings")
 precision_macro_string, recall_macro_string, f1_macro_string, precision_micro_string, recall_micro_string, f1_micro_string, fully_correct_string = eq.eval_query_json(input_path, dump_path)
 ```

 %% Output

    Evaluation of queries as strings
+    Begin evaluation
+
+    100%|██████████| 1161/1161 [00:00<00:00, 193337.31it/s]
+
+    Precision macro:  0.8030609762521919
+    Recall macro:  0.7913620401992508
+    F1 macro:  0.7947304198110867
+    Precision micro:  0.7912972085385879
+    Recall micro:  0.7802153323079414
+    F1 micro:  0.7857171972445279
+    TP micro:  9638
+    FP micro:  2542
+    FN micro:  2715
+    Fully correct:  162

-    ---------------------------------------------------------------------------
-    NameError                                 Traceback (most recent call last)
-    c:\Users\Albin\workspace\codebase\evaluate.ipynb Cell 4 in <cell line: 3>()
-          <a href='vscode-notebook-cell:/c%3A/Users/Albin/workspace/codebase/evaluate.ipynb#W3sZmlsZQ%3D%3D?line=0'>1</a> # Evaluate the expected query against the predicted query as string likness.
-          <a href='vscode-notebook-cell:/c%3A/Users/Albin/workspace/codebase/evaluate.ipynb#W3sZmlsZQ%3D%3D?line=1'>2</a> print("Evaluation of queries as strings")
-    ----> <a href='vscode-notebook-cell:/c%3A/Users/Albin/workspace/codebase/evaluate.ipynb#W3sZmlsZQ%3D%3D?line=2'>3</a> precision_macro_string, recall_macro_string, f1_macro_string, precision_micro_string, recall_micro_string, f1_micro_string, fully_correct_string = eq.eval_query_json(input_path, dump_path)
-    NameError: name 'eq' is not defined
+    

 %% Cell type:code id: tags:

 ``` python
 # Save the results to a file
 res_path = dump_path.replace("pred_responses", "eval").replace(".json", ".txt")
 with open(res_path, "w") as f:
    f.write("String evaluation\n\n")
    f.write(f"Precision macro: {precision_macro_string}\n")
    f.write(f"Recall macro: {recall_macro_string}\n")
    f.write(f"F1 macro: {f1_macro_string}\n")
    f.write(f"Precision micro: {precision_micro_string}\n")
    f.write(f"Recall micro: {recall_micro_string}\n")
    f.write(f"F1 micro: {f1_micro_string}\n")
    f.write(f"Fully correct: {fully_correct_string}\n\n")
    f.write("Query evaluation\n\n")
    f.write(f"Precision macro: {precision_macro_query}\n")
    f.write(f"Recall macro: {recall_macro_query}\n")
    f.write(f"F1 macro: {f1_macro_query}\n")
    f.write(f"Precision micro: {precision_micro_query}\n")
    f.write(f"Recall micro: {recall_micro_query}\n")
    f.write(f"F1 micro: {f1_micro_query}\n")
    f.write(f"Fully correct: {fully_correct_query}\n\n")

 ```