Added train and evaluation script

bart/generate_single_query.py

@@ -44,8 +44,6 @@ def predict_query(question, model, tokenizer):
     # Filter everything before the first " and after the last "
     predicted = predicted[predicted.find('"') + 1:predicted.rfind('"')]
 
-    print(predicted)
-
     # Replace all prefixes with their full URI
     for prefix, uri in prefixes.items():
         # Delete instances of dbo: abcd (with a space after the colon)

data/eval/lc-quad-requeried-linked-test-eval-append-1-10-epochs.txt

-String evaluation
-
-Precision macro: 0.7999486421321076
-Recall macro: 0.7918983280094406
-F1 macro: 0.7940058554799921
-Precision micro: 0.7883593558407586
-Recall micro: 0.7807010442807415
-F1 micro: 0.7845115106157975
-Fully correct: 159
-
-Query evaluation
-
-Precision macro: 0.33311461377912216
-Recall macro: 0.3381097337634078
-F1 macro: 0.33103393573338913
-Precision micro: 0.4086352134781064
-Recall micro: 0.6218258029926372
-F1 micro: 0.49317716180609467
-Fully correct: 354
-

data/eval/lc-quad-requeried-linked-test-eval-append-1-2-epochs.txt

-String evaluation
-
-Precision macro: 0.7983994066035431
-Recall macro: 0.783192489394042
-F1 macro: 0.7885399312080449
-Precision micro: 0.7865576748410535
-Recall micro: 0.7711487088156723
-F1 micro: 0.7787769784172661
-Fully correct: 160
-
-Query evaluation
-
-Precision macro: 0.3363067516286216
-Recall macro: 0.3336455226774758
-F1 macro: 0.32982010297836395
-Precision micro: 0.3767318069126427
-Recall micro: 0.41762994099102574
-F1 micro: 0.39612804288668463
-Fully correct: 350
-

data/eval/lc-quad-requeried-linked-test-eval-append-1-20-epochs.txt

-String evaluation
-
-Precision macro: 0.7969362224530196
-Recall macro: 0.7926608896893147
-F1 macro: 0.7927323139690946
-Precision micro: 0.78564453125
-Recall micro: 0.7815105642354084
-F1 micro: 0.7835720952883406
-Fully correct: 157
-
-Query evaluation
-
-Precision macro: 0.3064400609934479
-Recall macro: 0.30951725425841553
-F1 macro: 0.3029736426731874
-Precision micro: 0.291126333403053
-Recall micro: 0.702789699570811
-F1 micro: 0.4117059823589308
-Fully correct: 326
-

data/eval/lc-quad-requeried-linked-test-eval-append-1-3-epochs.txt

-String evaluation
-
-Precision macro: 0.7951440630381219
-Recall macro: 0.7821657909513223
-F1 macro: 0.7865405222942252
-Precision micro: 0.784142421495096
-Recall micro: 0.7701772848700721
-F1 micro: 0.7770971167197582
-Fully correct: 165
-
-Query evaluation
-
-Precision macro: 0.34744571448520384
-Recall macro: 0.34682380342766167
-F1 macro: 0.34174408432858916
-Precision micro: 0.3486522643940136
-Recall micro: 0.7296246370800461
-F1 micro: 0.47183665254123297
-Fully correct: 359
-

data/eval/lc-quad-requeried-linked-test-eval-append-1-4-epochs.txt

-String evaluation
-
-Precision macro: 0.7997795749087769
-Recall macro: 0.7902940991442298
-F1 macro: 0.7930972243375974
-Precision micro: 0.7881321203180067
-Recall micro: 0.7784343884076742
-F1 micro: 0.7832532377616681
-Fully correct: 168
-
-Query evaluation
-
-Precision macro: 0.3578564718337943
-Recall macro: 0.3604568928579589
-F1 macro: 0.3538064587750891
-Precision micro: 0.5226153669987389
-Recall micro: 0.5906943635540006
-F1 micro: 0.5545733466496597
-Fully correct: 373
-

data/eval/lc-quad-requeried-linked-test-eval-append-1-large-10-epochs.txt

-String evaluation
-
-Precision macro: 0.7837609705826785
-Recall macro: 0.7773669243953512
-F1 macro: 0.7778599845931444
-Precision micro: 0.7833761498301152
-Recall micro: 0.765239213146604
-F1 micro: 0.7742014742014742
-Fully correct: 152
-
-Query evaluation
-
-Precision macro: 0.3293007083724157
-Recall macro: 0.3320987435926118
-F1 macro: 0.3252376722062768
-Precision micro: 0.16137158132839782
-Recall micro: 0.5483607989285475
-F1 micro: 0.24936117249462159
-Fully correct: 343
-

data/eval/lc-quad-requeried-linked-test-eval-append-1-large-3-epochs.txt

-String evaluation
-
-Precision macro: 0.8030609762521919
-Recall macro: 0.7913620401992508
-F1 macro: 0.7947304198110867
-Precision micro: 0.7912972085385879
-Recall micro: 0.7802153323079414
-F1 micro: 0.7857171972445279
-Fully correct: 162
-
-Query evaluation
-
-Precision macro: 0.34168246759219223
-Recall macro: 0.34538766263173165
-F1 macro: 0.33773159161379057
-Precision micro: 0.2601919788161299
-Recall micro: 0.34827395237216047
-F1 micro: 0.29785756006408937
-Fully correct: 358
-

data/eval/lc-quad-requeried-linked-test-eval-append-1-large-4-epochs.txt

-String evaluation
-
-Precision macro: 0.7982044656463279
-Recall macro: 0.7850570068270347
-F1 macro: 0.7890307366150056
-Precision micro: 0.787020260253665
-Recall micro: 0.7735772686796729
-F1 micro: 0.7802408654827515
-Fully correct: 152
-
-Query evaluation
-
-Precision macro: 0.3288445236328654
-Recall macro: 0.33148790322402794
-F1 macro: 0.3254230404437138
-Precision micro: 0.4538291509556613
-Recall micro: 0.6749951653451918
-F1 micro: 0.5427460736574067
-Fully correct: 345
-

data/eval/lc-quad-requeried-linked-test-eval-append-1-large-5-epochs.txt

-String evaluation
-
-Precision macro: 0.8016261444556045
-Recall macro: 0.7919916216427856
-F1 macro: 0.7948908668704054
-Precision micro: 0.7902195281782438
-Recall micro: 0.7809439002671416
-F1 micro: 0.7855543341069176
-Fully correct: 158
-
-Query evaluation
-
-Precision macro: 0.3616446080157203
-Recall macro: 0.36645505877246554
-F1 macro: 0.3577359194201524
-Precision micro: 0.2869282722065482
-Recall micro: 0.6553542138561249
-F1 micro: 0.39911524415737887
-Fully correct: 371
-

data/eval/lc-quad-requeried-linked-test-eval-append-1-large.txt

-String evaluation
-
-Precision macro: 0.7919314392182623
-Recall macro: 0.7705734044881352
-F1 macro: 0.7785799816305221
-Precision micro: 0.7792661619103087
-Recall micro: 0.7581963895410022
-F1 micro: 0.7685869030034466
-Fully correct: 153
-
-Query evaluation
-
-Precision macro: 0.3124380200374491
-Recall macro: 0.3151890072254368
-F1 macro: 0.30831872223612
-Precision micro: 0.44101080834221007
-Recall micro: 0.35187659419409484
-F1 micro: 0.39143359000602723
-Fully correct: 330
-

data/eval/lc-quad-requeried-linked-test-eval-append-1.txt

-String evaluation
-
-Precision macro: 0.7901266763669879
-Recall macro: 0.7647025813692488
-F1 macro: 0.7745498297625997
-Precision micro: 0.7777126590756865
-Recall micro: 0.7519630858900672
-F1 micro: 0.7646211466436186
-Fully correct: 157
-
-Query evaluation
-
-Precision macro: 0.3086409620867658
-Recall macro: 0.31637035241374506
-F1 macro: 0.30623680414174614
-Precision micro: 0.5037086609781914
-Recall micro: 0.42694398487355034
-F1 micro: 0.46216036427516005
-Fully correct: 332
-

data/eval/lc-quad-requeried-linked-test-eval-append-2.txt

-String evaluation
-
-Precision macro: 0.7909642157058177
-Recall macro: 0.7592750870528654
-F1 macro: 0.7718536879219824
-Precision micro: 0.7782839787395596
-Recall micro: 0.7467821581801991
-F1 micro: 0.7622077170949351
-Fully correct: 151
-
-Query evaluation
-
-Precision macro: 0.3145370033483092
-Recall macro: 0.3201454048853542
-F1 macro: 0.3105830987501682
-Precision micro: 0.25646965145318235
-Recall micro: 0.3746827411167493
-F1 micro: 0.30450570459926785
-Fully correct: 331
-

data/eval/lc-quad-requeried-linked-test-eval-replace.txt

-String evaluation
-
-Precision macro: 0.7754669127408153
-Recall macro: 0.7246206348919532
-F1 macro: 0.7451672335545623
-Precision micro: 0.7614376247938189
-Recall micro: 0.7100299522383227
-F1 micro: 0.7348357908847184
-Fully correct: 111
-
-Query evaluation
-
-Precision macro: 0.23699566590718038
-Recall macro: 0.23965419392426118
-F1 macro: 0.23442770179220285
-Precision micro: 0.09682505164727577
-Recall micro: 0.3603804127883375
-F1 micro: 0.15263969828680532
-Fully correct: 250
-

data/eval/lc-quad-requeried-linked-test-eval.txt

-String evaluation
-
-Precision macro: 0.7448140050207241
-Recall macro: 0.7219392125722884
-F1 macro: 0.730242778290854
-Precision micro: 0.7337440679377237
-Recall micro: 0.7134299360479236
-F1 micro: 0.723444426202594
-Fully correct: 49
-
-Query evaluation
-
-Precision macro: 0.17180050387133347
-Recall macro: 0.17505599501193153
-F1 macro: 0.16871915502993529
-Precision micro: 0.23121119451394576
-Recall micro: 0.24292877659315398
-F1 micro: 0.23692519521152552
-Fully correct: 181
-

data/eval/lcquad-test-replace-base-0.txt

+String evaluation
+
+Precision macro: 0.6909234788125834
+Recall macro: 0.6472900586227427
+F1 macro: 0.664167445545291
+Precision micro: 0.6844166014095536
+Recall micro: 0.6387721542115842
+F1 micro: 0.6608071070787261
+Fully correct: 5
+
+Query evaluation
+
+Precision macro: 0.07518992021805711
+Recall macro: 0.08257631631560637
+F1 macro: 0.07511667474744187
+Precision micro: 0.007689093188466296
+Recall micro: 0.0063117453347968835
+F1 micro: 0.006932670158840723
+Fully correct: 34
+

data/eval/lcquad-test-replace-base-1.txt

+String evaluation
+
+Precision macro: 0.6877610230236684
+Recall macro: 0.6428884399215128
+F1 macro: 0.6602873172350482
+Precision micro: 0.6799372425965876
+Recall micro: 0.633473414946099
+F1 micro: 0.6558834657586076
+Fully correct: 5
+
+Query evaluation
+
+Precision macro: 0.07834860388307141
+Recall macro: 0.0924869166338132
+F1 macro: 0.07827437428632678
+Precision micro: 0.03128129602356397
+Recall micro: 0.17134559535333702
+F1 micro: 0.05290425423313134
+Fully correct: 32
+

data/eval_query.py

@@ -38,6 +38,8 @@ def eval_q(corr, pred):
 
         if tp + fp == 0:
             precision = recall = f1 = 0
+        elif tp + fn == 0: 
+            precision = recall = f1 = 1
         else:
             precision = tp / (tp + fp)
             recall = tp / (tp + fn)