Added requeried lc-quad dataset

EntityRelationLinker/data_linker_remote_sync.py

+import sys
+import json
+from threading import Thread
+from tqdm import tqdm
+import time
+import requests
+
+
+
+
+data = {}
+faults = 0
+success = 0
+
+def get_entities(question):
+    text = question['question'][0]["string"]
+
+    r = requests.post('https://labs.tib.eu/falcon/api?mode=long', json = {'text':text})
+    try:
+        res = r.json()
+    except:
+        print(r)
+        return {}
+    return res
+
+if __name__ == '__main__':
+    data_file = sys.argv[1]
+
+    with open(data_file, 'r', encoding='utf-8') as json_file:
+        data = json.load(json_file)
+
+    pbar = tqdm(total=len(data['questions']))
+
+    for i in range(len(data['questions'])):
+        if('entities' in data['questions'][i] and 'relations' in data['questions'][i]):
+            pbar.update(1)
+            continue
+        try:
+            res = get_entities(data['questions'][i])
+
+            if('entities' in res and 'relations' in res):
+                entities = res['entities']
+                relations = res['relations']
+                success += 1
+                #print(f"Thread {i} got {entities} and {relations}")
+                data['questions'][i]['entities'] = entities
+                data['questions'][i]['relations'] = relations
+            else:
+                faults += 1
+        except:
+            break
+        pbar.update(1)
+
+
+
+    pbar.close()
+
+    print("Total: ", len(data['questions']))
+    print("Success: ", success)
+    print("Faults: ", faults)
+
+    with open(data_file, 'w', encoding='utf-8') as output:
+        json.dump(data, output)
+
+