Skip to content
Snippets Groups Projects
Commit 9e461682 authored by Ludwig Forsberg's avatar Ludwig Forsberg
Browse files

Added requeried lc-quad dataset

parent 8e2574c6
Branches
Tags
No related merge requests found
import sys
import json
from threading import Thread
from tqdm import tqdm
import time
import requests
data = {}
faults = 0
success = 0
def get_entities(question):
text = question['question'][0]["string"]
r = requests.post('https://labs.tib.eu/falcon/api?mode=long', json = {'text':text})
try:
res = r.json()
except:
print(r)
return {}
return res
if __name__ == '__main__':
data_file = sys.argv[1]
with open(data_file, 'r', encoding='utf-8') as json_file:
data = json.load(json_file)
pbar = tqdm(total=len(data['questions']))
for i in range(len(data['questions'])):
if('entities' in data['questions'][i] and 'relations' in data['questions'][i]):
pbar.update(1)
continue
try:
res = get_entities(data['questions'][i])
if('entities' in res and 'relations' in res):
entities = res['entities']
relations = res['relations']
success += 1
#print(f"Thread {i} got {entities} and {relations}")
data['questions'][i]['entities'] = entities
data['questions'][i]['relations'] = relations
else:
faults += 1
except:
break
pbar.update(1)
pbar.close()
print("Total: ", len(data['questions']))
print("Success: ", success)
print("Faults: ", faults)
with open(data_file, 'w', encoding='utf-8') as output:
json.dump(data, output)
Source diff could not be displayed: it is too large. Options to address this: view the blob.
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment