Skip to content
Snippets Groups Projects
Commit 13dc9368 authored by Albin's avatar Albin
Browse files

Test and validation

parent fc6da773
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Neural graph module/ngm.ipynb
+ 211
− 87
View file @ 13dc9368
%% Cell type:code id: tags:
``` python
import datasets
import torch
import torch.nn as nn
import torch.optim as optim
import pandas as pd
import numpy as np
from transformers import BertTokenizer, BertModel
from transformers import Seq2SeqTrainer, Seq2SeqTrainingArguments
from tqdm import tqdm
import json
```
%% Cell type:code id: tags:
``` python
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
```
%% Cell type:code id: tags:
``` python
# Use GPU if available
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Print cuda version
print(device)
```
%% Output
cuda
%% Cell type:code id: tags:
``` python
class NgmOne(nn.Module):
def __init__(self, device):
def __init__(self, device, relations):
super(NgmOne, self).__init__()
self.tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
self.bert = BertModel.from_pretrained("bert-base-uncased").to(device)
self.linear = nn.Linear(768, 1396).to(device)
self.linear = nn.Linear(768, len(relations)).to(device)
self.softmax = nn.Softmax(dim=1).to(device)
self.device = device
def forward(self, tokenized_seq, tokenized_mask):
tokenized_seq= tokenized_seq.to(self.device)
tokenized_mask = tokenized_mask.to(self.device)
with torch.no_grad():
x = self.bert.forward(tokenized_seq, attention_mask=tokenized_mask)
x = x[0][:,0,:].to(self.device)
x = self.linear(x)
x = self.softmax(x)
return x
```
%% Cell type:code id: tags:
``` python
prefixes = {
"http://dbpedia.org/resource/": "res:",
"http://dbpedia.org/ontology/": "dbo:",
"http://dbpedia.org/property/": "dbp:",
"http://www.w3.org/2000/01/rdf-schema#": "rdfs:",
"http://www.w3.org/1999/02/22-rdf-syntax-ns#": "rdf:",
"http://dbpedia.org/class/yago/": "yago:",
"http://www.wikidata.org/prop/direct/": "wdt:",
"http://www.wikidata.org/entity/": "wd:",
"http://www.wikidata.org/prop/": "p:",
"https://w3id.org/payswarm#": "ps:",
"http://www.wikidata.org/prop/qualifier/": "pq:",
"http://www.bigdata.com/rdf#": "bd:",
"http://wikiba.se/ontology#": "wikibase:",
"http://www.w3.org/2004/02/skos/core#": "skos:",
}
prefixes_end = ["org", "se", "com", "nu"]
ALL_HTTP_PREFIXES = True
ALL_HTTP_PREFIXES = False
def make_batch():
def make_batch(src, http_prefix=False):
"""Triplet is a list of [subject entity, relation, object entity], None if not present"""
pred = src
gold = src
# Load predicted data
# "../data/qald-9-train-linked.json"
pred = "../LC-QuAD/combined-requeried-linked-train.json"
#pred = "../LC-QuAD/combined-requeried-linked-train.json"
#Load gold data
# "../LC-QuAD/combined-requeried-linked-train.json"
gold = "../LC-QuAD/combined-requeried-linked-train.json"
#gold = "../LC-QuAD/combined-requeried-linked-train.json"
print("Beginning making batch")
with open(pred, "r") as p, open(gold, "r") as g:
pred = json.load(p)
gold = json.load(g)
inputs = []
correct_rels = []
inputs_max_len = 0
for d in tqdm(pred["questions"]):
question = d["question"][0]["string"]
query = d["query"]["sparql"]
#Take the first tripletin query
trip = query.split("WHERE {")[1]
trip = trip.split("}")[0]
trip = trip.split("FILTER ")[0]
trip = trip.replace("{", "").replace("}", "")
trip = trip.replace(".", "")
trip = trip.replace(";", "")
triplet = trip.split(" ")
#remove empty strings
triplet = [x for x in triplet if x != ""]
if len(triplet) % 3 == 0 and " ".join(triplet).find("rdf") == -1:
for i in range(len(triplet)//3):
triplet_i = triplet[i*3:i*3+3]
for t in triplet_i:
if not(t.find("?")):
triplet_i[triplet_i.index(t)] = None
elif ALL_HTTP_PREFIXES:
elif http_prefix:
n = t.replace("<", "").replace(">", "")
n_sub = n.split("/")[-1]
for i in prefixes_end:
if n.find(i) != -1:
n_ = "".join(n.split(i)[0]) + "." + i + "".join(n.split(i)[1:])
break
for i in prefixes:
if n_.find(i) != -1:
new = prefixes[i]
break
#n_other = "/".join(n_.split("/")[0:-1]) + "/"
#new = prefixes[n_other]
triplet_i[triplet_i.index(t)] = new + n_sub
elif t.find("http") != -1:
for i in prefixes_end:
if t.find(i) != -1:
n_ = "".join(t.split(i)[0]) + "." + i + "".join(t.split(i)[1:])
break
triplet_i[triplet_i.index(t)] = n_
#seq = "[CLS] " + question + " [SEP] "
if triplet_i[0] is not None and triplet_i[1] is not None:
#seq += "[SUB] [SEP] " + triplet[0]
# , padding=True, truncation=True)
tokenized_seq = tokenizer(question, "[SUB]", triplet_i[0], padding=True, truncation=True)
elif triplet_i[2] is not None and triplet_i[1] is not None:
#seq += "[OBJ] [SEP] " + triplet[2]
tokenized_seq = tokenizer(question, "[OBJ]", triplet_i[2], padding=True, truncation=True)
else:
continue
if inputs_max_len < len(tokenized_seq["input_ids"]):
inputs_max_len = len(tokenized_seq["input_ids"])
inputs.append(list(tokenized_seq.values())[0])
correct_rels.append(triplet_i[1])
inputs_padded = np.array([i + [0]*(inputs_max_len-len(i)) for i in inputs])
#correct_rels_padded = np.array([i + [0]*(correct_rels_max_len-len(i)) for i in correct_rels])
inputs_attention_mask = np.where(inputs_padded != 0, 1, 0)
#correct_rels_attention_mask = np.where(correct_rels_padded != 0, 1, 0)
print("Finished with batches")
return torch.LongTensor(inputs_padded), torch.LongTensor(inputs_attention_mask), correct_rels #torch.IntTensor(correct_rels_padded), torch.LongTensor(correct_rels_attention_mask)
```
%% Cell type:code id: tags:
``` python
from torch.utils.data import Dataset, DataLoader
class MyDataset(Dataset):
def __init__(self, inputs, attention_mask, correct_rels, relations):
self.inputs = inputs
self.attention_mask = attention_mask
self.correct_rels = correct_rels
self.relations = relations
def __len__(self):
return len(self.inputs)
def __getitem__(self, idx):
return self.inputs[idx], self.attention_mask[idx], self.relations.index(self.correct_rels[idx])
```
%% Cell type:code id: tags:
``` python
from torch.utils.data import random_split
#Prepare data
def open_json(file):
with open(file, "r") as f:
return json.load(f)
relations = open_json("../data/relations-query-qald-9-linked.json")
inputs, attention_mask, correct_rels = make_batch()
#relations = open_json("../data/relations-query-qald-9-linked.json")
relations = open_json("../data/relations-all-lc-quad-no-http.json")
# "../data/qald-9-train-linked.json"
#pred = "../LC-QuAD/combined-requeried-linked-train.json"
inputs, attention_mask, correct_rels = make_batch(src="../LC-QuAD/combined-requeried-linked-train.json", http_prefix = True) #train
# relations = open_json("../data/relations-lcquad-without-http-train-linked.json")
# train_set = MyDataset(*make_batch(), relations=relations)
dataset = MyDataset(*make_batch(), relations=relations)
split_data = random_split(dataset, [0.8, 0.2], generator=torch.Generator().manual_seed(42))
dataset = MyDataset(inputs, attention_mask, correct_rels, relations=relations)
train_size = int(0.8 * len(dataset))
valid_size = len(dataset) - train_size
train_dataloader = DataLoader(train_set, batch_size=1, shuffle=True)
train_data, valid_data = random_split(dataset, [train_size, valid_size], generator=torch.Generator().manual_seed(42))
train_dataloader = DataLoader(train_data, batch_size=1, shuffle=True)
#show first entry
train_features, train_mask, train_label = next(iter(train_dataloader))
print("features:", train_features, "mask:",train_mask,"label_index", train_label[0])
valid_dataloader = DataLoader(valid_data, batch_size=1, shuffle=True)
valid_features, valid_mask, valid_label = next(iter(valid_dataloader))
print("valid features:", valid_features, "valid mask:",valid_mask,"valid label_index", valid_label[0])
```
%% Output
Beginning making batch
100%|██████████| 408/408 [00:00<00:00, 616.32it/s]
100%|██████████| 2052/2052 [00:03<00:00, 546.37it/s]
Finished with batches
Beginning making batch
100%|██████████| 408/408 [00:00<00:00, 620.42it/s]
Finished with batches
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
c:\Users\Albin\Documents\TDDE19\codebase\Neural graph module\ngm.ipynb Cell 7 in <cell line: 14>()
<a href='vscode-notebook-cell:/c%3A/Users/Albin/Documents/TDDE19/codebase/Neural%20graph%20module/ngm.ipynb#X11sZmlsZQ%3D%3D?line=8'>9</a> inputs, attention_mask, correct_rels = make_batch()
<a href='vscode-notebook-cell:/c%3A/Users/Albin/Documents/TDDE19/codebase/Neural%20graph%20module/ngm.ipynb#X11sZmlsZQ%3D%3D?line=12'>13</a> dataset = MyDataset(*make_batch(), relations=relations)
---> <a href='vscode-notebook-cell:/c%3A/Users/Albin/Documents/TDDE19/codebase/Neural%20graph%20module/ngm.ipynb#X11sZmlsZQ%3D%3D?line=13'>14</a> split_data = random_split(dataset, [0.8, 0.2], generator=torch.Generator().manual_seed(42))
<a href='vscode-notebook-cell:/c%3A/Users/Albin/Documents/TDDE19/codebase/Neural%20graph%20module/ngm.ipynb#X11sZmlsZQ%3D%3D?line=15'>16</a> train_dataloader = DataLoader(train_set, batch_size=1, shuffle=True)
<a href='vscode-notebook-cell:/c%3A/Users/Albin/Documents/TDDE19/codebase/Neural%20graph%20module/ngm.ipynb#X11sZmlsZQ%3D%3D?line=17'>18</a> #show first entry
File b:\Programs\Miniconda\envs\tdde19\lib\site-packages\torch\utils\data\dataset.py:311, in random_split(dataset, lengths, generator)
309 # Cannot verify that dataset is Sized
310 if sum(lengths) != len(dataset): # type: ignore[arg-type]
--> 311 raise ValueError("Sum of input lengths does not equal the length of the input dataset!")
313 indices = randperm(sum(lengths), generator=generator).tolist()
314 return [Subset(dataset, indices[offset - length : offset]) for offset, length in zip(_accumulate(lengths), lengths)]
ValueError: Sum of input lengths does not equal the length of the input dataset!
features: tensor([[ 101, 2054, 2003, 1996, 13314, 1997, 16122, 10230, 5400, 2100,
12378, 2136, 1029, 102, 1031, 4942, 1033, 102, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0]]) mask: tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]) label_index tensor(378)
valid features: tensor([[ 101, 2054, 4752, 1997, 7992, 13843, 5112, 2267, 2003, 2036,
1996, 9353, 4215, 26432, 2278, 3037, 1997, 2703, 27668, 3077,
1029, 102, 1031, 4942, 1033, 102, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0]]) valid mask: tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]) valid label_index tensor(28)
%% Cell type:code id: tags:
``` python
# Initialize model
model = NgmOne(device)
model = NgmOne(device, relations)
```
%% Output
Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias']
- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
%% Cell type:code id: tags:
``` python
# Train with data loader.
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.0001)
optimizer = optim.Adam(model.parameters(), lr=0.001)
epoch = 500
epoch = 50
batch_size = 64
train_dataloader = DataLoader(train_set, batch_size=batch_size, shuffle=True)
valid_dataloader = DataLoader(valid_set, batch_size=batch_size, shuffle=True)
train_dataloader = DataLoader(train_data, batch_size=batch_size, shuffle=True)
valid_dataloader = DataLoader(valid_data, batch_size=batch_size, shuffle=True)
for e in range(epoch):
train_loss_epoch = 0
valid_loss_epoch = 0
for i_batch, sample_batched in enumerate(train_dataloader):
for i_train, sample_batched_train in enumerate(train_dataloader):
optimizer.zero_grad()
train = sample_batched[0]
train_mask = sample_batched[1]
label_index = sample_batched[2].to(device)
train = sample_batched_train[0]
train_mask = sample_batched_train[1]
label_index = sample_batched_train[2].to(device)
# Forward pass
output = model(train, train_mask)
loss = criterion(output, label_index)
# backward and optimize
loss.backward()
optimizer.step()
train_loss_epoch = train_loss_epoch + loss.item()
for i_batch, sample_batched in enumerate(valid_dataloader):
valid = sample_batched[0]
valid_mask = sample_batched[1]
label_index = sample_batched[2].to(device)
for i_valid, sample_batched_valid in enumerate(valid_dataloader):
valid = sample_batched_valid[0]
valid_mask = sample_batched_valid[1]
label_index = sample_batched_valid[2].to(device)
# Forward pass
output = model(train, train_mask)
loss = criterion(output, label_index)
with torch.no_grad():
output = model(valid, valid_mask)
loss = criterion(output, label_index)
valid_loss_epoch = valid_loss_epoch + loss.item()
print(e+1, "Train", valid_loss_epoch/len(sample_batched), ", Valid ", valid_loss_epoch/len(sample_batched))
print(e+1, "Train", train_loss_epoch/i_train, ", Valid ", valid_loss_epoch/i_valid)
```
%% Output
1 Train 6.453075773575726 , Valid 7.615683317184448
2 Train 6.444019513971665 , Valid 7.6052809953689575
3 Train 6.43224432889153 , Valid 7.602278828620911
4 Train 6.415224019218893 , Valid 7.598743319511414
5 Train 6.391957339118509 , Valid 7.584347605705261
6 Train 6.368611419902129 , Valid 7.573382139205933
7 Train 6.339372747084674 , Valid 7.557032465934753
8 Train 6.318648646859562 , Valid 7.539492607116699
9 Train 6.297140289755428 , Valid 7.523481965065002
10 Train 6.281140131108901 , Valid 7.519673824310303
11 Train 6.265462370479808 , Valid 7.5268988609313965
12 Train 6.24550900739782 , Valid 7.504716753959656
13 Train 6.234297051149256 , Valid 7.505620360374451
14 Train 6.220176332137164 , Valid 7.500500917434692
15 Train 6.204408701728372 , Valid 7.493185758590698
16 Train 6.194800797630759 , Valid 7.488588452339172
17 Train 6.183391935685101 , Valid 7.466232180595398
18 Train 6.173011443194221 , Valid 7.457393527030945
19 Train 6.168144422418931 , Valid 7.45364773273468
20 Train 6.157315562753117 , Valid 7.452357649803162
21 Train 6.140512690824621 , Valid 7.456966400146484
22 Train 6.129443336935604 , Valid 7.464043855667114
23 Train 6.12704220940085 , Valid 7.443313360214233
24 Train 6.117525269003475 , Valid 7.4395164251327515
25 Train 6.116069064420812 , Valid 7.456610083580017
26 Train 6.108902790967156 , Valid 7.430782794952393
27 Train 6.108471814323874 , Valid 7.44858992099762
28 Train 6.100216697244083 , Valid 7.457515120506287
29 Train 6.099361447726979 , Valid 7.438013672828674
30 Train 6.092377662658691 , Valid 7.448408484458923
31 Train 6.088698302998262 , Valid 7.442046403884888
32 Train 6.083712998558493 , Valid 7.420018911361694
33 Train 6.081799563239603 , Valid 7.426819205284119
34 Train 6.070652428795309 , Valid 7.426627039909363
35 Train 6.069005825940301 , Valid 7.425306558609009
36 Train 6.059389002182904 , Valid 7.415045142173767
37 Train 6.0618573918062095 , Valid 7.418038249015808
38 Train 6.055309716392966 , Valid 7.434216380119324
39 Train 6.049336994395537 , Valid 7.43299674987793
40 Train 6.048270281623392 , Valid 7.410753607749939
41 Train 6.043416135451373 , Valid 7.410261392593384
42 Train 6.042512613184312 , Valid 7.4074866771698
43 Train 6.032251245835248 , Valid 7.390785813331604
44 Train 6.025217645308551 , Valid 7.370600938796997
45 Train 6.027591985814712 , Valid 7.377846837043762
46 Train 6.0253391826854035 , Valid 7.39339554309845
47 Train 6.019261051626766 , Valid 7.407409071922302
48 Train 6.014283713172464 , Valid 7.393349766731262
49 Train 6.010964337517233 , Valid 7.37293815612793
50 Train 6.010590104495778 , Valid 7.4031054973602295
%% Cell type:code id: tags:
``` python
# Predict
train, train_mask, corr_rels = make_batch()
train, train_mask, corr_rels = make_batch(src="../LC-QuAD/combined-requeried-linked-train.json", http_prefix = True)
test, test_mask, corr_rels_test = make_batch(src="../LC-QuAD/combined-requeried-linked-test.json", http_prefix = True)
test_data = MyDataset(test, test_mask, corr_rels_test, relations=relations)
test_dataloader = DataLoader(test_data, batch_size=len(test_data), shuffle=True)
test_batch, test_mask_batch, corr_rels_test_batch = next(iter(test_dataloader))
corr_rels_test_batch = corr_rels_test_batch.to(device)
with torch.no_grad():
output = model(train, train_mask)
output_train = model(train, train_mask)
output_test = model(test_batch, test_mask_batch)
loss = criterion(output_test, corr_rels_test_batch)
print("test loss", loss.item())
output_train = output_train.detach().cpu().numpy()
output_test = output_test.detach().cpu().numpy()
output = output.detach().cpu().numpy()
prediction = [relations[np.argmax(pred).item()]for pred in output]
probability = [pred[np.argmax(pred)] for pred in output]
correct_pred = [corr_rels[i] for i in range(len(output))]
prediction_train = [relations[np.argmax(pred).item()]for pred in output_train]
probability_train = [pred[np.argmax(pred)] for pred in output_train]
correct_pred_train = [corr_rels[i] for i in range(len(output_train))]
print("lowest confidence", min(probability))
prediction_test = [relations[np.argmax(pred).item()]for pred in output_test]
probability_test = [pred[np.argmax(pred)] for pred in output_test]
correct_pred_test = [corr_rels[i] for i in range(len(output_test))]
print("lowest confidence train", min(probability_train))
print("lowest confidence test", min(probability_test))
def accuracy_score(y_true, y_pred):
corr_preds=0
wrong_preds=0
for pred, correct in zip(y_pred, y_true):
if pred == correct:
corr_preds += 1
else:
wrong_preds += 1
return corr_preds/(corr_preds+wrong_preds)
print("Accuracy:", accuracy_score(correct_pred, prediction))
print("Accuracy train:", accuracy_score(correct_pred_train, prediction_train))
print("Accuracy test:", accuracy_score(correct_pred_test, prediction_test))
```
%% Output
Beginning making batch
100%|██████████| 2052/2052 [00:03<00:00, 577.85it/s]
Finished with batches
Beginning making batch
100%|██████████| 1161/1161 [00:02<00:00, 566.88it/s]
Finished with batches
test loss 6.031976699829102
lowest confidence train 0.08024344
lowest confidence test 0.08576707
Accuracy train: 0.3939828080229226
Accuracy test: 0.006361323155216285
%% Cell type:code id: tags:
``` python
```
......
data/relations-all-lc-quad-no-http.json 0 → 100644
+ 446
− 0
View file @ 13dc9368
[
"dbo:layout",
"dbp:owners",
"dbo:literaryGenre",
"dbp:teamName",
"dbp:style",
"dbo:genre",
"dbo:honours",
"dbo:child",
"dbo:writer",
"dbo:currency",
"dbp:president",
"dbp:editor",
"dbp:governor",
"dbo:placeOfBurial",
"dbp:governingBody",
"dbp:placeofburial",
"dbp:race",
"dbp:cityServed",
"dbo:subsidiary",
"dbp:constituency",
"dbp:league",
"dbo:ideology",
"dbp:district",
"dbp:deathPlace",
"dbp:headquarters",
"dbp:sisterNames",
"dbp:writers",
"dbp:veneratedIn",
"dbp:fields",
"dbp:branch",
"dbo:origin",
"dbp:school",
"dbp:firstAired",
"dbo:movement",
"dbp:prizes",
"dbo:river",
"dbo:battle",
"dbo:associatedBand",
"dbp:place",
"dbp:state",
"dbp:animator",
"dbp:artist",
"dbp:children",
"dbp:club",
"dbo:nearestCity",
"dbo:club",
"dbo:designer",
"dbp:manager",
"dbo:city",
"dbp:nationalOrigin",
"dbo:governmentType",
"dbp:junction",
"dbo:predecessor",
"dbp:designer",
"dbo:party",
"dbo:school",
"dbp:field",
"dbp:placeOfBirth",
"dbo:type",
"dbo:locationCity",
"dbo:kingdom",
"dbo:academicDiscipline",
"dbp:chancellor",
"dbo:doctoralAdvisor",
"dbp:poleDriver",
"dbp:locationTown",
"dbo:silverMedalist",
"dbp:chairman",
"dbp:carries",
"dbo:languageFamily",
"dbo:mayor",
"dbp:hometown",
"dbp:almaMater",
"dbp:beatifiedBy",
"dbo:riverMouth",
"dbo:managerClub",
"dbo:mouthMountain",
"dbo:sport",
"dbo:chairman",
"dbp:firstDriver",
"dbp:awards",
"dbo:mountainRange",
"dbp:leaderName",
"dbp:lieutenant",
"dbp:employer",
"dbo:distributor",
"dbo:militaryUnit",
"dbp:type",
"dbo:series",
"dbp:nearestCity",
"dbp:leaderTitle",
"dbo:lyrics",
"dbo:firstDriver",
"dbo:homeStadium",
"dbo:publisher",
"dbo:narrator",
"dbo:builder",
"dbp:highschool",
"dbp:meaning",
"dbp:presenter",
"dbp:garrison",
"dbp:director",
"dbp:birthDate",
"dbp:placeOfBurial",
"dbp:destinations",
"dbo:cpu",
"dbo:inflow",
"dbp:officialName",
"dbo:automobilePlatform",
"dbo:manufacturer",
"dbp:cinematography",
"dbp:lyrics",
"dbp:architecture",
"dbp:subject",
"dbp:licensee",
"dbp:playedFor",
"dbp:partner",
"dbo:parentOrganisation",
"dbp:residence",
"dbo:ingredient",
"dbo:locatedInArea",
"dbp:managerclubs",
"dbo:bandMember",
"dbp:nationality",
"dbp:team",
"dbp:pastMembers",
"dbo:bronzeMedalist",
"dbp:firstTeam",
"dbo:successor",
"dbo:illustrator",
"dbo:related",
"dbo:broadcastArea",
"dbo:militaryBranch",
"dbo:lieutenant",
"dbp:agencyName",
"dbo:institution",
"dbo:relation",
"dbp:architecturalStyle",
"dbp:affiliation",
"dbp:locationCountry",
"dbp:athletics",
"dbp:debutteam",
"dbp:developer",
"dbp:restingplace",
"dbp:religion",
"dbo:nationality",
"dbp:domain",
"dbp:cities",
"dbo:highschool",
"dbo:computingPlatform",
"dbp:broadcastArea",
"dbo:affiliation",
"dbp:hostCity",
"dbo:tenant",
"dbo:owner",
"dbp:narrated",
"dbo:artist",
"dbp:deathCause",
"dbo:nonFictionSubject",
"dbp:international",
"dbo:award",
"dbo:residence",
"dbp:keyPeople",
"dbo:order",
"dbo:headquarter",
"dbo:species",
"dbo:distributingLabel",
"dbp:borough",
"dbo:country",
"dbo:foundationPlace",
"dbo:starring",
"dbo:relative",
"dbo:creator",
"dbo:network",
"dbo:spouse",
"dbp:languages",
"dbo:operatingSystem",
"dbp:draftTeam",
"dbp:genre",
"dbo:ground",
"dbp:design",
"dbo:formerTeam",
"dbp:spouse",
"dbo:parent",
"dbp:role",
"dbo:editor",
"dbp:address",
"dbo:developer",
"dbo:instrument",
"dbo:foundedBy",
"dbo:recordedIn",
"dbp:membership",
"dbp:venue",
"dbo:programmingLanguage",
"dbp:rank",
"dbo:producer",
"dbp:editing",
"dbp:successor",
"dbp:guests",
"dbo:owningCompany",
"dbo:timeZone",
"dbo:maintainedBy",
"dbo:binomialAuthority",
"dbp:college",
"dbp:allegiance",
"dbp:notableInstruments",
"dbo:author",
"dbp:discipline",
"dbo:gender",
"dbo:garrison",
"dbp:starring",
"dbp:crosses",
"dbp:foundation",
"dbp:license",
"dbo:parentCompany",
"dbp:highSchool",
"dbp:largestCity",
"dbp:music",
"dbp:themeMusicComposer",
"dbp:university",
"dbp:affiliations",
"dbp:knownFor",
"dbp:currentMembers",
"dbp:os",
"dbo:architect",
"dbp:screenplay",
"dbp:title",
"dbo:campus",
"dbo:subsequentWork",
"dbp:deputy",
"dbo:university",
"dbo:targetAirport",
"dbp:arena",
"dbo:associatedMusicalArtist",
"dbo:breeder",
"dbo:religion",
"dbp:sisterStations",
"dbo:birthPlace",
"dbo:trainer",
"dbo:formerPartner",
"dbp:order",
"dbo:ceremonialCounty",
"dbo:season",
"dbp:programmingLanguage",
"dbo:hubAirport",
"dbo:notableWork",
"dbo:routeEnd",
"dbo:doctoralStudent",
"dbp:champion",
"dbp:deathDate",
"dbp:owner",
"dbp:predecessor",
"dbo:executiveProducer",
"dbp:battles",
"dbo:formerBandMember",
"dbo:manager",
"dbp:executiveProducer",
"dbp:doctoralAdvisor",
"dbo:editing",
"dbo:firstAscentPerson",
"dbo:territory",
"dbo:athletics",
"dbp:producer",
"dbo:album",
"dbp:birthplace",
"dbp:purpose",
"dbp:hubs",
"dbp:neighboringMunicipalities",
"dbo:discoverer",
"dbo:division",
"dbo:employer",
"dbp:name",
"dbo:wineRegion",
"dbp:ground",
"dbp:currency",
"dbo:otherParty",
"dbo:founder",
"dbo:voice",
"dbp:schooltype",
"dbo:family",
"dbp:buildingType",
"dbo:league",
"dbo:jurisdiction",
"dbo:monarch",
"dbp:origin",
"dbo:president",
"dbp:appointer",
"dbp:distributor",
"dbp:position",
"dbo:keyPerson",
"dbo:poleDriver",
"dbo:architecturalStyle",
"dbp:locationCity",
"dbp:notableworks",
"dbo:servingRailwayLine",
"dbp:author",
"dbp:stadium",
"dbp:products",
"dbp:associatedActs",
"dbp:creator",
"dbp:office",
"dbp:citizenship",
"dbo:colour",
"dbo:stadium",
"dbp:writer",
"dbo:region",
"dbo:incumbent",
"dbp:canonizedBy",
"dbp:party",
"dbo:profession",
"dbp:services",
"dbo:officialLanguage",
"dbo:leader",
"dbo:team",
"dbo:previousWork",
"dbp:youthclubs",
"dbp:workInstitutions",
"dbo:almaMater",
"dbo:capital",
"dbo:field",
"dbo:college",
"dbo:militaryRank",
"dbp:manufacturer",
"dbo:phylum",
"dbo:primeMinister",
"dbp:founded",
"dbp:region",
"dbp:homeStadium",
"dbp:birthName",
"dbp:operatingSystem",
"dbp:publisher",
"dbp:parent",
"dbp:commandStructure",
"dbo:citizenship",
"dbp:label",
"dbp:related",
"dbo:commander",
"dbo:county",
"dbo:training",
"dbp:location",
"dbp:houses",
"dbo:routeStart",
"dbp:city",
"dbo:cinematography",
"dbo:ethnicity",
"dbp:leader",
"dbp:religiousAffiliation",
"dbp:assembly",
"dbp:currentclub",
"dbp:area",
"dbo:language",
"dbo:board",
"dbp:mainIngredient",
"dbp:mainInterests",
"dbo:majorShrine",
"dbp:outflow",
"dbp:mother",
"dbp:training",
"dbo:director",
"dbp:line",
"dbp:architect",
"dbo:academicAdvisor",
"dbp:birthPlace",
"dbp:magazine",
"dbp:majorShrine",
"dbo:portrayer",
"dbp:jurisdiction",
"dbp:format",
"dbo:anthem",
"dbp:album",
"dbp:education",
"dbo:location",
"dbp:hairColor",
"dbo:stylisticOrigin",
"dbo:musicBy",
"dbp:inflow",
"dbo:neighboringMunicipality",
"dbo:countySeat",
"dbp:mascot",
"dbp:creators",
"dbo:regionServed",
"dbp:founder",
"dbp:commander",
"dbo:deathPlace",
"dbo:debutTeam",
"dbp:combatant",
"dbo:sisterStation",
"dbp:coverArtist",
"dbo:significantBuilding",
"dbp:occupation",
"dbo:commandStructure",
"dbo:programmeFormat",
"dbp:flagbearer",
"dbp:engine",
"dbp:company",
"dbo:knownFor",
"dbo:deathCause",
"dbo:product",
"dbp:pastteams",
"dbo:opponent",
"dbp:titles",
"dbp:doctoralStudents",
"dbo:service",
"dbo:veneratedIn",
"dbp:recorded",
"dbp:tenants",
"dbo:partner",
"dbp:province",
"dbo:license",
"dbp:material",
"dbp:nationalteam",
"dbo:stateOfOrigin",
"dbp:operator",
"dbo:race",
"dbo:operator",
"dbp:characters",
"dbo:outflow",
"dbp:language",
"dbp:gender",
"dbo:occupation",
"dbo:composer",
"dbp:mission",
"dbo:restingPlace",
"dbp:primeminister",
"dbp:nickname",
"dbo:federalState",
"dbo:assembly",
"dbp:placeOfDeath",
"dbp:notableCommanders",
"dbp:relatives",
"dbo:hometown",
"dbo:largestCity",
"dbo:authority",
"dbo:destination",
"dbp:archipelago",
"dbo:basedOn",
"dbp:trainer",
"dbp:country",
"dbo:education",
"dbo:denomination",
"dbp:coach",
"dbo:presenter",
"dbo:coach",
"dbo:launchSite"
]
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment