-
Notifications
You must be signed in to change notification settings - Fork 0
/
eval.sh
executable file
·66 lines (56 loc) · 1.62 KB
/
eval.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
#!/bin/bash
set -eu
# Setting variables
# Directory where fine-tuned models are stored: fine-tuned_models
MODEL_ROOT_DIR=$1
# Name of the model: qald9plus-finetune_mt5-base_lc-ent
MODEL_NAME=$2
# Path to the qald test file
TEST_FILE=$3
# Root directory where a new directory with model_name will be created to store predictions and results
OUTPUT_DIR=$4
# Comma separated values: en,de,es
LANGS=$5
# linguistic context : true/false
LC=$6
# entity knowledge : true/false
EK=$7
# Perform Gerbil evaluation: true/false
GE=$8
# Knowledge Graph: DBpedia / Wikidata
KNOWLEDGE_GRAPH=$9
#KNOWLEDGE_GRAPH="DBpedia"
#KNOWLEDGE_GRAPH="Wikidata"
question_padding_length=128
entity_padding_length=64
if [ "$LC" = true ]; then
linguistic_context="--linguistic_context"
else
linguistic_context="--no-linguistic_context"
fi
if [ "$EK" = true ]; then
entity_knowledge="--entity_knowledge"
else
entity_knowledge="--no-entity_knowledge"
fi
EXTRA_PARAMS=""
if [ "$GE" = "false" ]; then
echo "No Gerbil evaluation will be performed."
else
echo "Gerbil evaluation will be performed."
EXTRA_PARAMS+="--gerbil_eval "
fi
# Add --use_gold_ents to use gold entities
# Add --translate_target_lang en to translate all questions to en
python code/pred_build_eval_qald.py \
--model "${MODEL_ROOT_DIR}/${MODEL_NAME}" \
-t $TEST_FILE \
--knowledge_graph ${KNOWLEDGE_GRAPH} \
-o "${OUTPUT_DIR}/${MODEL_NAME}" \
-l $LANGS \
${linguistic_context} \
${entity_knowledge} \
--question_padding_length ${question_padding_length} \
--entity_padding_length ${entity_padding_length} \
$EXTRA_PARAMS \
2>&1