forked from fanyubupt/KERL
-
Notifications
You must be signed in to change notification settings - Fork 2
/
eval_metrics.py
100 lines (92 loc) · 2.76 KB
/
eval_metrics.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
import math
import numpy as np
from nltk.translate import bleu_score
from nltk.translate.bleu_score import SmoothingFunction
def bleu(hyps, refs):
"""
bleu
"""
bleu_4 = []
for hyp, ref in zip(hyps, refs):
try:
score = bleu_score.sentence_bleu(
[ref], hyp,
smoothing_function=SmoothingFunction().method7,
weights=[0.5, 0.5, 0, 0])
except:
score = 0
bleu_4.append(score)
bleu_4 = np.average(bleu_4)
return bleu_4
def bleu_each(hyps, refs):
"""
bleu
"""
bleu_4 = []
hyps=hyps.cpu().numpy()
refs=refs.cpu().numpy()
for hyp, ref in zip(hyps, refs):
try:
score = bleu_score.sentence_bleu(
[ref], hyp,
smoothing_function=SmoothingFunction().method7,
weights=[0.5, 0.5, 0, 0])
except:
score = 0
bleu_4.append(score)
return bleu_4
def precision_at_k(actual, predicted, topk,item_i):
sum_precision = 0.0
user = 0
num_users = len(predicted)
for i in range(num_users):
if actual[i][item_i]>0:
user +=1
act_set = actual[i][item_i]
pred_set = predicted[i]
if act_set in pred_set:
sum_precision += 1
else:
continue
#print(user)
return sum_precision / user
def ndcg_k(actual, predicted, topk,item_i):
k = min(topk, len(actual))
idcg = idcg_k(k)
res = 0
user = 0
for user_id in range(len(actual)):
if actual[user_id][item_i] > 0:
user +=1
dcg_k = sum([int(predicted[user_id][j] in [actual[user_id][item_i]]) / math.log(j+2, 2) for j in range(k)])
res += dcg_k
else:
continue
#print(user)
return res/user
def dcg_k(actual, predicted, topk):
k = min(topk, len(actual))
dcgs=[]
actual = actual.cpu().numpy()
predicted = predicted.cpu().numpy()
for user_id in range(len(actual)):
value = []
for i in predicted[user_id]:
try:
value += [topk -int(np.argwhere(actual[user_id]==i))]
#print(value)
except:
value += [0]
#dcg_k = sum([int(predicted[user_id][j] in set(actual[user_id])) / math.log(j+2, 2) for j in range(k)])
dcg_k = sum([value[j] / math.log(j+2, 2) for j in range(k)])
if dcg_k==0:
dcg_k=1e-5
dcgs.append(dcg_k)
return dcgs
# Calculates the ideal discounted cumulative gain at k
def idcg_k(k):
res = sum([1.0/math.log(i+2, 2) for i in range(k)])
if not res:
return 1.0
else:
return res