Added code to test analysis and new synth user testing

examples/other/synth_user_example.py

@@ -6,9 +6,14 @@
 
 
 import numpy as np
+import random
 import pickle
 import argparse
 import lop
+import matplotlib.pyplot as plt
+from tqdm import tqdm
+
+import pdb
 
 import sys
 # caution: path[0] is reserved for script path (or '' in REPL)
@@ -17,6 +22,94 @@
 from experiment_helper import get_synth_user, get_fake_func
 
 
+from sklearn_extra.cluster import KMedoids
+def downsample_kmed_rand(X, num_downselect, kmed_pct=0.5):
+    num_kmed = int(kmed_pct * num_downselect)
+    num_rand = num_downselect - num_kmed
+
+    rand_idxs = np.random.choice(X.shape[0], num_rand, replace=False)
+    non_rand_idxs = list(set(list(range(X.shape[0]))) - set(rand_idxs))
+
+    cent_pts = KMedoids(n_clusters=num_kmed).fit(X[non_rand_idxs]).cluster_centers_
+
+    return np.append(cent_pts, X[rand_idxs], axis=0)
+
+
+def test_trained_model(user_f, eval_r):
+    print(eval_r)
+    print(user_f)
+    #pdb.set_trace()
+
+    print('Rating')
+
+    down_r = downsample_kmed_rand(eval_r, 50)
+
+    num_pairs = int(down_r.shape[0] * (down_r.shape[0]-1) / 2)
+
+    rates = np.empty(num_pairs)
+    chooses = np.empty(num_pairs)
+    del_ys = np.empty(num_pairs)
+
+    cnt = 0
+    for i in tqdm(range(down_r.shape[0])):
+        for j in range(i+1, down_r.shape[0]):
+            pair = [i,j]
+
+            rate_avg, choose_avg, delta_y =  test_pair_error_model(user_f, eval_r, pair)
+            rates[cnt] = rate_avg
+            chooses[cnt] = choose_avg
+            del_ys[cnt] = delta_y
+
+            cnt += 1
+
+    n_bins = 200
+    plt.hist(rates,n_bins)
+    plt.title('Rate probability')
+
+    plt.figure()
+    plt.hist(chooses, n_bins)
+    plt.title('Choose probabilties')
+
+    plt.figure()
+    plt.hist(del_ys, n_bins)
+    plt.title('Delta ys')
+
+    plt.show()
+
+
+
+
+def test_pair_error_model(user_f, eval_r, pair):
+    N=1000
+    correct = 0
+
+    y = user_f.fake_f(eval_r[pair])
+    delta_y = y[0] - y[1]
+    for i in range(N):
+        r = user_f.rate(eval_r[pair])
+
+
+        #print(r)
+
+        if (r[0] > r[1]) == (y[0] > y[1]):
+            correct += 1
+
+    rate_avg = correct / N
+
+    correct = 0
+    for i in range(N):
+        r = user_f.choose(eval_r[pair])
+
+        #print(r)
+
+        if r == np.argmax(y):
+            correct += 1
+
+    choose_avg = correct / N
+
+    return rate_avg, choose_avg, delta_y
+
+
 
 def main():
     parser = argparse.ArgumentParser(description='bimodal example with different models and active learners')
@@ -49,8 +142,10 @@ def main():
 
     config = {'dim_rewards': dim_rewards}
 
+    np.random.seed(0)
+    random.seed(0)
 
-    for i in range(10):
+    for i in range(1):
         print('Starting run '+str(i)+' of tuning synth user')
         fake_f = get_fake_func(args.fake_func, config)
         user_f = get_synth_user(args.user, fake_f, config)
@@ -67,8 +162,9 @@ def main():
         print('user_f sigma: ' + str(user_f.sigma) +' beta: ' + str(user_f.beta))
         print('\n\n')
 
-
     print(user_f)
+    test_trained_model(user_f, eval_user_d)
+
 
 
 

experiments/active_learning_fake_func.sh

@@ -41,20 +41,20 @@ rbf_l=1.2
 
 for fake_func in logistic linear
 do
-    sel_type=rating
-    for selc in UCB RANDOM
-    do
-        for i_env in 0 1 2 3 4 5 6 7 8 9
-        do
-            stdbuf -oL python3 single_experiment.py --env $i_env --model $model --selector $selc --sel_type $sel_type --num_runs $number_runs --num_alts $num_alts --user $user --hyper $hyper_sel --def_pareto $def_pareto --fake_func $fake_func --kmedoid $kmedoid --p_synth_pair $p_synth_pair --p_synth_abs $p_synth_abs --sigma_pair $sigma_pair --sigma_abs $sigma_abs --v_abs $v --rbf_sigma $rbf_sigma --rbf_l $rbf_l  > results/console_output_${selc}_${model}_${i_env}_${hyper_sel}_${fake_func}.txt 2>&1 & 
-        done
-        wait
-    done
+    # sel_type=rating
+    # for selc in UCB RANDOM
+    # do
+    #     for i_env in 0 1 2 3 4 5 6 7 8 9
+    #     do
+    #         stdbuf -oL python3 single_experiment.py --env $i_env --model $model --selector $selc --sel_type $sel_type --num_runs $number_runs --num_alts $num_alts --user $user --hyper $hyper_sel --def_pareto $def_pareto --fake_func $fake_func --kmedoid $kmedoid --p_synth_pair $p_synth_pair --p_synth_abs $p_synth_abs --sigma_pair $sigma_pair --sigma_abs $sigma_abs --v_abs $v --rbf_sigma $rbf_sigma --rbf_l $rbf_l  > results/console_output_${selc}_${model}_${i_env}_${hyper_sel}_${fake_func}.txt 2>&1 & 
+    #     done
+    #     wait
+    # done
 
     sel_type=switch
 
     # SW_ACQ_RHO SW_ACQ_SPEAR MUTUAL_INFO ACQ_RHO RANDOM ACQ_SPEAR UCB SW_ACQ_LL ACQ_LL #MUTUAL_INFO UCB RANDOM ABS_ACQ_RHO SW_ACQ_LL ACQ_LL ABS_ACQ_LL SW_ACQ_SPEAR ACQ_SPEAR ABS_ACQ_SPEAR  # SW_ACQ_RHO ABS_ACQ_RHO
-    for selc in SW_UCB_RHO SW_UCB_SPEAR MUTUAL_INFO SW_FIXED_RHO SW_ACQ_RHO SW_ACQ_SPEAR SW_FIXED_SPEAR RANDOM
+    for selc in ACQ_RHO ACQ_SPEAR #SW_UCB_RHO SW_UCB_SPEAR MUTUAL_INFO SW_FIXED_RHO SW_ACQ_RHO SW_ACQ_SPEAR SW_FIXED_SPEAR RANDOM
     do
         for i_env in 0 1 2 3 4 5 6 7 8 9
         do