linear ensembling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "\n",
    "def objective_function(x, test, prob):\n",
    "    a, b, c = x\n",
    "    return np.sum((a * test[0] + b * test[1] + c * test[2] - prob) ** 2)\n",
    "\n",
    "def constraint(x):\n",
    "    return np.sum(x) - 1\n",
    "\n",
    "# create a random np vector of probabilities\n",
    "prob = np.random.random(500)\n",
    "\n",
    "test = np.random.random((3,500))\n",
    "\n",
    "x0 = np.array([1/3, 1/3, 1/3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "objective_function(x0, test, prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.optimize import minimize\n",
    "\n",
    "con = {'type': 'eq', 'fun': constraint}\n",
    "bounds = [(0, 1), (0, 1), (0, 1)]  # a, b, c >= 0 and <= 1\n",
    "\n",
    "result = minimize(objective_function, x0, args=(test, prob), bounds=bounds, constraints=con)\n",
    "\n",
    "print(\"Optimal values:\")\n",
    "print(\"a =\", result.x[0])\n",
    "print(\"b =\", result.x[1])\n",
    "print(\"c =\", result.x[2])\n",
    "print(\"Objective function value:\", result.fun)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hamming Distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "path = '/data1/malto/shroom'\n",
    "\n",
    "results1 = pd.read_csv(path + '/baseline_val.csv')\n",
    "results2 = pd.read_csv(path + '/crlft_val.csv')\n",
    "results3 = pd.read_csv(path + '/sequential_nogpt_aware.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r1 = 1 - results1['baseline'].to_numpy()\n",
    "r2 = results2['deberta-xlarge-mnli_0.01_frozen_22_24_1_0.772_63_sequential'].to_numpy()\n",
    "r3 = 1 - results3['sequential_nogpt_deberta_xlarge'].to_numpy()\n",
    "\n",
    "test = np.stack((r1, r2, r3), axis=0).mean(axis=0)\n",
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = pd.read_json(path + '/val.model-aware.json', lines=True)['p(Hallucination)'].to_numpy()\n",
    "labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = np.where(test > 0.5, 1, 0)\n",
    "labels = np.where(labels > 0.5, 1, 0)\n",
    "average_wrongs = np.logical_xor(predictions, labels).sum()\n",
    "average_wrongs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels1 = labels[:len(labels)//2]\n",
    "labels2 = labels[len(labels)//2:]\n",
    "\n",
    "r1a = r1[:len(r1)//2]\n",
    "r1b = r1[len(r1)//2:]\n",
    "\n",
    "r2a = r2[:len(r2)//2]\n",
    "r2b = r2[len(r2)//2:]\n",
    "\n",
    "r3a = r3[:len(r3)//2]\n",
    "r3b = r3[len(r3)//2:]\n",
    "\n",
    "test1 = np.stack((r1a, r2a, r3a), axis=0)\n",
    "test2 = np.stack((r1b, r2b, r3b), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "\n",
    "def objective_function(x, test, prob):\n",
    "    a, b, c = x\n",
    "    return np.sum((a * test[0] + b * test[1] + c * test[2] - prob) ** 2)\n",
    "\n",
    "def constraint(x):\n",
    "    return np.sum(x) - 1\n",
    "\n",
    "# create a random np vector of probabilities\n",
    "prob = labels1 \n",
    "\n",
    "test = np.stack((r1, r2, r3), axis=0)\n",
    "\n",
    "x0 = np.array([1/3, 1/3, 1/3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "objective_function(x0, test1, labels1), objective_function(x0, test2, labels2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.optimize import minimize\n",
    "\n",
    "con = {'type': 'eq', 'fun': constraint}\n",
    "bounds = [(0, 1), (0, 1), (0, 1)]  # a, b, c >= 0 and <= 1\n",
    "\n",
    "result = minimize(objective_function, x0, args=(test, labels), bounds=bounds, constraints=con)\n",
    "\n",
    "print(\"Optimal values:\")\n",
    "print(\"a =\", result.x[0])\n",
    "print(\"b =\", result.x[1])\n",
    "print(\"c =\", result.x[2])\n",
    "print(\"Objective function value:\", result.fun)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "objective_function(result.x, test2, labels2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "path = '/data1/malto/shroom'\n",
    "\n",
    "results1 = pd.read_csv(path + '/baseline.csv')\n",
    "results2 = pd.read_csv(path + '/crlft.csv')\n",
    "results3 = pd.read_csv(path + '/sequential_nogpt.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r1 = 1 - results1['baseline'].to_numpy()\n",
    "r2 = results2['crlft_alldata_deberta_xlarge_forzen_22_sequential'].to_numpy()\n",
    "r3 = 1 - results3['sequential_nogpt_deberta_xlarge'].to_numpy()\n",
    "\n",
    "test = np.stack((r1, r2, r3), axis=0).mean(axis=0)\n",
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r1_new = result.x[0] * r1\n",
    "r2_new = result.x[1] * r2\n",
    "r3_new = result.x[2] * r3\n",
    "\n",
    "res = r1_new + r2_new + r3_new\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "panda = pd.DataFrame(res, columns=['p(Hallucination)'])\n",
    "panda.to_csv(path + '/lp_ensemble.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = pd.read_csv(path + '/lp_ensemble.csv')['p(Hallucination)'].to_numpy()\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_sequential = np.where(r3 > 0.5, 1, 0)\n",
    "labels_ensemble = np.where(res > 0.5, 1, 0)\n",
    "differences = np.logical_xor(labels_sequential, labels_ensemble).sum()\n",
    "differences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Paper evaluation of ensembles "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read files in paper_results directory \n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy.optimize import minimize\n",
    "\n",
    "\n",
    "n = 4\n",
    "use_mnli = True\n",
    "result_path = 'paper_results_mnli/' if use_mnli else 'paper_results/'\n",
    "path = '/data1/malto/shroom'\n",
    "\n",
    "\n",
    "baseline_test = pd.read_csv(result_path + '/baseline_test' + str(n) + '.csv')['baseline'].to_numpy()\n",
    "baseline_val = pd.read_csv(result_path + '/baseline_val' + str(n) + '.csv')['baseline'].to_numpy()\n",
    "crlft_test = pd.read_csv(result_path + '/crlft_test' + str(n) + '.csv')['crlft'].to_numpy()\n",
    "crlft_val = pd.read_csv(result_path + '/crlft_val' + str(n) + '.csv')['crlft'].to_numpy()\n",
    "sequential_test = pd.read_csv(result_path + '/sequential_test' + str(n) + '.csv')['sequential'].to_numpy() \n",
    "sequential_val = pd.read_csv(result_path + '/sequential_val' + str(n) + '.csv')['sequential'].to_numpy()\n",
    "\n",
    "labels_val = pd.read_json(path + '/val.model-aware.json', lines=True)['p(Hallucination)'].to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# baseline: da ribaltare solo test \n",
    "# crlft: da ribaltare entrambi "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6207584830339321\n"
     ]
    }
   ],
   "source": [
    "predictions = np.where(sequential_val > 0.5, 1, 0)\n",
    "labels = np.where(labels_val > 0.5, 1, 0)\n",
    "print(np.sum(predictions == labels)/len(labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read test.model-agnostic.json \n",
    "from datasets import load_dataset, concatenate_datasets, DatasetDict\n",
    "\n",
    "test = load_dataset(\"json\", data_files=[\"test.model-agnostic.json\"])\n",
    "ground_truth_test = test['train']['label']\n",
    "ground_truth_test = [1 if x == 'Hallucination' else 0 for x in ground_truth_test]\n",
    "ground_truth_test = np.array(ground_truth_test)\n",
    "ground_truth_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1203\n"
     ]
    }
   ],
   "source": [
    "predictions = np.where(sequential_test > 0.5, 1, 0)\n",
    "labels = np.where(ground_truth_test > 0.5, 1, 0)\n",
    "print(np.sum(predictions == labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1500,), (1500,))"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions.shape, labels.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "ground_truth = labels_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# optimization problem between the three models\n",
    "import numpy as np\n",
    "\n",
    "def objective_function(x, res, ground_truth):\n",
    "    a, b, c = x\n",
    "    return np.sum((a * res[0] + b * res[1] + c * res[2] - ground_truth) ** 2)\n",
    "\n",
    "def constraint(x):\n",
    "    return np.sum(x) - 1\n",
    "\n",
    "con = {'type': 'eq', 'fun': constraint}\n",
    "bounds = [(0, 1), (0, 1), (0, 1)]  # a, b, c >= 0 and <= 1\n",
    "x0 = np.array([1/3, 1/3, 1/3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "42.76582729850774"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "objective_function(x0, res, ground_truth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = np.stack((baseline_val, sequential_val, crlft_val), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimal values:\n",
      "a = 0.2609552684620062\n",
      "b = 0.1315891612722595\n",
      "c = 0.6074555702657343\n",
      "Objective function value: 38.02350259054141\n"
     ]
    }
   ],
   "source": [
    "result = minimize(objective_function, x0, args=(res, ground_truth), bounds=bounds, constraints=con)\n",
    "\n",
    "print(\"Optimal values:\")\n",
    "print(\"a =\", result.x[0])\n",
    "print(\"b =\", result.x[1])\n",
    "print(\"c =\", result.x[2])\n",
    "print(\"Objective function value:\", result.fun)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "r1_new = result.x[0] * baseline_val\n",
    "r2_new = result.x[1] * sequential_val\n",
    "r3_new = result.x[2] * crlft_val\n",
    "\n",
    "res = r1_new + r2_new + r3_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.766\n"
     ]
    }
   ],
   "source": [
    "predictions = np.where(res > 0.5, 1, 0)\n",
    "labels = np.where(ground_truth > 0.5, 1, 0)\n",
    "print(np.sum(predictions == labels)/len(labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r1_new = result.x[0] * baseline_test\n",
    "r2_new = result.x[1] * sequential_test\n",
    "r3_new = result.x[2] * crlft_test\n",
    "\n",
    "res = r1_new + r2_new + r3_new\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "panda = pd.DataFrame(res, columns=['p(Hallucination)'])\n",
    "panda.to_csv(result_path + f'/ensemble_full{n}.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for 2-2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# attach together crlft_val and baseline_val in a (2, 500) array\n",
    "res1 = np.stack((crlft_val, baseline_val), axis=0)\n",
    "res1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# optimization problem between the three models\n",
    "import numpy as np\n",
    "\n",
    "def objective_function(x, res, ground_truth):\n",
    "    a, b = x\n",
    "    return np.sum((a * res[0] + b * res[1] - ground_truth) ** 2)\n",
    "\n",
    "def constraint(x):\n",
    "    return np.sum(x) - 1\n",
    "\n",
    "con = {'type': 'eq', 'fun': constraint}\n",
    "bounds = [(0, 1), (0, 1)]  # a, b >= 0 and <= 1\n",
    "x0 = np.array([1/2, 1/2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "objective_function(x0, res1, ground_truth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.optimize import minimize\n",
    "\n",
    "result = minimize(objective_function, x0, args=(res1, ground_truth), bounds=bounds, constraints=con)\n",
    "\n",
    "print(\"Optimal values:\")\n",
    "print(\"a =\", result.x[0])\n",
    "print(\"b =\", result.x[1])\n",
    "print(\"Objective function value:\", result.fun)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r1_new = result.x[0] * crlft_test\n",
    "r2_new = result.x[1] * baseline_test\n",
    "\n",
    "res = r1_new + r2_new\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "panda = pd.DataFrame(res, columns=['p(Hallucination)'])\n",
    "panda.to_csv(result_path + f'/ensemble_baseline_crlft{n}.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# attach together crlft_val and baseline_val in a (2, 500) array\n",
    "res1 = np.stack((crlft_val, sequential_val), axis=0)\n",
    "res1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# optimization problem between the three models\n",
    "import numpy as np\n",
    "\n",
    "def objective_function(x, res, ground_truth):\n",
    "    a, b = x\n",
    "    return np.sum((a * res[0] + b * res[1] - ground_truth) ** 2)\n",
    "\n",
    "def constraint(x):\n",
    "    return np.sum(x) - 1\n",
    "\n",
    "con = {'type': 'eq', 'fun': constraint}\n",
    "bounds = [(0, 1), (0, 1)]  # a, b >= 0 and <= 1\n",
    "x0 = np.array([1/2, 1/2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "objective_function(x0, res1, ground_truth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.optimize import minimize\n",
    "\n",
    "result = minimize(objective_function, x0, args=(res1, ground_truth), bounds=bounds, constraints=con)\n",
    "\n",
    "print(\"Optimal values:\")\n",
    "print(\"a =\", result.x[0])\n",
    "print(\"b =\", result.x[1])\n",
    "print(\"Objective function value:\", result.fun)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r1_new = result.x[0] * crlft_test\n",
    "r2_new = result.x[1] * sequential_test\n",
    "\n",
    "res = r1_new + r2_new\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "panda = pd.DataFrame(res, columns=['p(Hallucination)'])\n",
    "panda.to_csv(result_path + f'/ensemble_sequential_crlft{n}.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# attach together crlft_val and baseline_val in a (2, 500) array\n",
    "res1 = np.stack((baseline_val, sequential_val), axis=0)\n",
    "res1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# optimization problem between the three models\n",
    "import numpy as np\n",
    "\n",
    "def objective_function(x, res, ground_truth):\n",
    "    a, b = x\n",
    "    return np.sum((a * res[0] + b * res[1] - ground_truth) ** 2)\n",
    "\n",
    "def constraint(x):\n",
    "    return np.sum(x) - 1\n",
    "\n",
    "con = {'type': 'eq', 'fun': constraint}\n",
    "bounds = [(0, 1), (0, 1)]  # a, b >= 0 and <= 1\n",
    "x0 = np.array([1/2, 1/2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "objective_function(x0, res1, ground_truth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.optimize import minimize\n",
    "\n",
    "result = minimize(objective_function, x0, args=(res1, ground_truth), bounds=bounds, constraints=con)\n",
    "\n",
    "print(\"Optimal values:\")\n",
    "print(\"a =\", result.x[0])\n",
    "print(\"b =\", result.x[1])\n",
    "print(\"Objective function value:\", result.fun)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r1_new = result.x[0] * baseline_test\n",
    "r2_new = result.x[1] * sequential_test\n",
    "\n",
    "res = r1_new + r2_new\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "panda = pd.DataFrame(res, columns=['p(Hallucination)'])\n",
    "panda.to_csv(result_path + f'/ensemble_sequential_baseline{n}.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}