7. bert_scores_monolingual_layer_selection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "from scipy.stats import pearsonr \n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('seaborn-whitegrid')\n",
    "import numpy as np\n",
    "\n",
    "lang2mono = {'EN': 'bert-base-uncased',\n",
    "             'ID': 'indobert-base-uncased',\n",
    "             'FR': 'camembert-base',\n",
    "             'TR': 'bert-base-turkish-uncased',\n",
    "             'ZH': 'bert-base-chinese',\n",
    "             'RU': 'rubert-base-cased',\n",
    "             'DE': 'bert-base-german-dbmdz-uncased',\n",
    "             'ES': 'bert-base-spanish-wwm-uncased'}\n",
    "\n",
    "LANG = 'RU'\n",
    "pretrained= lang2mono[LANG] \n",
    "\n",
    "path_human_focus = f'mturk/annotation_result/{LANG}/human_focus_final.csv'\n",
    "path_human_coverage = f'mturk/annotation_result/{LANG}/human_coverage_final.csv'\n",
    "human_foc = pd.read_csv(path_human_focus)\n",
    "human_cov = pd.read_csv(path_human_coverage)\n",
    "\n",
    "human_foc_PG = human_foc[human_foc['model']=='PG']\n",
    "human_foc_BERT = human_foc[human_foc['model']=='BERT']\n",
    "human_cov_PG = human_cov[human_cov['model']=='PG']\n",
    "human_cov_BERT = human_cov[human_cov['model']=='BERT']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_array(arr):\n",
    "    arr = arr.replace('[','').replace(']','')\n",
    "    num = arr.split(', ')\n",
    "    num = [float(a) for a in num]\n",
    "    return num\n",
    "\n",
    "def read_array2(arr):\n",
    "    arr = arr.replace('[','').replace(']','')\n",
    "    num = arr.split(', ')\n",
    "    num = [a.replace('\\'','') for a in num]\n",
    "    return num\n",
    "\n",
    "def match(keys, values):\n",
    "    res = {}\n",
    "    for idx, key in enumerate(keys):\n",
    "        res[key] = values[idx]\n",
    "    return res\n",
    "        \n",
    "def correlation(human_metric, id2score):\n",
    "    filtered_human = []\n",
    "    filtered_metric = []\n",
    "    \n",
    "    for key in id2score.keys():\n",
    "        if len(human_metric[human_metric['id']==int(key)]) == 0:\n",
    "            continue\n",
    "        human_score = human_metric[human_metric['id'] == int(key)]['score'].tolist()[0]\n",
    "        filtered_human.append(human_score)\n",
    "        filtered_metric.append(id2score[key])\n",
    "    return pearsonr(filtered_human, filtered_metric)[0], len(filtered_human)\n",
    "\n",
    "def get_max(arr):\n",
    "    max_id = np.argmax(arr)\n",
    "    print(max_id+1, '--', arr[max_id])\n",
    "    min_id = np.argmin(arr)\n",
    "    print(min_id+1, '--', arr[min_id])\n",
    "    \n",
    "def get_best_rank(a,b):\n",
    "    r1 = [sorted(a).index(x) for x in a]\n",
    "    r2 = [sorted(b).index(x) for x in b]\n",
    "    score = 0\n",
    "    rank = -1\n",
    "    for i in range(len(r1)):\n",
    "        tmp = (a[i] + b[i])/2\n",
    "        if tmp > score:\n",
    "            rank = i\n",
    "            score = tmp\n",
    "    print('Best layer:', rank+1)\n",
    "    return rank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read(model, human_score, prec_or_rec):\n",
    "    path = f'bert_score/{LANG}--{model}--{pretrained}.csv'\n",
    "    df = pd.read_csv(path)\n",
    "    xlabel = []; cors = []\n",
    "    for idx, row in df.iterrows():\n",
    "        cur_layer = row['layer']\n",
    "        score = read_array(row[prec_or_rec])\n",
    "        doc_id = read_array2(row['fnames'])\n",
    "        id2score = match(doc_id, score)\n",
    "        \n",
    "        xlabel.append(cur_layer)\n",
    "        cors.append(correlation(human_score, id2score)[0])\n",
    "    return xlabel, cors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FOCUS for language: RU with model rubert-base-cased\n",
      "Best layer: 4\n",
      "PG: 0.42013286694278185\n",
      "BERT: 0.7052891181742704\n"
     ]
    }
   ],
   "source": [
    "# Focus\n",
    "print ('FOCUS for language:', LANG, 'with model', pretrained)\n",
    "\n",
    "xlabel_pg, cors_pg = read('PG', human_foc_PG, 'precision')\n",
    "xlabel_bert, cors_bert = read('BERT', human_foc_BERT, 'precision')\n",
    "layer = get_best_rank(cors_bert, cors_pg)\n",
    "\n",
    "print('PG:', cors_pg[layer])\n",
    "print('BERT:', cors_bert[layer])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "COVERAGE for language: RU with model rubert-base-cased\n",
      "Best layer: 12\n",
      "PG: 0.7222650638839587\n",
      "BERT: 0.6327758709514879\n"
     ]
    }
   ],
   "source": [
    "# Coverage\n",
    "print ('COVERAGE for language:', LANG,  'with model', pretrained)\n",
    "\n",
    "xlabel_pg, cors_pg = read('PG', human_cov_PG, 'recall')\n",
    "xlabel_bert, cors_bert = read('BERT', human_cov_BERT, 'recall')\n",
    "layer = get_best_rank(cors_bert, cors_pg)\n",
    "\n",
    "print('PG:', cors_pg[layer])\n",
    "print('BERT:', cors_bert[layer])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}