7. correlation_of_bert_scores_combined.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "from scipy.stats import pearsonr, spearmanr\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('seaborn-whitegrid')\n",
    "import numpy as np\n",
    "\n",
    "lang2mono = {'EN': 'bert-base-uncased',\n",
    "             'ID': 'indobert-base-uncased',\n",
    "             'FR': 'camembert-base',\n",
    "             'TR': 'bert-base-turkish-uncased',\n",
    "             'ZH': 'bert-base-chinese',\n",
    "             'RU': 'rubert-base-cased',\n",
    "             'DE': 'bert-base-german-dbmdz-uncased',\n",
    "             'ES': 'bert-base-spanish-wwm-uncased'}\n",
    "\n",
    "LANGS = ['EN', 'ID', 'FR', 'TR', 'ZH', 'RU', 'DE', 'ES']\n",
    "\n",
    "model2layer = {}\n",
    "model2layer ['focus']= {\n",
    "    'bert-base-multilingual-cased': 12,\n",
    "    'bert-base-multilingual-uncased': 12,\n",
    "    'xlm-roberta-base': 4,\n",
    "    'xlm-roberta-large': 10,\n",
    "    lang2mono['EN']: 1,\n",
    "    lang2mono['ID']: 2,\n",
    "    lang2mono['FR']: 10,\n",
    "    lang2mono['TR']: 12,\n",
    "    lang2mono['ZH']: 8,\n",
    "    lang2mono['RU']: 4,\n",
    "    lang2mono['DE']: 12,\n",
    "    lang2mono['ES']: 4\n",
    "}\n",
    "model2layer ['coverage']= {\n",
    "    'bert-base-multilingual-cased': 5,\n",
    "    'bert-base-multilingual-uncased': 6,\n",
    "    'xlm-roberta-base': 4,\n",
    "    'xlm-roberta-large': 9,\n",
    "    lang2mono['EN']: 2,\n",
    "    lang2mono['ID']: 2,\n",
    "    lang2mono['FR']: 9,\n",
    "    lang2mono['TR']: 4,\n",
    "    lang2mono['ZH']: 9,\n",
    "    lang2mono['RU']: 12,\n",
    "    lang2mono['DE']: 12,\n",
    "    lang2mono['ES']: 4\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_array(arr):\n",
    "    arr = arr.replace('[','').replace(']','')\n",
    "    num = arr.split(', ')\n",
    "    num = [float(a) for a in num]\n",
    "    return num\n",
    "\n",
    "def read_array2(arr):\n",
    "    arr = arr.replace('[','').replace(']','')\n",
    "    num = arr.split(', ')\n",
    "    num = [a.replace('\\'','') for a in num]\n",
    "    return num\n",
    "\n",
    "def align(score, doc_id, human_score, model):\n",
    "    human = []\n",
    "    machine = []\n",
    "    for idx, doc in enumerate(doc_id):\n",
    "        human.append(human_score[(human_score['model'] == model) & (human_score['id']==int(doc))]['score'].values[0])\n",
    "        machine.append(score[idx])\n",
    "    return human, machine\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_human_annotation(lang, types='focus'):\n",
    "    if types=='focus':\n",
    "        path_human = f'mturk/annotation_result/{lang}/human_focus_final.csv'\n",
    "    else:\n",
    "        assert types=='coverage'\n",
    "        path_human = f'mturk/annotation_result/{lang}/human_coverage_final.csv'\n",
    "    human = pd.read_csv(path_human)\n",
    "    return human\n",
    "\n",
    "def read(lang, human_score, prec_or_rec, pretrained):\n",
    "    path_BERT = f'bert_score/{lang}--BERT--{pretrained}.csv'\n",
    "    path_PG = f'bert_score/{lang}--PG--{pretrained}.csv'\n",
    "    \n",
    "    if prec_or_rec == 'precision':\n",
    "        layer = model2layer['focus'][pretrained]-1\n",
    "    elif prec_or_rec == 'recall':\n",
    "        layer = model2layer['coverage'][pretrained]-1\n",
    "    \n",
    "    humans = []; machines = []\n",
    "\n",
    "    row_BERT = pd.read_csv(path_BERT).iloc[layer]\n",
    "    score = read_array(row_BERT[prec_or_rec])\n",
    "    doc_id = read_array2(row_BERT['fnames'])\n",
    "    human, machine = align(score, doc_id, human_score, 'BERT')\n",
    "    humans += human\n",
    "    machines += machine\n",
    "    \n",
    "    row_PG = pd.read_csv(path_PG).iloc[layer]\n",
    "    score = read_array(row_PG[prec_or_rec])\n",
    "    doc_id = read_array2(row_PG['fnames'])\n",
    "    human, machine = align(score, doc_id, human_score, 'PG')\n",
    "    humans += human\n",
    "    machines += machine    \n",
    "    \n",
    "    #print(humans)\n",
    "    #print(machines)\n",
    "    #return spearmanr(humans, machines)[0], len(humans)\n",
    "    return pearsonr(humans, machines)[0], len(humans)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Focus\n",
      "======\n",
      "EN 270 bert-base-uncased 0.6158439623578744\n",
      "EN 270 bert-base-multilingual-cased 0.5612550112988286\n",
      "EN 270 bert-base-multilingual-uncased 0.6086249858484727\n",
      "EN 270 xlm-roberta-base 0.5917245686937769\n",
      "EN 270 xlm-roberta-large 0.6047171359808301\n",
      "\n",
      "ID 270 indobert-base-uncased 0.710191152161175\n",
      "ID 270 bert-base-multilingual-cased 0.7101476384194889\n",
      "ID 270 bert-base-multilingual-uncased 0.7060564258715736\n",
      "ID 270 xlm-roberta-base 0.6502071868701137\n",
      "ID 270 xlm-roberta-large 0.6593119982769342\n",
      "\n",
      "FR 270 camembert-base 0.7268621871710514\n",
      "FR 270 bert-base-multilingual-cased 0.7277591566792305\n",
      "FR 270 bert-base-multilingual-uncased 0.7163022976154125\n",
      "FR 270 xlm-roberta-base 0.6679124180113589\n",
      "FR 270 xlm-roberta-large 0.6788438303776856\n",
      "\n",
      "TR 270 bert-base-turkish-uncased 0.8289235634266731\n",
      "TR 270 bert-base-multilingual-cased 0.8257716370051335\n",
      "TR 270 bert-base-multilingual-uncased 0.8327118074734108\n",
      "TR 270 xlm-roberta-base 0.8284027329267982\n",
      "TR 270 xlm-roberta-large 0.8283367091383284\n",
      "\n",
      "ZH 270 bert-base-chinese 0.8176867368384672\n",
      "ZH 270 bert-base-multilingual-cased 0.7783866777516224\n",
      "ZH 270 bert-base-multilingual-uncased 0.7927240366081347\n",
      "ZH 270 xlm-roberta-base 0.7932861139233667\n",
      "ZH 270 xlm-roberta-large 0.7881140075992721\n",
      "\n",
      "RU 270 rubert-base-cased 0.511868531965126\n",
      "RU 270 bert-base-multilingual-cased 0.562080997164221\n",
      "RU 270 bert-base-multilingual-uncased 0.5479744510224577\n",
      "RU 270 xlm-roberta-base 0.3400527445445146\n",
      "RU 270 xlm-roberta-large 0.4232212018237026\n",
      "\n",
      "DE 270 bert-base-german-dbmdz-uncased 0.9081212953119638\n",
      "DE 270 bert-base-multilingual-cased 0.9007581108598679\n",
      "DE 270 bert-base-multilingual-uncased 0.8984424408420929\n",
      "DE 270 xlm-roberta-base 0.8899204194593939\n",
      "DE 270 xlm-roberta-large 0.8947190748551364\n",
      "\n",
      "ES 270 bert-base-spanish-wwm-uncased 0.6698665207864025\n",
      "ES 270 bert-base-multilingual-cased 0.5944742046993414\n",
      "ES 270 bert-base-multilingual-uncased 0.620465014453632\n",
      "ES 270 xlm-roberta-base 0.5810475749262012\n",
      "ES 270 xlm-roberta-large 0.6042276710310389\n",
      "\n"
     ]
    }
   ],
   "source": [
    "LANGS = ['EN', 'ID', 'FR', 'TR', 'ZH', 'RU', 'DE', 'ES']\n",
    "\n",
    "# Focus\n",
    "print('Focus\\n======')\n",
    "cors = {}\n",
    "for lang in LANGS:\n",
    "    human_score = read_human_annotation(lang, 'focus')    \n",
    "    for pretrained in [ lang2mono[lang], 'bert-base-multilingual-cased', 'bert-base-multilingual-uncased', \\\n",
    "                      'xlm-roberta-base', 'xlm-roberta-large']:     \n",
    "        cor, num = read(lang, human_score, 'precision', pretrained)\n",
    "        print(lang, num, pretrained, cor)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coverage\n",
      "========\n",
      "EN 270 bert-base-uncased 0.6546360874136736\n",
      "EN 270 bert-base-multilingual-cased 0.6686934400566628\n",
      "EN 270 bert-base-multilingual-uncased 0.636231907014395\n",
      "EN 270 xlm-roberta-base 0.6433817842783182\n",
      "EN 270 xlm-roberta-large 0.6524665845137287\n",
      "\n",
      "ID 270 indobert-base-uncased 0.7422220397649907\n",
      "ID 270 bert-base-multilingual-cased 0.7308153106706617\n",
      "ID 270 bert-base-multilingual-uncased 0.742331566969608\n",
      "ID 270 xlm-roberta-base 0.7108821215931811\n",
      "ID 270 xlm-roberta-large 0.6970983195778044\n",
      "\n",
      "FR 270 camembert-base 0.7676825283082779\n",
      "FR 270 bert-base-multilingual-cased 0.6968890673152879\n",
      "FR 270 bert-base-multilingual-uncased 0.7187792671951606\n",
      "FR 270 xlm-roberta-base 0.6641338712438261\n",
      "FR 270 xlm-roberta-large 0.6911131083932085\n",
      "\n",
      "TR 270 bert-base-turkish-uncased 0.880702636763056\n",
      "TR 270 bert-base-multilingual-cased 0.8734407931404846\n",
      "TR 270 bert-base-multilingual-uncased 0.8712624633768853\n",
      "TR 270 xlm-roberta-base 0.8639387861127322\n",
      "TR 270 xlm-roberta-large 0.8609023390296602\n",
      "\n",
      "ZH 270 bert-base-chinese 0.7980292436025067\n",
      "ZH 270 bert-base-multilingual-cased 0.7923847972909925\n",
      "ZH 270 bert-base-multilingual-uncased 0.7909406332430293\n",
      "ZH 270 xlm-roberta-base 0.7333662874894139\n",
      "ZH 270 xlm-roberta-large 0.7390261493713464\n",
      "\n",
      "RU 270 rubert-base-cased 0.6460983117014905\n",
      "RU 270 bert-base-multilingual-cased 0.7175679553405314\n",
      "RU 270 bert-base-multilingual-uncased 0.7018635737826328\n",
      "RU 270 xlm-roberta-base 0.6704706578853743\n",
      "RU 270 xlm-roberta-large 0.6576173533141751\n",
      "\n",
      "DE 270 bert-base-german-dbmdz-uncased 0.9194091150988696\n",
      "DE 270 bert-base-multilingual-cased 0.9008847281518553\n",
      "DE 270 bert-base-multilingual-uncased 0.9019524298624142\n",
      "DE 270 xlm-roberta-base 0.8959912264394879\n",
      "DE 270 xlm-roberta-large 0.901767960148332\n",
      "\n",
      "ES 270 bert-base-spanish-wwm-uncased 0.7402420902693314\n",
      "ES 270 bert-base-multilingual-cased 0.7101209920220557\n",
      "ES 270 bert-base-multilingual-uncased 0.7115652734514264\n",
      "ES 270 xlm-roberta-base 0.694986016721575\n",
      "ES 270 xlm-roberta-large 0.7041448056082251\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Coverage\n",
    "print('Coverage\\n========')\n",
    "pretrained = 'bert-base-multilingual-cased'\n",
    "cors = {}\n",
    "for lang in LANGS:\n",
    "    human_score = read_human_annotation(lang, 'coverage')\n",
    "    for pretrained in [ lang2mono[lang], 'bert-base-multilingual-cased', 'bert-base-multilingual-uncased', \\\n",
    "                      'xlm-roberta-base', 'xlm-roberta-large']:  \n",
    "        cor, num = read(lang, human_score, 'recall', pretrained)\n",
    "        print(lang, num, pretrained, cor)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}