0. create_samples.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First, please download, Liputan6, LCSTS, and MLSUM.\n",
    "# Separate the test set with the folder names below. \n",
    "# Reference summary is put in a folder: gold or reference\n",
    "# PG summary is put in a folder: PG or pg_author\n",
    "# BERT summary is put in a folder: indoBertExtAbs or mBertAbs or bert_author\n",
    "# Naming config for each doc-summary: Please use a unique file name for each file \n",
    "\n",
    "import glob\n",
    "import pandas as pd\n",
    "import random\n",
    "\n",
    "ID=('/Data/Liputan6_final/output_test', 'reference', 'PG', 'indoBertExtAbs')\n",
    "ZH=('/Data/LCSTS2.0/LCSTS2.0/output_test', 'reference', 'PG', 'mBertAbs')\n",
    "FR=('/Workspace/mlsum/data/fr/test', 'gold', 'pg_author', 'bert_author')\n",
    "RU=('/Workspace/mlsum/data/ru/test', 'gold', 'pg_author', 'bert_author')\n",
    "ES=('/Workspace/mlsum/data/es/test', 'gold', 'pg_author', 'bert_author')\n",
    "DE=('/Workspace/mlsum/data/de/test', 'gold', 'pg_author', 'bert_author')\n",
    "TR=('/Workspace/mlsum/data/tu/test', 'gold', 'pg_author', 'bert_author')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "TOTAL=135\n",
    "TRAP_SAME=30\n",
    "TRAP_DIFF=30\n",
    "\n",
    "def read(fname):\n",
    "    return ''.join(open(fname).readlines())\n",
    "\n",
    "def create_data(lang, path, gold_folder, pg_folder, bert_folder, use_oracle=False):\n",
    "    all_golds = glob.glob(path+'/'+gold_folder+'/*')\n",
    "    random.shuffle(all_golds)\n",
    "    golds = all_golds[:TOTAL]\n",
    "    \n",
    "    data = []\n",
    "    for idx in range(len(golds)):\n",
    "        fname = golds[idx].split('/')[-1]\n",
    "        ids = fname.replace('_reference','').replace('.txt','')\n",
    "        \n",
    "        ref = read(golds[idx])\n",
    "        sum_pg = read(path+'/'+pg_folder+'/'+fname.replace('reference', 'decoded'))\n",
    "        sum_bert = read(path+'/'+bert_folder+'/'+fname.replace('reference', 'decoded'))\n",
    "        \n",
    "        cur_id = f\"{lang}-{ids}-SYSTEM-REFERENCE-PG\"\n",
    "        data.append((cur_id,sum_pg,ref))\n",
    "\n",
    "        cur_id = f\"{lang}-{ids}-REFERENCE-SYSTEM-PG\"\n",
    "        data.append((cur_id,ref,sum_pg))\n",
    "\n",
    "        cur_id = f\"{lang}-{ids}-SYSTEM-REFERENCE-BERT\"\n",
    "        data.append((cur_id,sum_bert,ref))\n",
    "\n",
    "        cur_id = f\"{lang}-{ids}-REFERENCE-SYSTEM-BERT\"\n",
    "        data.append((cur_id,ref,sum_bert))\n",
    "        \n",
    "    trap = []\n",
    "    golds = all_golds[TOTAL:TOTAL+TRAP_SAME]\n",
    "    for idx in range(len(golds)):\n",
    "        fname = golds[idx].split('/')[-1]\n",
    "        ids = fname.replace('_reference','').replace('.txt','')\n",
    "        \n",
    "        ref = read(golds[idx])\n",
    "        noise = ''\n",
    "        if use_oracle:\n",
    "            noise = open(path+'/oracle/'+fname.replace('reference', 'decoded')).readlines()[-1]+'\\n'\n",
    "        cur_id = f\"{lang}-{ids}-TRAP-SAME\"\n",
    "        trap.append((cur_id, noise+ref, ref))\n",
    "        \n",
    "    while len(trap) < TRAP_SAME+TRAP_DIFF:\n",
    "        a = random.randint(TOTAL+TRAP_SAME, len(all_golds)-1)\n",
    "        b = random.randint(TOTAL+TRAP_SAME, len(all_golds)-1)\n",
    "        if a == b:\n",
    "            continue\n",
    "            \n",
    "        fname1 = all_golds[a].split('/')[-1]\n",
    "        ids1 = fname1.replace('_reference','').replace('.txt','')\n",
    "        fname2 = all_golds[b].split('/')[-1]\n",
    "        ids2 = fname2.replace('_reference','').replace('.txt','')\n",
    "        \n",
    "        cur_id = f\"{lang}-{ids1}_{ids2}-TRAP-DIFFERENT\"\n",
    "        ref1 = read(all_golds[a])\n",
    "        ref2 = read(all_golds[b])\n",
    "        trap.append((cur_id, ref1, ref2))\n",
    "    \n",
    "    return data, trap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_ID, trap_ID = create_data('ID', ID[0], ID[1], ID[2], ID[3], True)\n",
    "data_ZH, trap_ZH = create_data('ZH', ZH[0], ZH[1], ZH[2], ZH[3], False)\n",
    "data_FR, trap_FR = create_data('FR', FR[0], FR[1], FR[2], FR[3], True)\n",
    "data_RU, trap_RU = create_data('RU', RU[0], RU[1], RU[2], RU[3], True)\n",
    "data_ES, trap_ES = create_data('ES', ES[0], ES[1], ES[2], ES[3], True)\n",
    "data_TR, trap_TR = create_data('TR', TR[0], TR[1], TR[2], TR[3], True)\n",
    "data_DE, trap_DE = create_data('DE', DE[0], DE[1], DE[2], DE[3], True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir mturk/samples\n",
    "\n",
    "def save_to(d, fname):\n",
    "    df = pd.DataFrame()\n",
    "    ids = []; texts1 = []; texts2 = []\n",
    "    for id, text1, text2 in d:\n",
    "        ids.append(id)\n",
    "        texts1.append(text1)\n",
    "        texts2.append(text2)\n",
    "    df['id'] = ids\n",
    "    df['text1'] = texts1\n",
    "    df['text2'] = texts2\n",
    "    df.to_csv(fname,index=False)\n",
    "    \n",
    "save_to(data_ID, 'mturk/samples/data_ID.csv'); save_to(trap_ID, 'mturk/samples/trap_ID.csv')\n",
    "save_to(data_ZH, 'mturk/samples/data_ZH.csv'); save_to(trap_ZH, 'mturk/samples/trap_ZH.csv')\n",
    "save_to(data_FR, 'mturk/samples/data_FR.csv'); save_to(trap_FR, 'mturk/samples/trap_FR.csv')\n",
    "save_to(data_RU, 'mturk/samples/data_RU.csv'); save_to(trap_RU, 'mturk/samples/trap_RU.csv')\n",
    "save_to(data_ES, 'mturk/samples/data_ES.csv'); save_to(trap_ES, 'mturk/samples/trap_ES.csv')\n",
    "save_to(data_TR, 'mturk/samples/data_TR.csv'); save_to(trap_TR, 'mturk/samples/trap_TR.csv')\n",
    "save_to(data_DE, 'mturk/samples/data_DE.csv'); save_to(trap_DE, 'mturk/samples/trap_DE.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(data_ID) == len(data_ZH) == len(data_FR) == len(data_RU) == len(data_ES) == len(data_TR) == len(data_DE)\n",
    "assert len(trap_ID) == len(trap_ZH) == len(trap_FR) == len(trap_RU) == len(trap_ES) == len(trap_TR) == len(trap_DE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}