batched_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformer_lens import HookedTransformer\n",
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "from functools import partial\n",
    "from utils.backup_analysis import load_model\n",
    "from sys import getsizeof\n",
    "from collections.abc import Mapping, Container\n",
    "from utils.component_evaluation import compute_copy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch.autograd.grad_mode.set_grad_enabled at 0x7f11c4fd2d00>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.set_grad_enabled(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_gpu_memory_usage(label=\"\", device=\"cuda:0\"):\n",
    "    allocated = torch.cuda.memory_allocated(device) / (1024 ** 3)  # Convert bytes to GB\n",
    "    reserved = torch.cuda.memory_reserved(device) / (1024 ** 3)\n",
    "    print(f\"{label} - Memory Allocated: {allocated:.2f} GB, Memory Reserved: {reserved:.2f} GB\")\n",
    "\n",
    "def tensor_memory_size(tensor):\n",
    "    # Get the number of elements in the tensor\n",
    "    num_elements = tensor.numel()\n",
    "    # Determine the size of each element based on the data type\n",
    "    element_size = tensor.element_size()  # Returns size in bytes per element\n",
    "    # Calculate total memory footprint\n",
    "    memory_size = num_elements * element_size\n",
    "    return memory_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "BASE_MODEL = 'pythia-2.8b'\n",
    "VARIANT = None\n",
    "CACHE = 'model_cache'\n",
    "DEVICE = 'cuda:0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/ssd-1/ctigges/miniconda/envs/cot/lib/python3.8/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
      "  return self.fget.__get__(instance, owner)()\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded pretrained model pythia-2.8b into HookedTransformer\n",
      "After loading model - Memory Allocated: 10.60 GB, Memory Reserved: 10.61 GB\n"
     ]
    }
   ],
   "source": [
    "model = load_model(BASE_MODEL, VARIANT, 143000, CACHE, DEVICE, large_model=True)\n",
    "print_gpu_memory_usage(\"After loading model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.data_utils import generate_data_and_caches\n",
    "model.tokenizer.add_bos_token = False\n",
    "ioi_dataset, abc_dataset = generate_data_and_caches(model, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------\n",
      "Seq: Then, Alex and Anthony had a lot of fun at the school. Anthony gave a ring to Alex\n",
      "Target: Alex\n",
      "(1): a (2): the (3):less (4):, (5):\n",
      "\n",
      "-------\n",
      "Seq: Then, Alex and Anthony had a lot of fun at the school. Anthony gave a ring to Alex\n",
      "Target: Anthony\n",
      "(1):, (2): off (3):L (4):\n",
      " (5): it\n",
      "-------\n",
      "Seq: Then, Alex and Anthony had a lot of fun at the school. Anthony gave a ring to Alex\n",
      "Target: Anthony\n",
      "(1):, (2): off (3):\n",
      " (4):L (5):/\n",
      "-------\n",
      "Seq: Then, Connor and Roman were working at the house. Roman decided to give a basketball to Connor\n",
      "Target: Connor\n",
      "(1):. (2): of (3):, (4):\n",
      " (5): a\n",
      "-------\n",
      "Seq: Then, Connor and Roman were working at the house. Roman decided to give a basketball to Connor\n",
      "Target: Roman\n",
      "(1):\n",
      " (2): a (3): and (4): on (5): the\n",
      "-------\n",
      "Seq: Then, Connor and Roman were working at the house. Roman decided to give a basketball to Connor\n",
      "Target: Roman\n",
      "(1):\n",
      " (2): a (3): the (4): on (5): and\n",
      "-------\n",
      "Seq: Then, Eva and Mark were working at the store. Eva decided to give a bone to Mark\n",
      "Target: Mark\n",
      "(1): it (2):. (3):- (4): a (5): the\n",
      "-------\n",
      "Seq: Then, Eva and Mark were working at the store. Eva decided to give a bone to Mark\n",
      "Target: Eva\n",
      "(1):away (2): the (3): reason (4): coun (5): away\n",
      "-------\n",
      "Seq: Then, Eva and Mark were working at the store. Eva decided to give a bone to Mark\n",
      "Target: Eva\n",
      "(1): the (2): reason (3): on (4): ( (5): away\n",
      "-------\n",
      "Seq: Then, Ian and Jessica went to the restaurant. Ian gave a basketball to Jessica\n",
      "Target: Jessica\n",
      "(1): history (2): right (3): meaning (4): in (5): am\n",
      "-------\n",
      "Seq: Then, Ian and Jessica went to the restaurant. Ian gave a basketball to Jessica\n",
      "Target: Ian\n",
      "(1): meaning (2): ext (3): int (4):ively (5): positive\n",
      "-------\n",
      "Seq: Then, Ian and Jessica went to the restaurant. Ian gave a basketball to Jessica\n",
      "Target: Ian\n",
      "(1): meaning (2): positive (3): ext (4): business (5): sour\n",
      "-------\n",
      "Seq: Then, Austin and Ian had a lot of fun at the house. Austin gave a kiss to Ian\n",
      "Target: Ian\n",
      "(1): meaning (2): ext (3): int (4):ively (5): business\n",
      "-------\n",
      "Seq: Then, Austin and Ian had a lot of fun at the house. Austin gave a kiss to Ian\n",
      "Target: Austin\n",
      "(1): that (2): pol (3): be (4):ija (5): is\n",
      "-------\n",
      "Seq: Then, Austin and Ian had a lot of fun at the house. Austin gave a kiss to Ian\n",
      "Target: Austin\n",
      "(1): that (2): be (3): is (4):ija (5): fore\n",
      "-------\n",
      "Seq: Then, William and Jack had a long argument, and afterwards Jack said to William\n",
      "Target: William\n",
      "(1):, (2):\n",
      " (3):: (4):. (5):-\n",
      "-------\n",
      "Seq: Then, William and Jack had a long argument, and afterwards Jack said to William\n",
      "Target: Jack\n",
      "(1):\n",
      " (2):L (3): reasons (4): not (5):N\n",
      "-------\n",
      "Seq: Then, William and Jack had a long argument, and afterwards Jack said to William\n",
      "Target: Jack\n",
      "(1):\n",
      " (2):L (3): not (4): reasons (5): it\n",
      "-------\n",
      "Seq: Then, Lisa and Joseph went to the house. Lisa gave a kiss to Joseph\n",
      "Target: Joseph\n",
      "(1):/ (2):_ (3):\n",
      " (4): the (5): U\n",
      "-------\n",
      "Seq: Then, Lisa and Joseph went to the house. Lisa gave a kiss to Joseph\n",
      "Target: Lisa\n",
      "(1): where (2):l (3): \" (4): regions (5): high\n",
      "-------\n",
      "Seq: Then, Lisa and Joseph went to the house. Lisa gave a kiss to Joseph\n",
      "Target: Lisa\n",
      "(1): where (2):l (3): T (4): \" (5):,\n",
      "-------\n",
      "Seq: Then, Frank and Victoria were working at the office. Victoria decided to give a basketball to Frank\n",
      "Target: Frank\n",
      "(1):s (2):sub (3):aze (4):u (5):[]\n",
      "-------\n",
      "Seq: Then, Frank and Victoria were working at the office. Victoria decided to give a basketball to Frank\n",
      "Target: Victoria\n",
      "(1):\n",
      " (2): on (3): not (4):s (5): the\n",
      "-------\n",
      "Seq: Then, Frank and Victoria were working at the office. Victoria decided to give a basketball to Frank\n",
      "Target: Victoria\n",
      "(1):\n",
      " (2): not (3): on (4): the (5):s\n",
      "-------\n",
      "Seq: Then, Amy and Andrew went to the school. Amy gave a computer to Andrew\n",
      "Target: Andrew\n",
      "(1):. (2):, (3): all (4):\n",
      " (5): open\n",
      "-------\n",
      "Seq: Then, Amy and Andrew went to the school. Amy gave a computer to Andrew\n",
      "Target: Amy\n",
      "(1): the (2): or (3): a (4): all (5): information\n",
      "-------\n",
      "Seq: Then, Amy and Andrew went to the school. Amy gave a computer to Andrew\n",
      "Target: Amy\n",
      "(1): the (2): or (3):less (4): all (5): a\n",
      "-------\n",
      "Seq: After David and Arthur went to the hospital, Arthur gave a snack to David\n",
      "Target: David\n",
      "(1): the (2): than (3): of (4): off (5): object\n",
      "-------\n",
      "Seq: After David and Arthur went to the hospital, Arthur gave a snack to David\n",
      "Target: Arthur\n",
      "(1): on (2):. (3):- (4):\n",
      " (5):,\n",
      "-------\n",
      "Seq: After David and Arthur went to the hospital, Arthur gave a snack to David\n",
      "Target: Arthur\n",
      "(1): on (2):. (3):- (4):, (5): all\n",
      "-------\n",
      "Seq: Then, Jake and Victoria had a long argument, and afterwards Jake said to Victoria\n",
      "Target: Victoria\n",
      "(1): not (2):\n",
      " (3):s (4): on (5):adj\n",
      "-------\n",
      "Seq: Then, Jake and Victoria had a long argument, and afterwards Jake said to Victoria\n",
      "Target: Jake\n",
      "(1): the (2):\n",
      " (3):. (4):, (5):as\n",
      "-------\n",
      "Seq: Then, Jake and Victoria had a long argument, and afterwards Jake said to Victoria\n",
      "Target: Jake\n",
      "(1): the (2):. (3):\n",
      " (4):, (5):-\n",
      "-------\n",
      "Seq: When Eric and Peter got a bone at the garden, Eric decided to give it to Peter\n",
      "Target: Peter\n",
      "(1):. (2):s (3): the (4): and (5):,\n",
      "-------\n",
      "Seq: When Eric and Peter got a bone at the garden, Eric decided to give it to Peter\n",
      "Target: Eric\n",
      "(1): the (2): in (3): I (4): on (5):,\n",
      "-------\n",
      "Seq: When Eric and Peter got a bone at the garden, Eric decided to give it to Peter\n",
      "Target: Eric\n",
      "(1): the (2): on (3): in (4):, (5):\n",
      "\n",
      "-------\n",
      "Seq: Then, Ruby and Edward were working at the station. Edward decided to give a drink to Ruby\n",
      "Target: Ruby\n",
      "(1):\n",
      " (2): a (3):S (4): in (5):L\n",
      "-------\n",
      "Seq: Then, Ruby and Edward were working at the station. Edward decided to give a drink to Ruby\n",
      "Target: Edward\n",
      "(1):\n",
      " (2): not (3): C (4):, (5):s\n",
      "-------\n",
      "Seq: Then, Ruby and Edward were working at the station. Edward decided to give a drink to Ruby\n",
      "Target: Edward\n",
      "(1):\n",
      " (2): not (3):, (4): C (5): T\n",
      "-------\n",
      "Seq: When Michelle and Richard got a necklace at the school, Richard decided to give it to Michelle\n",
      "Target: Michelle\n",
      "(1): ( (2): \" (3): ' (4): C (5): less\n",
      "-------\n",
      "Seq: When Michelle and Richard got a necklace at the school, Richard decided to give it to Michelle\n",
      "Target: Richard\n",
      "(1):s (2): \" (3): or (4):\n",
      " (5): (\n",
      "-------\n",
      "Seq: When Michelle and Richard got a necklace at the school, Richard decided to give it to Michelle\n",
      "Target: Richard\n",
      "(1):s (2): \" (3):\n",
      " (4): or (5): most\n",
      "-------\n",
      "Seq: Then, Sean and Simon were thinking about going to the school. Simon wanted to give a ring to Sean\n",
      "Target: Sean\n",
      "(1):\n",
      " (2):, (3): on (4): all (5): the\n",
      "-------\n",
      "Seq: Then, Sean and Simon were thinking about going to the school. Simon wanted to give a ring to Sean\n",
      "Target: Simon\n",
      "(1): semic (2): that (3): partial (4): it (5):ermine\n",
      "-------\n",
      "Seq: Then, Sean and Simon were thinking about going to the school. Simon wanted to give a ring to Sean\n",
      "Target: Simon\n",
      "(1): partial (2): that (3): it (4): semic (5): depend\n",
      "-------\n",
      "Seq: Then, Jay and Graham had a lot of fun at the garden. Jay gave a basketball to Graham\n",
      "Target: Graham\n",
      "(1):usi (2):� (3):RG (4):arshal (5):xl\n",
      "-------\n",
      "Seq: Then, Jay and Graham had a lot of fun at the garden. Jay gave a basketball to Graham\n",
      "Target: Jay\n",
      "(1): the (2):\n",
      " (3): a (4): D (5):less\n",
      "-------\n",
      "Seq: Then, Jay and Graham had a lot of fun at the garden. Jay gave a basketball to Graham\n",
      "Target: Jay\n",
      "(1):\n",
      " (2): the (3): a (4): D (5):,\n",
      "-------\n",
      "Seq: Then, Graham and Roman were thinking about going to the station. Graham wanted to give a drink to Roman\n",
      "Target: Roman\n",
      "(1):\n",
      " (2): a (3): and (4): the (5): on\n",
      "-------\n",
      "Seq: Then, Graham and Roman were thinking about going to the station. Graham wanted to give a drink to Roman\n",
      "Target: Graham\n",
      "(1):usi (2):� (3):CFG (4):xl (5): arrangements\n",
      "-------\n",
      "Seq: Then, Graham and Roman were thinking about going to the station. Graham wanted to give a drink to Roman\n",
      "Target: Graham\n",
      "(1):� (2):usi (3):rb (4):xl (5):CFG\n",
      "-------\n",
      "Seq: Then, Charles and Joseph had a long argument, and afterwards Charles said to Joseph\n",
      "Target: Joseph\n",
      "(1):/ (2):_ (3):\n",
      " (4):, (5):.\n",
      "-------\n",
      "Seq: Then, Charles and Joseph had a long argument, and afterwards Charles said to Joseph\n",
      "Target: Charles\n",
      "(1):sub (2):s (3):alk (4):ivities (5):\n",
      "\n",
      "-------\n",
      "Seq: Then, Charles and Joseph had a long argument, and afterwards Charles said to Joseph\n",
      "Target: Charles\n",
      "(1):sub (2):s (3):Partial (4):ivities (5): Partial\n",
      "-------\n",
      "Seq: Then, Lisa and Madison went to the house. Lisa gave a drink to Madison\n",
      "Target: Madison\n",
      "(1): partner (2): ( (3): familiar (4): than (5): Home\n",
      "-------\n",
      "Seq: Then, Lisa and Madison went to the house. Lisa gave a drink to Madison\n",
      "Target: Lisa\n",
      "(1): where (2):l (3): \" (4): regions (5): high\n",
      "-------\n",
      "Seq: Then, Lisa and Madison went to the house. Lisa gave a drink to Madison\n",
      "Target: Lisa\n",
      "(1): where (2):l (3): T (4): \" (5):,\n",
      "-------\n",
      "Seq: Then, Daniel and Carter had a long argument, and afterwards Carter said to Daniel\n",
      "Target: Daniel\n",
      "(1):\n",
      " (2):, (3): exp (4): on (5): of\n",
      "-------\n",
      "Seq: Then, Daniel and Carter had a long argument, and afterwards Carter said to Daniel\n",
      "Target: Carter\n",
      "(1): that (2): o (3):\n",
      " (4):, (5): the\n",
      "-------\n",
      "Seq: Then, Daniel and Carter had a long argument, and afterwards Carter said to Daniel\n",
      "Target: Carter\n",
      "(1): that (2): o (3): of (4): the (5):,\n",
      "-------\n",
      "Seq: Then, Blake and Anderson had a lot of fun at the restaurant. Blake gave a drink to Anderson\n",
      "Target: Anderson\n",
      "(1):shots (2): seg (3): Baltic (4):shot (5): attention\n",
      "-------\n",
      "Seq: Then, Blake and Anderson had a lot of fun at the restaurant. Blake gave a drink to Anderson\n",
      "Target: Blake\n",
      "(1): phase (2): into (3): side (4):FI (5): masking\n",
      "-------\n",
      "Seq: Then, Blake and Anderson had a lot of fun at the restaurant. Blake gave a drink to Anderson\n",
      "Target: Blake\n",
      "(1): side (2): phase (3): masking (4): root (5): into\n",
      "-------\n",
      "Seq: Then, Jane and Andrew were thinking about going to the station. Jane wanted to give a computer to Andrew\n",
      "Target: Andrew\n",
      "(1):. (2):, (3): all (4):\n",
      " (5)::\n",
      "-------\n",
      "Seq: Then, Jane and Andrew were thinking about going to the station. Jane wanted to give a computer to Andrew\n",
      "Target: Jane\n",
      "(1):_ (2): | (3): most (4):IA (5):ай\n",
      "-------\n",
      "Seq: Then, Jane and Andrew were thinking about going to the station. Jane wanted to give a computer to Andrew\n",
      "Target: Jane\n",
      "(1): | (2):_ (3):IA (4): most (5):sec\n",
      "-------\n",
      "Seq: Then, Adam and Rachel were working at the garden. Rachel decided to give a computer to Adam\n",
      "Target: Adam\n",
      "(1): the (2):, (3):. (4): even (5): \n",
      "-------\n",
      "Seq: Then, Adam and Rachel were working at the garden. Rachel decided to give a computer to Adam\n",
      "Target: Rachel\n",
      "(1): the (2):\n",
      " (3): on (4):, (5): in\n",
      "-------\n",
      "Seq: Then, Adam and Rachel were working at the garden. Rachel decided to give a computer to Adam\n",
      "Target: Rachel\n",
      "(1): the (2):\n",
      " (3): on (4):, (5): in\n",
      "-------\n",
      "Seq: Then, Ian and Morgan had a lot of fun at the hospital. Ian gave a drink to Morgan\n",
      "Target: Morgan\n",
      "(1):, (2): the (3): a (4): gen (5): ambit\n",
      "-------\n",
      "Seq: Then, Ian and Morgan had a lot of fun at the hospital. Ian gave a drink to Morgan\n",
      "Target: Ian\n",
      "(1): meaning (2): ext (3): int (4):ively (5): positive\n",
      "-------\n",
      "Seq: Then, Ian and Morgan had a lot of fun at the hospital. Ian gave a drink to Morgan\n",
      "Target: Ian\n",
      "(1): meaning (2): ext (3): positive (4): int (5): business\n",
      "-------\n",
      "Seq: Then, Joseph and Aaron went to the hospital. Aaron gave a bone to Joseph\n",
      "Target: Joseph\n",
      "(1):/ (2):\n",
      " (3): the (4):_ (5):,\n",
      "-------\n",
      "Seq: Then, Joseph and Aaron went to the hospital. Aaron gave a bone to Joseph\n",
      "Target: Aaron\n",
      "(1):gt (2): the (3):\n",
      " (4): all (5):PT\n",
      "-------\n",
      "Seq: Then, Joseph and Aaron went to the hospital. Aaron gave a bone to Joseph\n",
      "Target: Aaron\n",
      "(1):gt (2): the (3):\n",
      " (4):. (5):ie\n",
      "-------\n",
      "Seq: Then, Austin and Michael were working at the school. Michael decided to give a bone to Austin\n",
      "Target: Austin\n",
      "(1): that (2): pol (3): be (4):ija (5): is\n",
      "-------\n",
      "Seq: Then, Austin and Michael were working at the school. Michael decided to give a bone to Austin\n",
      "Target: Michael\n",
      "(1):\n",
      " (2): ( (3):, (4): the (5):.\n",
      "-------\n",
      "Seq: Then, Austin and Michael were working at the school. Michael decided to give a bone to Austin\n",
      "Target: Michael\n",
      "(1):\n",
      " (2): the (3):, (4): ( (5):.\n",
      "-------\n",
      "Seq: Then, Scott and Christian had a long argument, and afterwards Christian said to Scott\n",
      "Target: Scott\n",
      "(1):\n",
      " (2): c (3):r (4):, (5): sy\n",
      "-------\n",
      "Seq: Then, Scott and Christian had a long argument, and afterwards Christian said to Scott\n",
      "Target: Christian\n",
      "(1): of (2): ( (3):Close (4): \" (5): and\n",
      "-------\n",
      "Seq: Then, Scott and Christian had a long argument, and afterwards Christian said to Scott\n",
      "Target: Christian\n",
      "(1): of (2): \" (3): ( (4): high (5): should\n",
      "-------\n",
      "Seq: Then, Blake and Alan were working at the hospital. Blake decided to give a basketball to Alan\n",
      "Target: Alan\n",
      "(1): may (2): ol (3): all (4):uben (5): C\n",
      "-------\n",
      "Seq: Then, Blake and Alan were working at the hospital. Blake decided to give a basketball to Alan\n",
      "Target: Blake\n",
      "(1): phase (2): into (3): side (4):FI (5): masking\n",
      "-------\n",
      "Seq: Then, Blake and Alan were working at the hospital. Blake decided to give a basketball to Alan\n",
      "Target: Blake\n",
      "(1): phase (2):Ath (3): side (4): masking (5): yet\n",
      "-------\n",
      "Seq: Then, Dean and Marco had a lot of fun at the garden. Marco gave a snack to Dean\n",
      "Target: Dean\n",
      "(1): on (2): that (3): a (4): sub (5):\n",
      "\n",
      "-------\n",
      "Seq: Then, Dean and Marco had a lot of fun at the garden. Marco gave a snack to Dean\n",
      "Target: Marco\n",
      "(1):\n",
      " (2):s (3): on (4):, (5): (\n",
      "-------\n",
      "Seq: Then, Dean and Marco had a lot of fun at the garden. Marco gave a snack to Dean\n",
      "Target: Marco\n",
      "(1):\n",
      " (2): on (3):s (4):, (5): cases\n",
      "-------\n",
      "Seq: Then, Andrew and Scott were thinking about going to the restaurant. Scott wanted to give a bone to Andrew\n",
      "Target: Andrew\n",
      "(1):. (2):, (3):\n",
      " (4): all (5): on\n",
      "-------\n",
      "Seq: Then, Andrew and Scott were thinking about going to the restaurant. Scott wanted to give a bone to Andrew\n",
      "Target: Scott\n",
      "(1):\n",
      " (2):, (3):. (4): as (5): c\n",
      "-------\n",
      "Seq: Then, Andrew and Scott were thinking about going to the restaurant. Scott wanted to give a bone to Andrew\n",
      "Target: Scott\n",
      "(1):\n",
      " (2): c (3):, (4):. (5): sy\n",
      "-------\n",
      "Seq: Then, Luke and Justin had a long argument, and afterwards Luke said to Justin\n",
      "Target: Justin\n",
      "(1): of (2):, (3): and (4):s (5): a\n",
      "-------\n",
      "Seq: Then, Luke and Justin had a long argument, and afterwards Luke said to Justin\n",
      "Target: Luke\n",
      "(1):, (2):: (3):. (4): and (5): a\n",
      "-------\n",
      "Seq: Then, Luke and Justin had a long argument, and afterwards Luke said to Justin\n",
      "Target: Luke\n",
      "(1):, (2):: (3):. (4): the (5): a\n",
      "-------\n",
      "Seq: When Sarah and Louis got a computer at the school, Sarah decided to give it to Louis\n",
      "Target: Louis\n",
      "(1):eday (2): vs (3):下 (4): depend (5): and\n",
      "-------\n",
      "Seq: When Sarah and Louis got a computer at the school, Sarah decided to give it to Louis\n",
      "Target: Sarah\n",
      "(1): the (2): foreign (3): either (4): on (5):\n",
      "\n",
      "-------\n",
      "Seq: When Sarah and Louis got a computer at the school, Sarah decided to give it to Louis\n",
      "Target: Sarah\n",
      "(1): the (2):\n",
      " (3): on (4):/ (5): foreign\n",
      "-------\n",
      "Seq: After Patrick and Peter went to the station, Patrick gave a snack to Peter\n",
      "Target: Peter\n",
      "(1):. (2):, (3):- (4):s (5): the\n",
      "-------\n",
      "Seq: After Patrick and Peter went to the station, Patrick gave a snack to Peter\n",
      "Target: Patrick\n",
      "(1):, (2): than (3):. (4): primary (5):-\n",
      "-------\n",
      "Seq: After Patrick and Peter went to the station, Patrick gave a snack to Peter\n",
      "Target: Patrick\n",
      "(1):, (2):. (3): than (4): as (5): ad\n",
      "-------\n",
      "Seq: After Edward and Daniel went to the office, Daniel gave a ring to Edward\n",
      "Target: Edward\n",
      "(1):\n",
      " (2): not (3): C (4):, (5):.\n",
      "-------\n",
      "Seq: After Edward and Daniel went to the office, Daniel gave a ring to Edward\n",
      "Target: Daniel\n",
      "(1):\n",
      " (2):, (3): am (4):itude (5): on\n",
      "-------\n",
      "Seq: After Edward and Daniel went to the office, Daniel gave a ring to Edward\n",
      "Target: Daniel\n",
      "(1):\n",
      " (2):, (3):— (4): on (5): exp\n",
      "-------\n",
      "Seq: Then, David and Luke were thinking about going to the school. David wanted to give a ring to Luke\n",
      "Target: Luke\n",
      "(1):, (2):. (3):: (4):_ (5):-\n",
      "-------\n",
      "Seq: Then, David and Luke were thinking about going to the school. David wanted to give a ring to Luke\n",
      "Target: David\n",
      "(1): the (2):\n",
      " (3): than (4): of (5):,\n",
      "-------\n",
      "Seq: Then, David and Luke were thinking about going to the school. David wanted to give a ring to Luke\n",
      "Target: David\n",
      "(1): the (2): than (3): of (4):\n",
      " (5): object\n",
      "-------\n",
      "Seq: After Robert and Michael went to the school, Michael gave a kiss to Robert\n",
      "Target: Robert\n",
      "(1):\n",
      " (2): off (3):, (4):- (5):.\n",
      "-------\n",
      "Seq: After Robert and Michael went to the school, Michael gave a kiss to Robert\n",
      "Target: Michael\n",
      "(1):\n",
      " (2): ( (3):. (4):, (5): all\n",
      "-------\n",
      "Seq: After Robert and Michael went to the school, Michael gave a kiss to Robert\n",
      "Target: Michael\n",
      "(1):\n",
      " (2): the (3): ( (4):, (5): all\n",
      "-------\n",
      "Seq: Then, Henry and Andy had a lot of fun at the school. Andy gave a computer to Henry\n",
      "Target: Henry\n",
      "(1):, (2):\n",
      " (3):. (4):  (5): O\n",
      "-------\n",
      "Seq: Then, Henry and Andy had a lot of fun at the school. Andy gave a computer to Henry\n",
      "Target: Andy\n",
      "(1): the (2): a (3): and (4):, (5): at\n",
      "-------\n",
      "Seq: Then, Henry and Andy had a lot of fun at the school. Andy gave a computer to Henry\n",
      "Target: Andy\n",
      "(1): the (2): a (3): and (4): at (5):,\n",
      "-------\n",
      "Seq: Then, Crystal and Kelly had a long argument, and afterwards Crystal said to Kelly\n",
      "Target: Kelly\n",
      "(1): the (2): near (3):, (4): open (5): right\n",
      "-------\n",
      "Seq: Then, Crystal and Kelly had a long argument, and afterwards Crystal said to Kelly\n",
      "Target: Crystal\n",
      "(1): on (2): at (3): not (4): so (5):sc\n",
      "-------\n",
      "Seq: Then, Crystal and Kelly had a long argument, and afterwards Crystal said to Kelly\n",
      "Target: Crystal\n",
      "(1): on (2): not (3): so (4): at (5):sc\n",
      "-------\n",
      "Seq: When Justin and Stephen got a basketball at the house, Justin decided to give it to Stephen\n",
      "Target: Stephen\n",
      "(1):\n",
      " (2):, (3):. (4): that (5): T\n",
      "-------\n",
      "Seq: When Justin and Stephen got a basketball at the house, Justin decided to give it to Stephen\n",
      "Target: Justin\n",
      "(1): an (2): of (3): a (4):, (5): all\n",
      "-------\n",
      "Seq: When Justin and Stephen got a basketball at the house, Justin decided to give it to Stephen\n",
      "Target: Justin\n",
      "(1): of (2):s (3): a (4): an (5):,\n",
      "-------\n",
      "Seq: After Arthur and Paul went to the office, Paul gave a drink to Arthur\n",
      "Target: Arthur\n",
      "(1): on (2):. (3):- (4): all (5): any\n",
      "-------\n",
      "Seq: After Arthur and Paul went to the office, Paul gave a drink to Arthur\n",
      "Target: Paul\n",
      "(1): the (2):, (3): to (4):\n",
      " (5): of\n",
      "-------\n",
      "Seq: After Arthur and Paul went to the office, Paul gave a drink to Arthur\n",
      "Target: Paul\n",
      "(1): the (2):, (3): to (4):\n",
      " (5): or\n",
      "-------\n",
      "Seq: After Ruby and Joshua went to the garden, Ruby gave a ring to Joshua\n",
      "Target: Joshua\n",
      "(1):, (2): C (3):\n",
      " (4): ( (5)::\n",
      "-------\n",
      "Seq: After Ruby and Joshua went to the garden, Ruby gave a ring to Joshua\n",
      "Target: Ruby\n",
      "(1): a (2):\n",
      " (3): the (4): in (5):S\n",
      "-------\n",
      "Seq: After Ruby and Joshua went to the garden, Ruby gave a ring to Joshua\n",
      "Target: Ruby\n",
      "(1):\n",
      " (2): a (3):S (4): in (5):L\n",
      "-------\n",
      "Seq: After Ian and Ruby went to the garden, Ian gave a basketball to Ruby\n",
      "Target: Ruby\n",
      "(1):\n",
      " (2): a (3):S (4): in (5):PROT\n",
      "-------\n",
      "Seq: After Ian and Ruby went to the garden, Ian gave a basketball to Ruby\n",
      "Target: Ian\n",
      "(1): meaning (2): open (3): part (4): business (5): ext\n",
      "-------\n",
      "Seq: After Ian and Ruby went to the garden, Ian gave a basketball to Ruby\n",
      "Target: Ian\n",
      "(1): meaning (2): positive (3): ext (4): part (5): sour\n",
      "-------\n",
      "Seq: When Lisa and Collins got a drink at the restaurant, Lisa decided to give it to Collins\n",
      "Target: Collins\n",
      "(1): as (2):ee (3): among (4): on (5): that\n",
      "-------\n",
      "Seq: When Lisa and Collins got a drink at the restaurant, Lisa decided to give it to Collins\n",
      "Target: Lisa\n",
      "(1): where (2): out (3):INCLUDING (4):NODE (5): cause\n",
      "-------\n",
      "Seq: When Lisa and Collins got a drink at the restaurant, Lisa decided to give it to Collins\n",
      "Target: Lisa\n",
      "(1): where (2):l (3): Lud (4): WO (5): \"\n",
      "-------\n",
      "Seq: Then, Eva and Joshua went to the garden. Joshua gave a snack to Eva\n",
      "Target: Eva\n",
      "(1):away (2): the (3): reason (4): coun (5): away\n",
      "-------\n",
      "Seq: Then, Eva and Joshua went to the garden. Joshua gave a snack to Eva\n",
      "Target: Joshua\n",
      "(1):, (2): C (3):\n",
      " (4): ( (5)::\n",
      "-------\n",
      "Seq: Then, Eva and Joshua went to the garden. Joshua gave a snack to Eva\n",
      "Target: Joshua\n",
      "(1):, (2):\n",
      " (3): C (4): ( (5):/\n",
      "-------\n",
      "Seq: Then, Arthur and Luke had a lot of fun at the store. Arthur gave a kiss to Luke\n",
      "Target: Luke\n",
      "(1):: (2):. (3):, (4):_ (5):;\n",
      "-------\n",
      "Seq: Then, Arthur and Luke had a lot of fun at the store. Arthur gave a kiss to Luke\n",
      "Target: Arthur\n",
      "(1): on (2): all (3):, (4): according (5):.\n",
      "-------\n",
      "Seq: Then, Arthur and Luke had a lot of fun at the store. Arthur gave a kiss to Luke\n",
      "Target: Arthur\n",
      "(1): on (2):, (3):\n",
      " (4):. (5):-\n",
      "-------\n",
      "Seq: Then, Dean and William had a lot of fun at the house. Dean gave a snack to William\n",
      "Target: William\n",
      "(1):\n",
      " (2):, (3):: (4):- (5):.\n",
      "-------\n",
      "Seq: Then, Dean and William had a lot of fun at the house. Dean gave a snack to William\n",
      "Target: Dean\n",
      "(1): on (2): that (3): a (4): sub (5):\n",
      "\n",
      "-------\n",
      "Seq: Then, Dean and William had a lot of fun at the house. Dean gave a snack to William\n",
      "Target: Dean\n",
      "(1): on (2): ter (3): that (4): first (5): dis\n",
      "-------\n",
      "Seq: Then, Jay and Crystal were working at the restaurant. Jay decided to give a drink to Crystal\n",
      "Target: Crystal\n",
      "(1): not (2): on (3):sc (4): so (5): at\n",
      "-------\n",
      "Seq: Then, Jay and Crystal were working at the restaurant. Jay decided to give a drink to Crystal\n",
      "Target: Jay\n",
      "(1): the (2):\n",
      " (3): a (4): D (5):less\n",
      "-------\n",
      "Seq: Then, Jay and Crystal were working at the restaurant. Jay decided to give a drink to Crystal\n",
      "Target: Jay\n",
      "(1):\n",
      " (2): the (3): a (4): D (5): (\n",
      "-------\n",
      "Seq: Then, Jane and Emily had a lot of fun at the station. Emily gave a computer to Jane\n",
      "Target: Jane\n",
      "(1):_ (2): | (3): most (4):IA (5):ай\n",
      "-------\n",
      "Seq: Then, Jane and Emily had a lot of fun at the station. Emily gave a computer to Jane\n",
      "Target: Emily\n",
      "(1): in (2): on (3): all (4):\n",
      " (5):,\n",
      "-------\n",
      "Seq: Then, Jane and Emily had a lot of fun at the station. Emily gave a computer to Jane\n",
      "Target: Emily\n",
      "(1): in (2): on (3):\n",
      " (4): all (5):,\n",
      "-------\n",
      "Seq: After Ruby and River went to the station, Ruby gave a bone to River\n",
      "Target: River\n",
      "(1):, (2):\n",
      " (3): and (4): [ (5):.\n",
      "-------\n",
      "Seq: After Ruby and River went to the station, Ruby gave a bone to River\n",
      "Target: Ruby\n",
      "(1): a (2):\n",
      " (3): the (4): in (5):S\n",
      "-------\n",
      "Seq: After Ruby and River went to the station, Ruby gave a bone to River\n",
      "Target: Ruby\n",
      "(1):\n",
      " (2): a (3):S (4):L (5): A\n",
      "-------\n",
      "Seq: When Crystal and Frank got a necklace at the store, Crystal decided to give it to Frank\n",
      "Target: Frank\n",
      "(1):sub (2): Ded (3):[' (4):s (5):aze\n",
      "-------\n",
      "Seq: When Crystal and Frank got a necklace at the store, Crystal decided to give it to Frank\n",
      "Target: Crystal\n",
      "(1): on (2): at (3): af (4): not (5): so\n",
      "-------\n",
      "Seq: When Crystal and Frank got a necklace at the store, Crystal decided to give it to Frank\n",
      "Target: Crystal\n",
      "(1): on (2): not (3): at (4): so (5):sc\n",
      "-------\n",
      "Seq: Then, Anderson and Maria had a lot of fun at the restaurant. Maria gave a ring to Anderson\n",
      "Target: Anderson\n",
      "(1):shots (2): Baltic (3): seg (4):shot (5): attention\n",
      "-------\n",
      "Seq: Then, Anderson and Maria had a lot of fun at the restaurant. Maria gave a ring to Anderson\n",
      "Target: Maria\n",
      "(1): out (2): all (3): the (4):, (5): where\n",
      "-------\n",
      "Seq: Then, Anderson and Maria had a lot of fun at the restaurant. Maria gave a ring to Anderson\n",
      "Target: Maria\n",
      "(1): out (2): all (3): the (4):, (5):\n",
      "\n",
      "-------\n",
      "Seq: Then, Anna and William were working at the garden. William decided to give a kiss to Anna\n",
      "Target: Anna\n",
      "(1): all (2): the (3):, (4): and (5): either\n",
      "-------\n",
      "Seq: Then, Anna and William were working at the garden. William decided to give a kiss to Anna\n",
      "Target: William\n",
      "(1):, (2):\n",
      " (3):: (4):- (5):.\n",
      "-------\n",
      "Seq: Then, Anna and William were working at the garden. William decided to give a kiss to Anna\n",
      "Target: William\n",
      "(1):, (2):\n",
      " (3):: (4):. (5):-\n",
      "-------\n",
      "Seq: Then, Jay and Dean had a long argument, and afterwards Jay said to Dean\n",
      "Target: Dean\n",
      "(1): dis (2): on (3): ter (4): first (5): sub\n",
      "-------\n",
      "Seq: Then, Jay and Dean had a long argument, and afterwards Jay said to Dean\n",
      "Target: Jay\n",
      "(1): the (2):\n",
      " (3): a (4): D (5):less\n",
      "-------\n",
      "Seq: Then, Jay and Dean had a long argument, and afterwards Jay said to Dean\n",
      "Target: Jay\n",
      "(1):\n",
      " (2): the (3): a (4): D (5): am\n",
      "-------\n",
      "Seq: Then, Jeremy and Anna were thinking about going to the office. Jeremy wanted to give a computer to Anna\n",
      "Target: Anna\n",
      "(1): all (2): the (3):, (4): with (5): and\n",
      "-------\n",
      "Seq: Then, Jeremy and Anna were thinking about going to the office. Jeremy wanted to give a computer to Anna\n",
      "Target: Jeremy\n",
      "(1):\n",
      " (2): the (3): ( (4): in (5): that\n",
      "-------\n",
      "Seq: Then, Jeremy and Anna were thinking about going to the office. Jeremy wanted to give a computer to Anna\n",
      "Target: Jeremy\n",
      "(1):\n",
      " (2): the (3): in (4): ( (5): de\n",
      "-------\n",
      "Seq: When Roman and Peter got a computer at the school, Roman decided to give it to Peter\n",
      "Target: Peter\n",
      "(1):. (2):, (3): and (4):- (5): the\n",
      "-------\n",
      "Seq: When Roman and Peter got a computer at the school, Roman decided to give it to Peter\n",
      "Target: Roman\n",
      "(1): a (2):\n",
      " (3): on (4): the (5): and\n",
      "-------\n",
      "Seq: When Roman and Peter got a computer at the school, Roman decided to give it to Peter\n",
      "Target: Roman\n",
      "(1): a (2): the (3):\n",
      " (4): on (5): and\n",
      "-------\n",
      "Seq: After Peter and Brian went to the hospital, Peter gave a snack to Brian\n",
      "Target: Brian\n",
      "(1): than (2): hom (3): all (4): of (5): the\n",
      "-------\n",
      "Seq: After Peter and Brian went to the hospital, Peter gave a snack to Brian\n",
      "Target: Peter\n",
      "(1):- (2): the (3): and (4):. (5):,\n",
      "-------\n",
      "Seq: After Peter and Brian went to the hospital, Peter gave a snack to Brian\n",
      "Target: Peter\n",
      "(1):. (2): the (3):- (4):s (5):,\n",
      "-------\n",
      "Seq: Then, Sarah and Andy had a long argument, and afterwards Andy said to Sarah\n",
      "Target: Sarah\n",
      "(1): the (2):\n",
      " (3): or (4): on (5):/\n",
      "-------\n",
      "Seq: Then, Sarah and Andy had a long argument, and afterwards Andy said to Sarah\n",
      "Target: Andy\n",
      "(1): the (2): a (3): and (4): an (5): at\n",
      "-------\n",
      "Seq: Then, Sarah and Andy had a long argument, and afterwards Andy said to Sarah\n",
      "Target: Andy\n",
      "(1): the (2): a (3): and (4): at (5):,\n",
      "-------\n",
      "Seq: After Lisa and Crystal went to the hospital, Crystal gave a bone to Lisa\n",
      "Target: Lisa\n",
      "(1): where (2):INCLUDING (3): out (4): Tal (5): regions\n",
      "-------\n",
      "Seq: After Lisa and Crystal went to the hospital, Crystal gave a bone to Lisa\n",
      "Target: Crystal\n",
      "(1): on (2): not (3): af (4): so (5):sc\n",
      "-------\n",
      "Seq: After Lisa and Crystal went to the hospital, Crystal gave a bone to Lisa\n",
      "Target: Crystal\n",
      "(1): on (2): not (3): so (4):sc (5): at\n",
      "-------\n",
      "Seq: Then, Andrew and Edward had a long argument, and afterwards Edward said to Andrew\n",
      "Target: Andrew\n",
      "(1):. (2):, (3):\n",
      " (4): all (5): on\n",
      "-------\n",
      "Seq: Then, Andrew and Edward had a long argument, and afterwards Edward said to Andrew\n",
      "Target: Edward\n",
      "(1):\n",
      " (2): not (3): C (4):, (5): with\n",
      "-------\n",
      "Seq: Then, Andrew and Edward had a long argument, and afterwards Edward said to Andrew\n",
      "Target: Edward\n",
      "(1):\n",
      " (2): not (3): C (4):, (5): T\n",
      "-------\n",
      "Seq: Then, Louis and Victoria were working at the hospital. Louis decided to give a ring to Victoria\n",
      "Target: Victoria\n",
      "(1):\n",
      " (2): in (3): on (4):s (5):,\n",
      "-------\n",
      "Seq: Then, Louis and Victoria were working at the hospital. Louis decided to give a ring to Victoria\n",
      "Target: Louis\n",
      "(1):\n",
      " (2):St (3): St (4): and (5):�\n",
      "-------\n",
      "Seq: Then, Louis and Victoria were working at the hospital. Louis decided to give a ring to Victoria\n",
      "Target: Louis\n",
      "(1):St (2):\n",
      " (3): St (4): and (5):�\n",
      "-------\n",
      "Seq: Then, Luke and Thomas had a long argument, and afterwards Luke said to Thomas\n",
      "Target: Thomas\n",
      "(1):, (2): and (3):\n",
      " (4):. (5): the\n",
      "-------\n",
      "Seq: Then, Luke and Thomas had a long argument, and afterwards Luke said to Thomas\n",
      "Target: Luke\n",
      "(1):, (2):: (3):. (4): and (5): a\n",
      "-------\n",
      "Seq: Then, Luke and Thomas had a long argument, and afterwards Luke said to Thomas\n",
      "Target: Luke\n",
      "(1):, (2):: (3):. (4): and (5): the\n",
      "-------\n",
      "Seq: When Blake and Austin got a computer at the restaurant, Austin decided to give it to Blake\n",
      "Target: Blake\n",
      "(1):Ath (2): into (3):FI (4): phase (5): side\n",
      "-------\n",
      "Seq: When Blake and Austin got a computer at the restaurant, Austin decided to give it to Blake\n",
      "Target: Austin\n",
      "(1): be (2): that (3):ija (4): is (5): fore\n",
      "-------\n",
      "Seq: When Blake and Austin got a computer at the restaurant, Austin decided to give it to Blake\n",
      "Target: Austin\n",
      "(1): that (2): fore (3):ija (4): be (5): is\n",
      "-------\n",
      "Seq: When Mary and Crew got a basketball at the office, Crew decided to give it to Mary\n",
      "Target: Mary\n",
      "(1): out (2): all (3):GENER (4):. (5):mie\n",
      "-------\n",
      "Seq: When Mary and Crew got a basketball at the office, Crew decided to give it to Mary\n",
      "Target: Crew\n",
      "(1): on (2): at (3): Rag (4): in (5):\n",
      "\n",
      "-------\n",
      "Seq: When Mary and Crew got a basketball at the office, Crew decided to give it to Mary\n",
      "Target: Crew\n",
      "(1): on (2): back (3): in (4):\n",
      " (5): that\n",
      "-------\n",
      "Seq: Then, Emily and Michelle had a long argument, and afterwards Emily said to Michelle\n",
      "Target: Michelle\n",
      "(1): ( (2): ' (3): \" (4):\n",
      " (5): C\n",
      "-------\n",
      "Seq: Then, Emily and Michelle had a long argument, and afterwards Emily said to Michelle\n",
      "Target: Emily\n",
      "(1): in (2): on (3): all (4):\n",
      " (5):,\n",
      "-------\n",
      "Seq: Then, Emily and Michelle had a long argument, and afterwards Emily said to Michelle\n",
      "Target: Emily\n",
      "(1): in (2): on (3): all (4):, (5):\n",
      "\n",
      "-------\n",
      "Seq: Then, Anderson and Anthony were thinking about going to the house. Anthony wanted to give a computer to Anderson\n",
      "Target: Anderson\n",
      "(1):shots (2): Baltic (3): seg (4):shot (5): attention\n",
      "-------\n",
      "Seq: Then, Anderson and Anthony were thinking about going to the house. Anthony wanted to give a computer to Anderson\n",
      "Target: Anthony\n",
      "(1):, (2): off (3):\n",
      " (4):L (5):.\n",
      "-------\n",
      "Seq: Then, Anderson and Anthony were thinking about going to the house. Anthony wanted to give a computer to Anderson\n",
      "Target: Anthony\n",
      "(1):, (2): off (3):\n",
      " (4):L (5):.\n",
      "-------\n",
      "Seq: Then, Aaron and Henry had a lot of fun at the restaurant. Henry gave a computer to Aaron\n",
      "Target: Aaron\n",
      "(1):gt (2): the (3):\n",
      " (4):alk (5):PT\n",
      "-------\n",
      "Seq: Then, Aaron and Henry had a lot of fun at the restaurant. Henry gave a computer to Aaron\n",
      "Target: Henry\n",
      "(1):\n",
      " (2):, (3):. (4): O (5): \n",
      "-------\n",
      "Seq: Then, Aaron and Henry had a lot of fun at the restaurant. Henry gave a computer to Aaron\n",
      "Target: Henry\n",
      "(1):\n",
      " (2):, (3):. (4):  (5): O\n",
      "-------\n",
      "Seq: Then, Russell and Jonathan went to the house. Jonathan gave a ring to Russell\n",
      "Target: Russell\n",
      "(1): [ (2): and (3): or (4):r (5):\n",
      "\n",
      "-------\n",
      "Seq: Then, Russell and Jonathan went to the house. Jonathan gave a ring to Russell\n",
      "Target: Jonathan\n",
      "(1):gt (2):los (3):tn (4):ALL (5):L\n",
      "-------\n",
      "Seq: Then, Russell and Jonathan went to the house. Jonathan gave a ring to Russell\n",
      "Target: Jonathan\n",
      "(1): a (2):L (3): the (4):los (5):gt\n",
      "-------\n",
      "Seq: After Andy and Richard went to the store, Richard gave a necklace to Andy\n",
      "Target: Andy\n",
      "(1): the (2): a (3): and (4):. (5): at\n",
      "-------\n",
      "Seq: After Andy and Richard went to the store, Richard gave a necklace to Andy\n",
      "Target: Richard\n",
      "(1):s (2): \" (3): and (4):\n",
      " (5): or\n",
      "-------\n",
      "Seq: After Andy and Richard went to the store, Richard gave a necklace to Andy\n",
      "Target: Richard\n",
      "(1):s (2):\n",
      " (3): \" (4):. (5): or\n",
      "-------\n",
      "Seq: Then, Charles and Luke were thinking about going to the restaurant. Charles wanted to give a ring to Luke\n",
      "Target: Luke\n",
      "(1):: (2):, (3):. (4):_ (5):;\n",
      "-------\n",
      "Seq: Then, Charles and Luke were thinking about going to the restaurant. Charles wanted to give a ring to Luke\n",
      "Target: Charles\n",
      "(1):sub (2):s (3):alk (4):ivities (5):\n",
      "\n",
      "-------\n",
      "Seq: Then, Charles and Luke were thinking about going to the restaurant. Charles wanted to give a ring to Luke\n",
      "Target: Charles\n",
      "(1):sub (2): arrow (3):s (4):\n",
      " (5): Barr\n",
      "-------\n",
      "Seq: When Jake and Steven got a basketball at the office, Steven decided to give it to Jake\n",
      "Target: Jake\n",
      "(1): the (2):. (3):\n",
      " (4): means (5):as\n",
      "-------\n",
      "Seq: When Jake and Steven got a basketball at the office, Steven decided to give it to Jake\n",
      "Target: Steven\n",
      "(1): ( (2):\n",
      " (3):, (4): and (5): \"\n",
      "-------\n",
      "Seq: When Jake and Steven got a basketball at the office, Steven decided to give it to Jake\n",
      "Target: Steven\n",
      "(1): ( (2):\n",
      " (3): T (4):, (5): \"\n",
      "-------\n",
      "Seq: Then, Peter and Edward had a long argument, and afterwards Peter said to Edward\n",
      "Target: Edward\n",
      "(1):\n",
      " (2): not (3):, (4): of (5): C\n",
      "-------\n",
      "Seq: Then, Peter and Edward had a long argument, and afterwards Peter said to Edward\n",
      "Target: Peter\n",
      "(1):, (2):s (3):. (4): the (5): and\n",
      "-------\n",
      "Seq: Then, Peter and Edward had a long argument, and afterwards Peter said to Edward\n",
      "Target: Peter\n",
      "(1):s (2):, (3):. (4): and (5): the\n",
      "-------\n",
      "Seq: Then, Charlie and Ryan were thinking about going to the restaurant. Charlie wanted to give a bone to Ryan\n",
      "Target: Ryan\n",
      "(1): the (2): and (3):, (4):\n",
      " (5): a\n",
      "-------\n",
      "Seq: Then, Charlie and Ryan were thinking about going to the restaurant. Charlie wanted to give a bone to Ryan\n",
      "Target: Charlie\n",
      "(1):alk (2):rus (3):Mess (4):\n",
      " (5): a\n",
      "-------\n",
      "Seq: Then, Charlie and Ryan were thinking about going to the restaurant. Charlie wanted to give a bone to Ryan\n",
      "Target: Charlie\n",
      "(1):\n",
      " (2):rus (3): a (4): Bes (5): of\n",
      "-------\n",
      "Seq: After Clark and Alex went to the office, Clark gave a drink to Alex\n",
      "Target: Alex\n",
      "(1):less (2): the (3):, (4): a (5): than\n",
      "-------\n",
      "Seq: After Clark and Alex went to the office, Clark gave a drink to Alex\n",
      "Target: Clark\n",
      "(1): and (2):\n",
      " (3):. (4):, (5):_\n",
      "-------\n",
      "Seq: After Clark and Alex went to the office, Clark gave a drink to Alex\n",
      "Target: Clark\n",
      "(1):\n",
      " (2): it (3): the (4): more (5):,\n",
      "-------\n",
      "Seq: Then, Russell and Peter had a lot of fun at the school. Russell gave a computer to Peter\n",
      "Target: Peter\n",
      "(1):s (2):. (3):, (4): and (5): the\n",
      "-------\n",
      "Seq: Then, Russell and Peter had a lot of fun at the school. Russell gave a computer to Peter\n",
      "Target: Russell\n",
      "(1): [ (2): and (3): or (4):r (5):\n",
      "\n",
      "-------\n",
      "Seq: Then, Russell and Peter had a lot of fun at the school. Russell gave a computer to Peter\n",
      "Target: Russell\n",
      "(1): and (2): [ (3):\n",
      " (4): or (5):/\n",
      "-------\n",
      "Seq: Then, Stephen and River went to the house. River gave a ring to Stephen\n",
      "Target: Stephen\n",
      "(1):\n",
      " (2):, (3):. (4): that (5): T\n",
      "-------\n",
      "Seq: Then, Stephen and River went to the house. River gave a ring to Stephen\n",
      "Target: River\n",
      "(1):, (2):\n",
      " (3): and (4): in (5): [\n",
      "-------\n",
      "Seq: Then, Stephen and River went to the house. River gave a ring to Stephen\n",
      "Target: River\n",
      "(1):, (2):\n",
      " (3): [ (4):. (5): and\n",
      "-------\n",
      "Seq: Then, Justin and Daniel were thinking about going to the station. Daniel wanted to give a necklace to Justin\n",
      "Target: Justin\n",
      "(1): of (2):, (3): a (4):s (5): an\n",
      "-------\n",
      "Seq: Then, Justin and Daniel were thinking about going to the station. Daniel wanted to give a necklace to Justin\n",
      "Target: Daniel\n",
      "(1):\n",
      " (2):, (3): of (4): exp (5):—\n",
      "-------\n",
      "Seq: Then, Justin and Daniel were thinking about going to the station. Daniel wanted to give a necklace to Justin\n",
      "Target: Daniel\n",
      "(1):\n",
      " (2):, (3): of (4): exp (5):—\n",
      "-------\n",
      "Seq: When Tyler and Steven got a ring at the school, Steven decided to give it to Tyler\n",
      "Target: Tyler\n",
      "(1): and (2): in (3):\n",
      " (4): at (5): the\n",
      "-------\n",
      "Seq: When Tyler and Steven got a ring at the school, Steven decided to give it to Tyler\n",
      "Target: Steven\n",
      "(1): ( (2):\n",
      " (3): \" (4):, (5): and\n",
      "-------\n",
      "Seq: When Tyler and Steven got a ring at the school, Steven decided to give it to Tyler\n",
      "Target: Steven\n",
      "(1): ( (2):\n",
      " (3): T (4):, (5): \"\n",
      "-------\n",
      "Seq: Then, Jay and Collins had a long argument, and afterwards Jay said to Collins\n",
      "Target: Collins\n",
      "(1):ee (2): as (3): among (4): amongst (5):ed\n",
      "-------\n",
      "Seq: Then, Jay and Collins had a long argument, and afterwards Jay said to Collins\n",
      "Target: Jay\n",
      "(1): the (2):\n",
      " (3): a (4): D (5):less\n",
      "-------\n",
      "Seq: Then, Jay and Collins had a long argument, and afterwards Jay said to Collins\n",
      "Target: Jay\n",
      "(1):\n",
      " (2): the (3): a (4): D (5): am\n",
      "-------\n",
      "Seq: Then, Jack and Eva had a lot of fun at the house. Jack gave a necklace to Eva\n",
      "Target: Eva\n",
      "(1): coun (2):ANI (3): reason (4): the (5):away\n",
      "-------\n",
      "Seq: Then, Jack and Eva had a lot of fun at the house. Jack gave a necklace to Eva\n",
      "Target: Jack\n",
      "(1):\n",
      " (2):L (3): not (4): reasons (5):less\n",
      "-------\n",
      "Seq: Then, Jack and Eva had a lot of fun at the house. Jack gave a necklace to Eva\n",
      "Target: Jack\n",
      "(1):\n",
      " (2):L (3): yet (4):less (5): not\n",
      "-------\n",
      "Seq: Then, Blake and Emily had a lot of fun at the hospital. Emily gave a ring to Blake\n",
      "Target: Blake\n",
      "(1): phase (2): into (3): side (4):FI (5): masking\n",
      "-------\n",
      "Seq: Then, Blake and Emily had a lot of fun at the hospital. Emily gave a ring to Blake\n",
      "Target: Emily\n",
      "(1): in (2): on (3): all (4):\n",
      " (5):,\n",
      "-------\n",
      "Seq: Then, Blake and Emily had a lot of fun at the hospital. Emily gave a ring to Blake\n",
      "Target: Emily\n",
      "(1): in (2): on (3): all (4):\n",
      " (5):,\n",
      "-------\n",
      "Seq: After Martin and John went to the office, John gave a drink to Martin\n",
      "Target: Martin\n",
      "(1):s (2): ME (3): and (4):ME (5): where\n",
      "-------\n",
      "Seq: After Martin and John went to the office, John gave a drink to Martin\n",
      "Target: John\n",
      "(1): ever (2):micro (3): bar (4):\n",
      " (5):-\n",
      "-------\n",
      "Seq: After Martin and John went to the office, John gave a drink to Martin\n",
      "Target: John\n",
      "(1): ever (2): bar (3):\n",
      " (4): most (5):micro\n",
      "-------\n",
      "Seq: Then, Blake and Laura were thinking about going to the school. Blake wanted to give a computer to Laura\n",
      "Target: Laura\n",
      "(1): the (2): ( (3): and (4):, (5): that\n",
      "-------\n",
      "Seq: Then, Blake and Laura were thinking about going to the school. Blake wanted to give a computer to Laura\n",
      "Target: Blake\n",
      "(1): phase (2): into (3): side (4):FI (5): masking\n",
      "-------\n",
      "Seq: Then, Blake and Laura were thinking about going to the school. Blake wanted to give a computer to Laura\n",
      "Target: Blake\n",
      "(1): masking (2): phase (3): side (4):<|endoftext|> (5): yet\n",
      "-------\n",
      "Seq: Then, Robert and John had a lot of fun at the school. John gave a necklace to Robert\n",
      "Target: Robert\n",
      "(1):\n",
      " (2):, (3): the (4): off (5): in\n",
      "-------\n",
      "Seq: Then, Robert and John had a lot of fun at the school. John gave a necklace to Robert\n",
      "Target: John\n",
      "(1):\n",
      " (2): bar (3): ever (4):. (5): most\n",
      "-------\n",
      "Seq: Then, Robert and John had a lot of fun at the school. John gave a necklace to Robert\n",
      "Target: John\n",
      "(1):\n",
      " (2): bar (3): ever (4): most (5):,\n",
      "-------\n",
      "Seq: After Peter and Andrew went to the office, Peter gave a kiss to Andrew\n",
      "Target: Andrew\n",
      "(1):. (2):, (3): all (4):- (5): on\n",
      "-------\n",
      "Seq: After Peter and Andrew went to the office, Peter gave a kiss to Andrew\n",
      "Target: Peter\n",
      "(1):- (2): the (3): and (4):. (5):,\n",
      "-------\n",
      "Seq: After Peter and Andrew went to the office, Peter gave a kiss to Andrew\n",
      "Target: Peter\n",
      "(1):. (2):s (3):, (4): the (5): and\n",
      "-------\n",
      "Seq: Then, Kyle and George went to the house. Kyle gave a necklace to George\n",
      "Target: George\n",
      "(1):ter (2):reo (3): cast (4): Bes (5): partial\n",
      "-------\n",
      "Seq: Then, Kyle and George went to the house. Kyle gave a necklace to George\n",
      "Target: Kyle\n",
      "(1):. (2): the (3):. (4):\n",
      " (5): a\n",
      "-------\n",
      "Seq: Then, Kyle and George went to the house. Kyle gave a necklace to George\n",
      "Target: Kyle\n",
      "(1):. (2): the (3):. (4):\n",
      " (5): at\n",
      "-------\n",
      "Seq: After Frank and River went to the restaurant, Frank gave a basketball to River\n",
      "Target: River\n",
      "(1):, (2):\n",
      " (3): [ (4): and (5): in\n",
      "-------\n",
      "Seq: After Frank and River went to the restaurant, Frank gave a basketball to River\n",
      "Target: Frank\n",
      "(1): Ded (2):[' (3): Bes (4):U (5):s\n",
      "-------\n",
      "Seq: After Frank and River went to the restaurant, Frank gave a basketball to River\n",
      "Target: Frank\n",
      "(1):s (2):sub (3):aze (4):U (5):u\n",
      "-------\n",
      "Seq: After Jamie and Peter went to the restaurant, Jamie gave a computer to Peter\n",
      "Target: Peter\n",
      "(1):. (2):- (3):, (4): and (5): the\n",
      "-------\n",
      "Seq: After Jamie and Peter went to the restaurant, Jamie gave a computer to Peter\n",
      "Target: Jamie\n",
      "(1): than (2):. (3): meaning (4):uuid (5): on\n",
      "-------\n",
      "Seq: After Jamie and Peter went to the restaurant, Jamie gave a computer to Peter\n",
      "Target: Jamie\n",
      "(1): than (2):. (3): meaning (4):- (5):,\n",
      "-------\n",
      "Seq: After Victoria and Luke went to the hospital, Victoria gave a computer to Luke\n",
      "Target: Luke\n",
      "(1):: (2):. (3):, (4):; (5):_\n",
      "-------\n",
      "Seq: After Victoria and Luke went to the hospital, Victoria gave a computer to Luke\n",
      "Target: Victoria\n",
      "(1): on (2): the (3):\n",
      " (4): in (5): be\n",
      "-------\n",
      "Seq: After Victoria and Luke went to the hospital, Victoria gave a computer to Luke\n",
      "Target: Victoria\n",
      "(1):\n",
      " (2): on (3): the (4): not (5): in\n",
      "-------\n",
      "Seq: When Stephen and John got a snack at the office, Stephen decided to give it to John\n",
      "Target: John\n",
      "(1):\n",
      " (2): bar (3): ever (4): than (5):ter\n",
      "-------\n",
      "Seq: When Stephen and John got a snack at the office, Stephen decided to give it to John\n",
      "Target: Stephen\n",
      "(1):\n",
      " (2):. (3):, (4): T (5): on\n",
      "-------\n",
      "Seq: When Stephen and John got a snack at the office, Stephen decided to give it to John\n",
      "Target: Stephen\n",
      "(1):, (2):\n",
      " (3):. (4): that (5): T\n",
      "-------\n",
      "Seq: Then, Prince and Jonathan were working at the station. Prince decided to give a bone to Jonathan\n",
      "Target: Jonathan\n",
      "(1):gt (2):los (3):tn (4): loud (5):gen\n",
      "-------\n",
      "Seq: Then, Prince and Jonathan were working at the station. Prince decided to give a bone to Jonathan\n",
      "Target: Prince\n",
      "(1):\n",
      " (2):. (3):, (4): and (5):sub\n",
      "-------\n",
      "Seq: Then, Prince and Jonathan were working at the station. Prince decided to give a bone to Jonathan\n",
      "Target: Prince\n",
      "(1):\n",
      " (2):. (3):, (4): and (5): all\n",
      "-------\n",
      "Seq: Then, Ford and Anna had a long argument, and afterwards Ford said to Anna\n",
      "Target: Anna\n",
      "(1): all (2): the (3):, (4): with (5): according\n",
      "-------\n",
      "Seq: Then, Ford and Anna had a long argument, and afterwards Ford said to Anna\n",
      "Target: Ford\n",
      "(1): directions (2): partial (3): them (4): direction (5): long\n",
      "-------\n",
      "Seq: Then, Ford and Anna had a long argument, and afterwards Ford said to Anna\n",
      "Target: Ford\n",
      "(1): partial (2): directions (3): them (4): direction (5): long\n",
      "-------\n",
      "Seq: Then, Sarah and Dean were thinking about going to the hospital. Dean wanted to give a basketball to Sarah\n",
      "Target: Sarah\n",
      "(1): the (2):\n",
      " (3): or (4): on (5):/\n",
      "-------\n",
      "Seq: Then, Sarah and Dean were thinking about going to the hospital. Dean wanted to give a basketball to Sarah\n",
      "Target: Dean\n",
      "(1): on (2): that (3): subord (4): dis (5): ter\n",
      "-------\n",
      "Seq: Then, Sarah and Dean were thinking about going to the hospital. Dean wanted to give a basketball to Sarah\n",
      "Target: Dean\n",
      "(1): on (2): that (3): ter (4): dis (5): first\n",
      "-------\n",
      "Seq: Then, Charles and Connor had a lot of fun at the store. Charles gave a basketball to Connor\n",
      "Target: Connor\n",
      "(1):. (2): of (3):, (4): see (5): to\n",
      "-------\n",
      "Seq: Then, Charles and Connor had a lot of fun at the store. Charles gave a basketball to Connor\n",
      "Target: Charles\n",
      "(1):sub (2):s (3):alk (4):ivities (5):\n",
      "\n",
      "-------\n",
      "Seq: Then, Charles and Connor had a lot of fun at the store. Charles gave a basketball to Connor\n",
      "Target: Charles\n",
      "(1):sub (2):s (3):ivities (4):\n",
      " (5): Barr\n",
      "-------\n",
      "Seq: Then, William and Matthew were working at the garden. William decided to give a snack to Matthew\n",
      "Target: Matthew\n",
      "(1):- (2):, (3):: (4): he (5):.\n",
      "-------\n",
      "Seq: Then, William and Matthew were working at the garden. William decided to give a snack to Matthew\n",
      "Target: William\n",
      "(1):, (2):\n",
      " (3):: (4):. (5):-\n",
      "-------\n",
      "Seq: Then, William and Matthew were working at the garden. William decided to give a snack to Matthew\n",
      "Target: William\n",
      "(1):, (2):\n",
      " (3):: (4):. (5):-\n",
      "-------\n",
      "Seq: After Marcus and Michael went to the house, Marcus gave a snack to Michael\n",
      "Target: Michael\n",
      "(1):\n",
      " (2): the (3): ( (4):, (5): all\n",
      "-------\n",
      "Seq: After Marcus and Michael went to the house, Marcus gave a snack to Michael\n",
      "Target: Marcus\n",
      "(1): the (2): and (3): a (4): on (5): in\n",
      "-------\n",
      "Seq: After Marcus and Michael went to the house, Marcus gave a snack to Michael\n",
      "Target: Marcus\n",
      "(1): and (2): the (3): on (4): a (5): in\n",
      "-------\n",
      "Seq: After Charles and Edward went to the store, Charles gave a kiss to Edward\n",
      "Target: Edward\n",
      "(1):\n",
      " (2): not (3): C (4):, (5):.\n",
      "-------\n",
      "Seq: After Charles and Edward went to the store, Charles gave a kiss to Edward\n",
      "Target: Charles\n",
      "(1):sub (2):s (3):\n",
      "\n",
      " (4):aga (5):Partial\n",
      "-------\n",
      "Seq: After Charles and Edward went to the store, Charles gave a kiss to Edward\n",
      "Target: Charles\n",
      "(1):sub (2):Partial (3):s (4): Barr (5): arrow\n",
      "-------\n",
      "Seq: Then, Daniel and Jessica had a lot of fun at the station. Daniel gave a ring to Jessica\n",
      "Target: Jessica\n",
      "(1): history (2): right (3): in (4): meaning (5):c\n",
      "-------\n",
      "Seq: Then, Daniel and Jessica had a lot of fun at the station. Daniel gave a ring to Jessica\n",
      "Target: Daniel\n",
      "(1):\n",
      " (2):, (3): exp (4): on (5): of\n",
      "-------\n",
      "Seq: Then, Daniel and Jessica had a lot of fun at the station. Daniel gave a ring to Jessica\n",
      "Target: Daniel\n",
      "(1):\n",
      " (2):, (3):— (4): exp (5): of\n",
      "-------\n",
      "Seq: Then, Edward and William were working at the station. William decided to give a bone to Edward\n",
      "Target: Edward\n",
      "(1):\n",
      " (2): not (3): C (4):, (5):s\n",
      "-------\n",
      "Seq: Then, Edward and William were working at the station. William decided to give a bone to Edward\n",
      "Target: William\n",
      "(1):, (2):\n",
      " (3):: (4):. (5):-\n",
      "-------\n",
      "Seq: Then, Edward and William were working at the station. William decided to give a bone to Edward\n",
      "Target: William\n",
      "(1):, (2):\n",
      " (3):: (4):. (5):-\n",
      "-------\n",
      "Seq: Then, Peter and Joshua had a lot of fun at the office. Peter gave a snack to Joshua\n",
      "Target: Joshua\n",
      "(1):, (2):\n",
      " (3): C (4): ( (5)::\n",
      "-------\n",
      "Seq: Then, Peter and Joshua had a lot of fun at the office. Peter gave a snack to Joshua\n",
      "Target: Peter\n",
      "(1):, (2):s (3):. (4): the (5): and\n",
      "-------\n",
      "Seq: Then, Peter and Joshua had a lot of fun at the office. Peter gave a snack to Joshua\n",
      "Target: Peter\n",
      "(1):s (2):, (3):. (4): the (5): and\n",
      "-------\n",
      "Seq: Then, Rose and Thomas were thinking about going to the house. Rose wanted to give a snack to Thomas\n",
      "Target: Thomas\n",
      "(1):, (2):\n",
      " (3): and (4):. (5): the\n",
      "-------\n",
      "Seq: Then, Rose and Thomas were thinking about going to the house. Rose wanted to give a snack to Thomas\n",
      "Target: Rose\n",
      "(1):\n",
      " (2):. (3): a (4): on (5): E\n",
      "-------\n",
      "Seq: Then, Rose and Thomas were thinking about going to the house. Rose wanted to give a snack to Thomas\n",
      "Target: Rose\n",
      "(1):. (2):\n",
      " (3): a (4): on (5): E\n",
      "-------\n",
      "Seq: Then, Alex and Anthony had a lot of fun at the school. Anthony gave a ring to Alex\n",
      "Target: Alex\n",
      "(1): single (2): changed (3):Inside (4): modern (5): new\n",
      "-------\n",
      "Seq: Then, Alex and Anthony had a lot of fun at the school. Anthony gave a ring to Alex\n",
      "Target: Anthony\n",
      "(1): second (2):exception (3):illy (4): Exception (5):Exception\n",
      "-------\n",
      "Seq: Then, Alex and Anthony had a lot of fun at the school. Anthony gave a ring to Alex\n",
      "Target: Anthony\n",
      "(1): second (2): Second (3):illy (4):exception (5): Person\n",
      "-------\n",
      "Seq: Then, Connor and Roman were working at the house. Roman decided to give a basketball to Connor\n",
      "Target: Connor\n",
      "(1): straight (2):OY (3):Response (4):Sec (5): second\n",
      "-------\n",
      "Seq: Then, Connor and Roman were working at the house. Roman decided to give a basketball to Connor\n",
      "Target: Roman\n",
      "(1): formal (2): inclusion (3): rigid (4): complic (5): regular\n",
      "-------\n",
      "Seq: Then, Connor and Roman were working at the house. Roman decided to give a basketball to Connor\n",
      "Target: Roman\n",
      "(1): rigid (2): formal (3): regular (4): complic (5): both\n",
      "-------\n",
      "Seq: Then, Eva and Mark were working at the store. Eva decided to give a bone to Mark\n",
      "Target: Mark\n",
      "(1): both (2): final (3): permanent (4): added (5): identical\n",
      "-------\n",
      "Seq: Then, Eva and Mark were working at the store. Eva decided to give a bone to Mark\n",
      "Target: Eva\n",
      "(1): before (2): immediately (3): woman (4): fresh (5): immediate\n",
      "-------\n",
      "Seq: Then, Eva and Mark were working at the store. Eva decided to give a bone to Mark\n",
      "Target: Eva\n",
      "(1): immediately (2): immediate (3):- (4): before (5): woman\n",
      "-------\n",
      "Seq: Then, Ian and Jessica went to the restaurant. Ian gave a basketball to Jessica\n",
      "Target: Jessica\n",
      "(1): and (2):- (3):\n",
      " (4): require (5):.\n",
      "-------\n",
      "Seq: Then, Ian and Jessica went to the restaurant. Ian gave a basketball to Jessica\n",
      "Target: Ian\n",
      "(1): rigid (2):BLOCK (3):, (4): mon (5):.\n",
      "-------\n",
      "Seq: Then, Ian and Jessica went to the restaurant. Ian gave a basketball to Jessica\n",
      "Target: Ian\n",
      "(1): rigid (2):BLOCK (3):, (4): mon (5): isolated\n",
      "-------\n",
      "Seq: Then, Austin and Ian had a lot of fun at the house. Austin gave a kiss to Ian\n",
      "Target: Ian\n",
      "(1): rigid (2):, (3):BLOCK (4): mon (5):.\n",
      "-------\n",
      "Seq: Then, Austin and Ian had a lot of fun at the house. Austin gave a kiss to Ian\n",
      "Target: Austin\n",
      "(1): Blue (2): A (3): Isaac (4): starter (5):expensive\n",
      "-------\n",
      "Seq: Then, Austin and Ian had a lot of fun at the house. Austin gave a kiss to Ian\n",
      "Target: Austin\n",
      "(1): A (2): Isaac (3): Blue (4): starter (5): Man\n",
      "-------\n",
      "Seq: Then, William and Jack had a long argument, and afterwards Jack said to William\n",
      "Target: William\n",
      "(1):\n",
      " (2):, (3):. (4): and (5):-\n",
      "-------\n",
      "Seq: Then, William and Jack had a long argument, and afterwards Jack said to William\n",
      "Target: Jack\n",
      "(1): straight (2): man (3): single (4):, (5): fast\n",
      "-------\n",
      "Seq: Then, William and Jack had a long argument, and afterwards Jack said to William\n",
      "Target: Jack\n",
      "(1): straight (2): single (3): fast (4): man (5): quick\n",
      "-------\n",
      "Seq: Then, Lisa and Joseph went to the house. Lisa gave a kiss to Joseph\n",
      "Target: Joseph\n",
      "(1):\n",
      " (2):, (3):. (4): shy (5): and\n",
      "-------\n",
      "Seq: Then, Lisa and Joseph went to the house. Lisa gave a kiss to Joseph\n",
      "Target: Lisa\n",
      "(1): and (2):. (3):, (4):\n",
      " (5):-\n",
      "-------\n",
      "Seq: Then, Lisa and Joseph went to the house. Lisa gave a kiss to Joseph\n",
      "Target: Lisa\n",
      "(1): and (2):\n",
      " (3):, (4):. (5): et\n",
      "-------\n",
      "Seq: Then, Frank and Victoria were working at the office. Victoria decided to give a basketball to Frank\n",
      "Target: Frank\n",
      "(1):\n",
      " (2): and (3): before (4):, (5):-\n",
      "-------\n",
      "Seq: Then, Frank and Victoria were working at the office. Victoria decided to give a basketball to Frank\n",
      "Target: Victoria\n",
      "(1): expensive (2): impos (3): costly (4): heavy (5):heavy\n",
      "-------\n",
      "Seq: Then, Frank and Victoria were working at the office. Victoria decided to give a basketball to Frank\n",
      "Target: Victoria\n",
      "(1): costly (2): expensive (3): impos (4): heavy (5):heavy\n",
      "-------\n",
      "Seq: Then, Amy and Andrew went to the school. Amy gave a computer to Andrew\n",
      "Target: Andrew\n",
      "(1):- (2): personal (3): individual (4): person (5):.\n",
      "-------\n",
      "Seq: Then, Amy and Andrew went to the school. Amy gave a computer to Andrew\n",
      "Target: Amy\n",
      "(1): second (2):- (3): new (4): Second (5): girl\n",
      "-------\n",
      "Seq: Then, Amy and Andrew went to the school. Amy gave a computer to Andrew\n",
      "Target: Amy\n",
      "(1): second (2):- (3): Second (4): and (5):second\n",
      "-------\n",
      "Seq: After David and Arthur went to the hospital, Arthur gave a snack to David\n",
      "Target: David\n",
      "(1): summ (2): celebrity (3): after (4): Red (5):derived\n",
      "-------\n",
      "Seq: After David and Arthur went to the hospital, Arthur gave a snack to David\n",
      "Target: Arthur\n",
      "(1): full (2): higher (3):重 (4): high (5):high\n",
      "-------\n",
      "Seq: After David and Arthur went to the hospital, Arthur gave a snack to David\n",
      "Target: Arthur\n",
      "(1): full (2): higher (3): broad (4): universal (5):universal\n",
      "-------\n",
      "Seq: Then, Jake and Victoria had a long argument, and afterwards Jake said to Victoria\n",
      "Target: Victoria\n",
      "(1): expensive (2): impos (3): costly (4): equal (5):dual\n",
      "-------\n",
      "Seq: Then, Jake and Victoria had a long argument, and afterwards Jake said to Victoria\n",
      "Target: Jake\n",
      "(1): dynamic (2): high (3): fast (4):, (5):\n",
      "\n",
      "-------\n",
      "Seq: Then, Jake and Victoria had a long argument, and afterwards Jake said to Victoria\n",
      "Target: Jake\n",
      "(1): dynamic (2): high (3):\n",
      " (4):, (5):high\n",
      "-------\n",
      "Seq: When Eric and Peter got a bone at the garden, Eric decided to give it to Peter\n",
      "Target: Peter\n",
      "(1):ADD (2): higher (3): add (4): high (5): added\n",
      "-------\n",
      "Seq: When Eric and Peter got a bone at the garden, Eric decided to give it to Peter\n",
      "Target: Eric\n",
      "(1): exclusive (2): straight (3): exclusively (4): limited (5): narrow\n",
      "-------\n",
      "Seq: When Eric and Peter got a bone at the garden, Eric decided to give it to Peter\n",
      "Target: Eric\n",
      "(1): exclusive (2): exclusively (3): limited (4): restricted (5): narrow\n",
      "-------\n",
      "Seq: Then, Ruby and Edward were working at the station. Edward decided to give a drink to Ruby\n",
      "Target: Ruby\n",
      "(1): Under (2): human (3):. (4):\n",
      " (5): under\n",
      "-------\n",
      "Seq: Then, Ruby and Edward were working at the station. Edward decided to give a drink to Ruby\n",
      "Target: Edward\n",
      "(1): costly (2): expensive (3): further (4):. (5): require\n",
      "-------\n",
      "Seq: Then, Ruby and Edward were working at the station. Edward decided to give a drink to Ruby\n",
      "Target: Edward\n",
      "(1): costly (2): expensive (3):. (4): further (5): formal\n",
      "-------\n",
      "Seq: When Michelle and Richard got a necklace at the school, Richard decided to give it to Michelle\n",
      "Target: Michelle\n",
      "(1): second (2): doubled (3):second (4): immediate (5): twin\n",
      "-------\n",
      "Seq: When Michelle and Richard got a necklace at the school, Richard decided to give it to Michelle\n",
      "Target: Richard\n",
      "(1): rigid (2): refined (3): expensive (4): strict (5): rational\n",
      "-------\n",
      "Seq: When Michelle and Richard got a necklace at the school, Richard decided to give it to Michelle\n",
      "Target: Richard\n",
      "(1): rigid (2): expensive (3): strict (4): formal (5): refined\n",
      "-------\n",
      "Seq: Then, Sean and Simon were thinking about going to the school. Simon wanted to give a ring to Sean\n",
      "Target: Sean\n",
      "(1): limited (2): segmented (3): requiring (4):limited (5): exclusive\n",
      "-------\n",
      "Seq: Then, Sean and Simon were thinking about going to the school. Simon wanted to give a ring to Sean\n",
      "Target: Simon\n",
      "(1): bright (2): themselves (3): prince (4): awake (5): +\n",
      "-------\n",
      "Seq: Then, Sean and Simon were thinking about going to the school. Simon wanted to give a ring to Sean\n",
      "Target: Simon\n",
      "(1): él (2): beyond (3): bright (4): themselves (5): leader\n",
      "-------\n",
      "Seq: Then, Jay and Graham had a lot of fun at the garden. Jay gave a basketball to Graham\n",
      "Target: Graham\n",
      "(1): general (2): common (3): short (4):Fund (5): both\n",
      "-------\n",
      "Seq: Then, Jay and Graham had a lot of fun at the garden. Jay gave a basketball to Graham\n",
      "Target: Jay\n",
      "(1): intellectual (2):. (3):, (4): high (5):-\n",
      "-------\n",
      "Seq: Then, Jay and Graham had a lot of fun at the garden. Jay gave a basketball to Graham\n",
      "Target: Jay\n",
      "(1): intellectual (2):. (3):, (4): Ph (5):-\n",
      "-------\n",
      "Seq: Then, Graham and Roman were thinking about going to the station. Graham wanted to give a drink to Roman\n",
      "Target: Roman\n",
      "(1): formal (2): rigid (3): regular (4): both (5): politician\n",
      "-------\n",
      "Seq: Then, Graham and Roman were thinking about going to the station. Graham wanted to give a drink to Roman\n",
      "Target: Graham\n",
      "(1): general (2): common (3): both (4):Base (5): mainstream\n",
      "-------\n",
      "Seq: Then, Graham and Roman were thinking about going to the station. Graham wanted to give a drink to Roman\n",
      "Target: Graham\n",
      "(1): common (2): general (3): short (4): both (5): without\n",
      "-------\n",
      "Seq: Then, Charles and Joseph had a long argument, and afterwards Charles said to Joseph\n",
      "Target: Joseph\n",
      "(1):, (2):\n",
      " (3):. (4): and (5): not\n",
      "-------\n",
      "Seq: Then, Charles and Joseph had a long argument, and afterwards Charles said to Joseph\n",
      "Target: Charles\n",
      "(1): main (2): major (3): and (4):Main (5): central\n",
      "-------\n",
      "Seq: Then, Charles and Joseph had a long argument, and afterwards Charles said to Joseph\n",
      "Target: Charles\n",
      "(1): main (2): major (3): and (4): heavy (5):heavy\n",
      "-------\n",
      "Seq: Then, Lisa and Madison went to the house. Lisa gave a drink to Madison\n",
      "Target: Madison\n",
      "(1): afterwards (2): after (3): artific (4): refinement (5): second\n",
      "-------\n",
      "Seq: Then, Lisa and Madison went to the house. Lisa gave a drink to Madison\n",
      "Target: Lisa\n",
      "(1): and (2):. (3):, (4):\n",
      " (5):-\n",
      "-------\n",
      "Seq: Then, Lisa and Madison went to the house. Lisa gave a drink to Madison\n",
      "Target: Lisa\n",
      "(1): and (2):, (3):\n",
      " (4):. (5): et\n",
      "-------\n",
      "Seq: Then, Daniel and Carter had a long argument, and afterwards Carter said to Daniel\n",
      "Target: Daniel\n",
      "(1): resolutions (2): post (3): elim (4): reduction (5):-\n",
      "-------\n",
      "Seq: Then, Daniel and Carter had a long argument, and afterwards Carter said to Daniel\n",
      "Target: Carter\n",
      "(1):. (2):, (3): demanding (4): blood (5): to\n",
      "-------\n",
      "Seq: Then, Daniel and Carter had a long argument, and afterwards Carter said to Daniel\n",
      "Target: Carter\n",
      "(1):. (2):, (3): blood (4): and (5): demanding\n",
      "-------\n",
      "Seq: Then, Blake and Anderson had a lot of fun at the restaurant. Blake gave a drink to Anderson\n",
      "Target: Anderson\n",
      "(1): and (2):metal (3):- (4): straight (5): awake\n",
      "-------\n",
      "Seq: Then, Blake and Anderson had a lot of fun at the restaurant. Blake gave a drink to Anderson\n",
      "Target: Blake\n",
      "(1): vigorous (2): high (3):high (4): rugged (5): brute\n",
      "-------\n",
      "Seq: Then, Blake and Anderson had a lot of fun at the restaurant. Blake gave a drink to Anderson\n",
      "Target: Blake\n",
      "(1): vigorous (2): rugged (3): brute (4): rough (5): crude\n",
      "-------\n",
      "Seq: Then, Jane and Andrew were thinking about going to the station. Jane wanted to give a computer to Andrew\n",
      "Target: Andrew\n",
      "(1):- (2):. (3):, (4): ( (5): and\n",
      "-------\n",
      "Seq: Then, Jane and Andrew were thinking about going to the station. Jane wanted to give a computer to Andrew\n",
      "Target: Jane\n",
      "(1):. (2):, (3): and (4):- (5):\n",
      "\n",
      "-------\n",
      "Seq: Then, Jane and Andrew were thinking about going to the station. Jane wanted to give a computer to Andrew\n",
      "Target: Jane\n",
      "(1):. (2):, (3):\n",
      " (4): requiring (5): and\n",
      "-------\n",
      "Seq: Then, Adam and Rachel were working at the garden. Rachel decided to give a computer to Adam\n",
      "Target: Adam\n",
      "(1): before (2): first (3): and (4): mand (5): immediately\n",
      "-------\n",
      "Seq: Then, Adam and Rachel were working at the garden. Rachel decided to give a computer to Adam\n",
      "Target: Rachel\n",
      "(1): Red (2): import (3): reverse (4): caring (5): reversed\n",
      "-------\n",
      "Seq: Then, Adam and Rachel were working at the garden. Rachel decided to give a computer to Adam\n",
      "Target: Rachel\n",
      "(1): Red (2): Ret (3): import (4): Med (5): labeled\n",
      "-------\n",
      "Seq: Then, Ian and Morgan had a lot of fun at the hospital. Ian gave a drink to Morgan\n",
      "Target: Morgan\n",
      "(1): sil (2): fully (3): proper (4): later (5): princ\n",
      "-------\n",
      "Seq: Then, Ian and Morgan had a lot of fun at the hospital. Ian gave a drink to Morgan\n",
      "Target: Ian\n",
      "(1): rigid (2):BLOCK (3):, (4): mon (5):.\n",
      "-------\n",
      "Seq: Then, Ian and Morgan had a lot of fun at the hospital. Ian gave a drink to Morgan\n",
      "Target: Ian\n",
      "(1): rigid (2):BLOCK (3): mon (4):, (5): individual\n",
      "-------\n",
      "Seq: Then, Joseph and Aaron went to the hospital. Aaron gave a bone to Joseph\n",
      "Target: Joseph\n",
      "(1):\n",
      " (2):, (3):. (4): ( (5): and\n",
      "-------\n",
      "Seq: Then, Joseph and Aaron went to the hospital. Aaron gave a bone to Joseph\n",
      "Target: Aaron\n",
      "(1): shock (2): star (3):illion (4):shock (5): fat\n",
      "-------\n",
      "Seq: Then, Joseph and Aaron went to the hospital. Aaron gave a bone to Joseph\n",
      "Target: Aaron\n",
      "(1): star (2): shock (3):shock (4):illion (5): totally\n",
      "-------\n",
      "Seq: Then, Austin and Michael were working at the school. Michael decided to give a bone to Austin\n",
      "Target: Austin\n",
      "(1): Blue (2): A (3): Isaac (4): starter (5):expensive\n",
      "-------\n",
      "Seq: Then, Austin and Michael were working at the school. Michael decided to give a bone to Austin\n",
      "Target: Michael\n",
      "(1): straight (2): next (3): Iv (4): second (5):expensive\n",
      "-------\n",
      "Seq: Then, Austin and Michael were working at the school. Michael decided to give a bone to Austin\n",
      "Target: Michael\n",
      "(1): straight (2): next (3): Iv (4):achine (5):IV\n",
      "-------\n",
      "Seq: Then, Scott and Christian had a long argument, and afterwards Christian said to Scott\n",
      "Target: Scott\n",
      "(1): stay (2): male (3): stays (4): stayed (5): and\n",
      "-------\n",
      "Seq: Then, Blake and Alan were working at the hospital. Blake decided to give a basketball to Alan\n",
      "Target: Alan\n",
      "(1):mon (2): mon (3): intellectual (4): Gent (5): linearly\n",
      "-------\n",
      "Seq: Then, Blake and Alan were working at the hospital. Blake decided to give a basketball to Alan\n",
      "Target: Blake\n",
      "(1): vigorous (2): high (3):high (4): rugged (5): brute\n",
      "-------\n",
      "Seq: Then, Blake and Alan were working at the hospital. Blake decided to give a basketball to Alan\n",
      "Target: Blake\n",
      "(1): rugged (2): vigorous (3): strong (4): brute (5): massive\n",
      "-------\n",
      "Seq: Then, Dean and Marco had a lot of fun at the garden. Marco gave a snack to Dean\n",
      "Target: Dean\n",
      "(1):. (2): intellectual (3): intellect (4): best (5): shift\n",
      "-------\n",
      "Seq: Then, Dean and Marco had a lot of fun at the garden. Marco gave a snack to Dean\n",
      "Target: Marco\n",
      "(1): second (2):second (3): multi (4): halfway (5): moderately\n",
      "-------\n",
      "Seq: Then, Dean and Marco had a lot of fun at the garden. Marco gave a snack to Dean\n",
      "Target: Marco\n",
      "(1): second (2):second (3): multi (4): twice (5): multiple\n",
      "-------\n",
      "Seq: Then, Andrew and Scott were thinking about going to the restaurant. Scott wanted to give a bone to Andrew\n",
      "Target: Andrew\n",
      "(1):- (2): new (3):new (4): individual (5):.\n",
      "-------\n",
      "Seq: Then, Andrew and Scott were thinking about going to the restaurant. Scott wanted to give a bone to Andrew\n",
      "Target: Scott\n",
      "(1): Mr (2): male (3): and (4):, (5):.\n",
      "-------\n",
      "Seq: Then, Andrew and Scott were thinking about going to the restaurant. Scott wanted to give a bone to Andrew\n",
      "Target: Scott\n",
      "(1): stay (2): male (3): and (4): stays (5): Mr\n",
      "-------\n",
      "Seq: Then, Luke and Justin had a long argument, and afterwards Luke said to Justin\n",
      "Target: Justin\n",
      "(1): decreasing (2):- (3): only (4): after (5):,\n",
      "-------\n",
      "Seq: Then, Luke and Justin had a long argument, and afterwards Luke said to Justin\n",
      "Target: Luke\n",
      "(1):children (2): kids (3): kid (4): shy (5): children\n",
      "-------\n",
      "Seq: Then, Luke and Justin had a long argument, and afterwards Luke said to Justin\n",
      "Target: Luke\n",
      "(1):children (2): kids (3): shy (4): kid (5): Part\n",
      "-------\n",
      "Seq: When Sarah and Louis got a computer at the school, Sarah decided to give it to Louis\n",
      "Target: Louis\n",
      "(1): second (2): Second (3): not (4):second (5): later\n",
      "-------\n",
      "Seq: When Sarah and Louis got a computer at the school, Sarah decided to give it to Louis\n",
      "Target: Sarah\n",
      "(1):ine (2): First (3): first (4):   (5): costly\n",
      "-------\n",
      "Seq: When Sarah and Louis got a computer at the school, Sarah decided to give it to Louis\n",
      "Target: Sarah\n",
      "(1):\n",
      " (2):   (3):ine (4): and (5): et\n",
      "-------\n",
      "Seq: After Patrick and Peter went to the station, Patrick gave a snack to Peter\n",
      "Target: Peter\n",
      "(1):ADD (2): high (3): higher (4): Par (5): expensive\n",
      "-------\n",
      "Seq: After Patrick and Peter went to the station, Patrick gave a snack to Peter\n",
      "Target: Patrick\n",
      "(1): intellectual (2): superhero (3):rection (4): beforehand (5):expensive\n",
      "-------\n",
      "Seq: After Patrick and Peter went to the station, Patrick gave a snack to Peter\n",
      "Target: Patrick\n",
      "(1): beforehand (2): intellectual (3): alert (4): superhero (5):Sem\n",
      "-------\n",
      "Seq: After Edward and Daniel went to the office, Daniel gave a ring to Edward\n",
      "Target: Edward\n",
      "(1): expensive (2): costly (3): further (4): upgraded (5):expensive\n",
      "-------\n",
      "Seq: After Edward and Daniel went to the office, Daniel gave a ring to Edward\n",
      "Target: Daniel\n",
      "(1): resolutions (2): Major (3):named (4): named (5): splits\n",
      "-------\n",
      "Seq: After Edward and Daniel went to the office, Daniel gave a ring to Edward\n",
      "Target: Daniel\n",
      "(1): resolutions (2): post (3):- (4): segmented (5):named\n",
      "-------\n",
      "Seq: Then, David and Luke were thinking about going to the school. David wanted to give a ring to Luke\n",
      "Target: Luke\n",
      "(1):children (2): kids (3): children (4): kid (5):Children\n",
      "-------\n",
      "Seq: Then, David and Luke were thinking about going to the school. David wanted to give a ring to Luke\n",
      "Target: David\n",
      "(1): after (2): answers (3): and (4):- (5): Red\n",
      "-------\n",
      "Seq: Then, David and Luke were thinking about going to the school. David wanted to give a ring to Luke\n",
      "Target: David\n",
      "(1): after (2): Red (3): answers (4): and (5):.\n",
      "-------\n",
      "Seq: After Robert and Michael went to the school, Michael gave a kiss to Robert\n",
      "Target: Robert\n",
      "(1):, (2):\n",
      " (3):. (4): and (5): Mr\n",
      "-------\n",
      "Seq: After Robert and Michael went to the school, Michael gave a kiss to Robert\n",
      "Target: Michael\n",
      "(1): Iv (2): next (3):next (4):expensive (5):IV\n",
      "-------\n",
      "Seq: After Robert and Michael went to the school, Michael gave a kiss to Robert\n",
      "Target: Michael\n",
      "(1): next (2):next (3): Iv (4): straight (5):achine\n",
      "-------\n",
      "Seq: Then, Henry and Andy had a lot of fun at the school. Andy gave a computer to Henry\n",
      "Target: Henry\n",
      "(1): second (2):Title (3): titles (4):Headers (5):second\n",
      "-------\n",
      "Seq: Then, Henry and Andy had a lot of fun at the school. Andy gave a computer to Henry\n",
      "Target: Andy\n",
      "(1):high (2): high (3): after (4):- (5): hi\n",
      "-------\n",
      "Seq: Then, Henry and Andy had a lot of fun at the school. Andy gave a computer to Henry\n",
      "Target: Andy\n",
      "(1):high (2): after (3): individual (4):- (5): high\n",
      "-------\n",
      "Seq: Then, Crystal and Kelly had a long argument, and afterwards Crystal said to Kelly\n",
      "Target: Kelly\n",
      "(1): mon (2):mon (3): luxury (4): my (5): player\n",
      "-------\n",
      "Seq: Then, Crystal and Kelly had a long argument, and afterwards Crystal said to Kelly\n",
      "Target: Crystal\n",
      "(1):\n",
      " (2):x (3):, (4):. (5):N\n",
      "-------\n",
      "Seq: Then, Crystal and Kelly had a long argument, and afterwards Crystal said to Kelly\n",
      "Target: Crystal\n",
      "(1):\n",
      " (2):x (3):, (4):. (5): and\n",
      "-------\n",
      "Seq: When Justin and Stephen got a basketball at the house, Justin decided to give it to Stephen\n",
      "Target: Stephen\n",
      "(1): second (2):second (3):. (4):expensive (5): son\n",
      "-------\n",
      "Seq: When Justin and Stephen got a basketball at the house, Justin decided to give it to Stephen\n",
      "Target: Justin\n",
      "(1): close (2):- (3): only (4): youngest (5): not\n",
      "-------\n",
      "Seq: When Justin and Stephen got a basketball at the house, Justin decided to give it to Stephen\n",
      "Target: Justin\n",
      "(1):- (2):, (3): and (4): close (5):.\n",
      "-------\n",
      "Seq: After Arthur and Paul went to the office, Paul gave a drink to Arthur\n",
      "Target: Arthur\n",
      "(1): full (2): higher (3):Full (4):full (5): Full\n",
      "-------\n",
      "Seq: After Arthur and Paul went to the office, Paul gave a drink to Arthur\n",
      "Target: Paul\n",
      "(1): twice (2): intellectual (3): twin (4):. (5): second\n",
      "-------\n",
      "Seq: After Arthur and Paul went to the office, Paul gave a drink to Arthur\n",
      "Target: Paul\n",
      "(1): twin (2): twice (3): second (4):2 (5):.\n",
      "-------\n",
      "Seq: After Ruby and Joshua went to the garden, Ruby gave a ring to Joshua\n",
      "Target: Joshua\n",
      "(1): straight (2): masculine (3): highest (4):nikov (5):high\n",
      "-------\n",
      "Seq: After Ruby and Joshua went to the garden, Ruby gave a ring to Joshua\n",
      "Target: Ruby\n",
      "(1): human (2): personal (3): Human (4):human (5): Under\n",
      "-------\n",
      "Seq: After Ruby and Joshua went to the garden, Ruby gave a ring to Joshua\n",
      "Target: Ruby\n",
      "(1): human (2): Under (3):. (4):\n",
      " (5): Human\n",
      "-------\n",
      "Seq: After Ian and Ruby went to the garden, Ian gave a basketball to Ruby\n",
      "Target: Ruby\n",
      "(1): human (2):. (3): Under (4):\n",
      " (5):Load\n",
      "-------\n",
      "Seq: After Ian and Ruby went to the garden, Ian gave a basketball to Ruby\n",
      "Target: Ian\n",
      "(1): rigid (2):BLOCK (3): requiring (4): mon (5): straight\n",
      "-------\n",
      "Seq: After Ian and Ruby went to the garden, Ian gave a basketball to Ruby\n",
      "Target: Ian\n",
      "(1): rigid (2):, (3):BLOCK (4): mon (5):.\n",
      "-------\n",
      "Seq: When Lisa and Collins got a drink at the restaurant, Lisa decided to give it to Collins\n",
      "Target: Collins\n",
      "(1): addition (2): added (3): additive (4): adding (5):adding\n",
      "-------\n",
      "Seq: When Lisa and Collins got a drink at the restaurant, Lisa decided to give it to Collins\n",
      "Target: Lisa\n",
      "(1): requiring (2): shy (3): his (4): first (5): require\n",
      "-------\n",
      "Seq: When Lisa and Collins got a drink at the restaurant, Lisa decided to give it to Collins\n",
      "Target: Lisa\n",
      "(1): and (2):\n",
      " (3):, (4): et (5):.\n",
      "-------\n",
      "Seq: Then, Eva and Joshua went to the garden. Joshua gave a snack to Eva\n",
      "Target: Eva\n",
      "(1): before (2): immediately (3): woman (4): fresh (5): immediate\n",
      "-------\n",
      "Seq: Then, Eva and Joshua went to the garden. Joshua gave a snack to Eva\n",
      "Target: Joshua\n",
      "(1): straight (2):high (3): high (4):nikov (5): highest\n",
      "-------\n",
      "Seq: Then, Eva and Joshua went to the garden. Joshua gave a snack to Eva\n",
      "Target: Joshua\n",
      "(1): straight (2):nikov (3): masculine (4):CLOCK (5): fully\n",
      "-------\n",
      "Seq: Then, Arthur and Luke had a lot of fun at the store. Arthur gave a kiss to Luke\n",
      "Target: Luke\n",
      "(1):children (2): shy (3): kids (4): kid (5): children\n",
      "-------\n",
      "Seq: Then, Arthur and Luke had a lot of fun at the store. Arthur gave a kiss to Luke\n",
      "Target: Arthur\n",
      "(1): full (2): universal (3):universal (4): broad (5):Full\n",
      "-------\n",
      "Seq: Then, Arthur and Luke had a lot of fun at the store. Arthur gave a kiss to Luke\n",
      "Target: Arthur\n",
      "(1): full (2): universal (3):universal (4):Full (5): higher\n",
      "-------\n",
      "Seq: Then, Dean and William had a lot of fun at the house. Dean gave a snack to William\n",
      "Target: William\n",
      "(1):\n",
      " (2):. (3):, (4): and (5):-\n",
      "-------\n",
      "Seq: Then, Dean and William had a lot of fun at the house. Dean gave a snack to William\n",
      "Target: Dean\n",
      "(1):. (2): intellectual (3): intellect (4): best (5): shift\n",
      "-------\n",
      "Seq: Then, Dean and William had a lot of fun at the house. Dean gave a snack to William\n",
      "Target: Dean\n",
      "(1):. (2): best (3): hierarchical (4): Rec (5): intellectual\n",
      "-------\n",
      "Seq: Then, Jay and Crystal were working at the restaurant. Jay decided to give a drink to Crystal\n",
      "Target: Crystal\n",
      "(1):\n",
      " (2):, (3):. (4): and (5):x\n",
      "-------\n",
      "Seq: Then, Jay and Crystal were working at the restaurant. Jay decided to give a drink to Crystal\n",
      "Target: Jay\n",
      "(1): intellectual (2):. (3):, (4): high (5):-\n",
      "-------\n",
      "Seq: Then, Jay and Crystal were working at the restaurant. Jay decided to give a drink to Crystal\n",
      "Target: Jay\n",
      "(1): intellectual (2): Ph (3):. (4):, (5): M\n",
      "-------\n",
      "Seq: Then, Jane and Emily had a lot of fun at the station. Emily gave a computer to Jane\n",
      "Target: Jane\n",
      "(1):. (2):, (3): and (4):- (5):\n",
      "\n",
      "-------\n",
      "Seq: Then, Jane and Emily had a lot of fun at the station. Emily gave a computer to Jane\n",
      "Target: Emily\n",
      "(1):- (2):, (3): un (4): and (5):.\n",
      "-------\n",
      "Seq: Then, Jane and Emily had a lot of fun at the station. Emily gave a computer to Jane\n",
      "Target: Emily\n",
      "(1):- (2): un (3): and (4): woman (5):,\n",
      "-------\n",
      "Seq: After Ruby and River went to the station, Ruby gave a bone to River\n",
      "Target: River\n",
      "(1): direct (2):\n",
      " (3): and (4): real (5): outdoors\n",
      "-------\n",
      "Seq: After Ruby and River went to the station, Ruby gave a bone to River\n",
      "Target: Ruby\n",
      "(1): human (2): personal (3): Human (4):human (5): Under\n",
      "-------\n",
      "Seq: After Ruby and River went to the station, Ruby gave a bone to River\n",
      "Target: Ruby\n",
      "(1): human (2): Under (3):. (4):\n",
      " (5):Load\n",
      "-------\n",
      "Seq: When Crystal and Frank got a necklace at the store, Crystal decided to give it to Frank\n",
      "Target: Frank\n",
      "(1):\n",
      " (2): and (3): before (4):, (5):free\n",
      "-------\n",
      "Seq: When Crystal and Frank got a necklace at the store, Crystal decided to give it to Frank\n",
      "Target: Crystal\n",
      "(1):aryn (2): replaced (3): equality (4):x (5):\n",
      "\n",
      "-------\n",
      "Seq: When Crystal and Frank got a necklace at the store, Crystal decided to give it to Frank\n",
      "Target: Crystal\n",
      "(1):\n",
      " (2):, (3):. (4):x (5): and\n",
      "-------\n",
      "Seq: Then, Anderson and Maria had a lot of fun at the restaurant. Maria gave a ring to Anderson\n",
      "Target: Anderson\n",
      "(1):metal (2): and (3): real (4): actual (5): first\n",
      "-------\n",
      "Seq: Then, Anderson and Maria had a lot of fun at the restaurant. Maria gave a ring to Anderson\n",
      "Target: Maria\n",
      "(1): elim (2): internal (3): women (4): subdu (5): collapsed\n",
      "-------\n",
      "Seq: Then, Anderson and Maria had a lot of fun at the restaurant. Maria gave a ring to Anderson\n",
      "Target: Maria\n",
      "(1): elim (2): women (3): eliminated (4):collapse (5): internal\n",
      "-------\n",
      "Seq: Then, Anna and William were working at the garden. William decided to give a kiss to Anna\n",
      "Target: Anna\n",
      "(1): softer (2): lady (3): soft (4):LEN (5): addition\n",
      "-------\n",
      "Seq: Then, Anna and William were working at the garden. William decided to give a kiss to Anna\n",
      "Target: William\n",
      "(1):\n",
      " (2):, (3): and (4):. (5):-\n",
      "-------\n",
      "Seq: Then, Anna and William were working at the garden. William decided to give a kiss to Anna\n",
      "Target: William\n",
      "(1):\n",
      " (2):, (3):. (4): and (5):-\n",
      "-------\n",
      "Seq: Then, Jay and Dean had a long argument, and afterwards Jay said to Dean\n",
      "Target: Dean\n",
      "(1): shift (2):Rec (3): Rec (4): elev (5): higher\n",
      "-------\n",
      "Seq: Then, Jay and Dean had a long argument, and afterwards Jay said to Dean\n",
      "Target: Jay\n",
      "(1): intellectual (2):. (3):, (4): high (5):-\n",
      "-------\n",
      "Seq: Then, Jay and Dean had a long argument, and afterwards Jay said to Dean\n",
      "Target: Jay\n",
      "(1): intellectual (2):. (3):, (4): Ph (5): high\n",
      "-------\n",
      "Seq: Then, Jeremy and Anna were thinking about going to the office. Jeremy wanted to give a computer to Anna\n",
      "Target: Anna\n",
      "(1): softer (2):LEN (3): soft (4): lady (5): addition\n",
      "-------\n",
      "Seq: Then, Jeremy and Anna were thinking about going to the office. Jeremy wanted to give a computer to Anna\n",
      "Target: Jeremy\n",
      "(1): single (2): person (3):. (4): direct (5): individual\n",
      "-------\n",
      "Seq: Then, Jeremy and Anna were thinking about going to the office. Jeremy wanted to give a computer to Anna\n",
      "Target: Jeremy\n",
      "(1): person (2):. (3): single (4): individual (5):,\n",
      "-------\n",
      "Seq: When Roman and Peter got a computer at the school, Roman decided to give it to Peter\n",
      "Target: Peter\n",
      "(1):ADD (2): next (3): él (4): fanc (5): Par\n",
      "-------\n",
      "Seq: When Roman and Peter got a computer at the school, Roman decided to give it to Peter\n",
      "Target: Roman\n",
      "(1): rigid (2): politician (3): inclusion (4): fixed (5): homogen\n",
      "-------\n",
      "Seq: When Roman and Peter got a computer at the school, Roman decided to give it to Peter\n",
      "Target: Roman\n",
      "(1): regular (2): formal (3): inclusion (4): complic (5): included\n",
      "-------\n",
      "Seq: After Peter and Brian went to the hospital, Peter gave a snack to Brian\n",
      "Target: Brian\n",
      "(1): to (2): and (3): Super (4): under (5):\n",
      "\n",
      "-------\n",
      "Seq: After Peter and Brian went to the hospital, Peter gave a snack to Brian\n",
      "Target: Peter\n",
      "(1): higher (2):ADD (3): elite (4): high (5): expensive\n",
      "-------\n",
      "Seq: After Peter and Brian went to the hospital, Peter gave a snack to Brian\n",
      "Target: Peter\n",
      "(1):ADD (2): added (3): higher (4): add (5): adding\n",
      "-------\n",
      "Seq: Then, Sarah and Andy had a long argument, and afterwards Andy said to Sarah\n",
      "Target: Sarah\n",
      "(1):- (2): and (3): under (4):\n",
      " (5): the\n",
      "-------\n",
      "Seq: Then, Sarah and Andy had a long argument, and afterwards Andy said to Sarah\n",
      "Target: Andy\n",
      "(1):high (2): after (3): high (4): individual (5):-\n",
      "-------\n",
      "Seq: Then, Sarah and Andy had a long argument, and afterwards Andy said to Sarah\n",
      "Target: Andy\n",
      "(1):high (2): after (3):- (4): high (5): individual\n",
      "-------\n",
      "Seq: After Lisa and Crystal went to the hospital, Crystal gave a bone to Lisa\n",
      "Target: Lisa\n",
      "(1): impos (2): requiring (3):   (4): his (5): +\n",
      "-------\n",
      "Seq: After Lisa and Crystal went to the hospital, Crystal gave a bone to Lisa\n",
      "Target: Crystal\n",
      "(1):\n",
      " (2):x (3): again (4):. (5):,\n",
      "-------\n",
      "Seq: After Lisa and Crystal went to the hospital, Crystal gave a bone to Lisa\n",
      "Target: Crystal\n",
      "(1):\n",
      " (2):x (3):. (4):, (5): and\n",
      "-------\n",
      "Seq: Then, Andrew and Edward had a long argument, and afterwards Edward said to Andrew\n",
      "Target: Andrew\n",
      "(1):- (2): new (3):new (4): individual (5):.\n",
      "-------\n",
      "Seq: Then, Andrew and Edward had a long argument, and afterwards Edward said to Andrew\n",
      "Target: Edward\n",
      "(1): costly (2): expensive (3): further (4):. (5): formal\n",
      "-------\n",
      "Seq: Then, Andrew and Edward had a long argument, and afterwards Edward said to Andrew\n",
      "Target: Edward\n",
      "(1): costly (2): expensive (3): further (4):. (5): require\n",
      "-------\n",
      "Seq: Then, Louis and Victoria were working at the hospital. Louis decided to give a ring to Victoria\n",
      "Target: Victoria\n",
      "(1): impos (2): heavy (3): expensive (4):heavy (5): costly\n",
      "-------\n",
      "Seq: Then, Louis and Victoria were working at the hospital. Louis decided to give a ring to Victoria\n",
      "Target: Louis\n",
      "(1): second (2):, (3):. (4): Second (5): later\n",
      "-------\n",
      "Seq: Then, Louis and Victoria were working at the hospital. Louis decided to give a ring to Victoria\n",
      "Target: Louis\n",
      "(1): second (2): Second (3):, (4):. (5): not\n",
      "-------\n",
      "Seq: Then, Luke and Thomas had a long argument, and afterwards Luke said to Thomas\n",
      "Target: Thomas\n",
      "(1): and (2): ( (3): shy (4):, (5): about\n",
      "-------\n",
      "Seq: Then, Luke and Thomas had a long argument, and afterwards Luke said to Thomas\n",
      "Target: Luke\n",
      "(1):children (2): kids (3): kid (4): shy (5): children\n",
      "-------\n",
      "Seq: Then, Luke and Thomas had a long argument, and afterwards Luke said to Thomas\n",
      "Target: Luke\n",
      "(1):children (2): kids (3): kid (4): shy (5): Part\n",
      "-------\n",
      "Seq: When Blake and Austin got a computer at the restaurant, Austin decided to give it to Blake\n",
      "Target: Blake\n",
      "(1): vigorous (2): brute (3): high (4):high (5): rough\n",
      "-------\n",
      "Seq: When Blake and Austin got a computer at the restaurant, Austin decided to give it to Blake\n",
      "Target: Austin\n",
      "(1):expensive (2): Isaac (3): Blue (4): Man (5): second\n",
      "-------\n",
      "Seq: When Blake and Austin got a computer at the restaurant, Austin decided to give it to Blake\n",
      "Target: Austin\n",
      "(1): Blue (2): A (3): starter (4): Isaac (5): Man\n",
      "-------\n",
      "Seq: When Mary and Crew got a basketball at the office, Crew decided to give it to Mary\n",
      "Target: Mary\n",
      "(1):oon (2): singer (3): subjective (4): woman (5): women\n",
      "-------\n",
      "Seq: When Mary and Crew got a basketball at the office, Crew decided to give it to Mary\n",
      "Target: Crew\n",
      "(1): complete (2):complete (3): full (4):full (5): fully\n",
      "-------\n",
      "Seq: When Mary and Crew got a basketball at the office, Crew decided to give it to Mary\n",
      "Target: Crew\n",
      "(1): complete (2):complete (3): full (4):full (5): fully\n",
      "-------\n",
      "Seq: Then, Emily and Michelle had a long argument, and afterwards Emily said to Michelle\n",
      "Target: Michelle\n",
      "(1): second (2):second (3): Second (4): twin (5): dual\n",
      "-------\n",
      "Seq: Then, Emily and Michelle had a long argument, and afterwards Emily said to Michelle\n",
      "Target: Emily\n",
      "(1):- (2): and (3):, (4):. (5): woman\n",
      "-------\n",
      "Seq: Then, Emily and Michelle had a long argument, and afterwards Emily said to Michelle\n",
      "Target: Emily\n",
      "(1):- (2): and (3):, (4): woman (5):.\n",
      "-------\n",
      "Seq: Then, Anderson and Anthony were thinking about going to the house. Anthony wanted to give a computer to Anderson\n",
      "Target: Anderson\n",
      "(1):metal (2): and (3): real (4): actual (5): first\n",
      "-------\n",
      "Seq: Then, Anderson and Anthony were thinking about going to the house. Anthony wanted to give a computer to Anderson\n",
      "Target: Anthony\n",
      "(1): second (2): Second (3):illy (4):Exception (5): Exception\n",
      "-------\n",
      "Seq: Then, Anderson and Anthony were thinking about going to the house. Anthony wanted to give a computer to Anderson\n",
      "Target: Anthony\n",
      "(1): second (2): Second (3):illy (4):Exception (5): Person\n",
      "-------\n",
      "Seq: Then, Aaron and Henry had a lot of fun at the restaurant. Henry gave a computer to Aaron\n",
      "Target: Aaron\n",
      "(1): shock (2): star (3):shock (4):illion (5):engeance\n",
      "-------\n",
      "Seq: Then, Aaron and Henry had a lot of fun at the restaurant. Henry gave a computer to Aaron\n",
      "Target: Henry\n",
      "(1): second (2):second (3):Title (4): titles (5): successive\n",
      "-------\n",
      "Seq: Then, Aaron and Henry had a lot of fun at the restaurant. Henry gave a computer to Aaron\n",
      "Target: Henry\n",
      "(1):Title (2): headers (3):長 (4):Headers (5): honour\n",
      "-------\n",
      "Seq: Then, Russell and Jonathan went to the house. Jonathan gave a ring to Russell\n",
      "Target: Russell\n",
      "(1): Rec (2): travel (3): requiring (4):Runner (5):Rec\n",
      "-------\n",
      "Seq: Then, Russell and Jonathan went to the house. Jonathan gave a ring to Russell\n",
      "Target: Jonathan\n",
      "(1):first (2): msgid (3): gentleman (4): handsome (5): commute\n",
      "-------\n",
      "Seq: Then, Russell and Jonathan went to the house. Jonathan gave a ring to Russell\n",
      "Target: Jonathan\n",
      "(1):first (2): commute (3):quick (4):SHORT (5):CLOCK\n",
      "-------\n",
      "Seq: After Andy and Richard went to the store, Richard gave a necklace to Andy\n",
      "Target: Andy\n",
      "(1): after (2):high (3): personal (4): hi (5): individual\n",
      "-------\n",
      "Seq: After Andy and Richard went to the store, Richard gave a necklace to Andy\n",
      "Target: Richard\n",
      "(1): refined (2): rigid (3): rational (4): formal (5): intellectual\n",
      "-------\n",
      "Seq: After Andy and Richard went to the store, Richard gave a necklace to Andy\n",
      "Target: Richard\n",
      "(1): rigid (2): strict (3): formal (4): refined (5): expensive\n",
      "-------\n",
      "Seq: Then, Charles and Luke were thinking about going to the restaurant. Charles wanted to give a ring to Luke\n",
      "Target: Luke\n",
      "(1):children (2): kids (3): shy (4): kid (5): children\n",
      "-------\n",
      "Seq: Then, Charles and Luke were thinking about going to the restaurant. Charles wanted to give a ring to Luke\n",
      "Target: Charles\n",
      "(1): main (2): major (3): and (4):Main (5): central\n",
      "-------\n",
      "Seq: Then, Charles and Luke were thinking about going to the restaurant. Charles wanted to give a ring to Luke\n",
      "Target: Charles\n",
      "(1): sil (2): main (3): and (4):\n",
      " (5):,\n",
      "-------\n",
      "Seq: When Jake and Steven got a basketball at the office, Steven decided to give it to Jake\n",
      "Target: Jake\n",
      "(1): dynamic (2): momenta (3): moving (4): party (5): forward\n",
      "-------\n",
      "Seq: When Jake and Steven got a basketball at the office, Steven decided to give it to Jake\n",
      "Target: Steven\n",
      "(1):expensive (2): intellectual (3): intellect (4): male (5): expensive\n",
      "-------\n",
      "Seq: When Jake and Steven got a basketball at the office, Steven decided to give it to Jake\n",
      "Target: Steven\n",
      "(1):expensive (2): intellectual (3): expensive (4): intellect (5): intellectuals\n",
      "-------\n",
      "Seq: Then, Peter and Edward had a long argument, and afterwards Peter said to Edward\n",
      "Target: Edward\n",
      "(1): expensive (2): costly (3): further (4): require (5): formal\n",
      "-------\n",
      "Seq: Then, Peter and Edward had a long argument, and afterwards Peter said to Edward\n",
      "Target: Peter\n",
      "(1):ADD (2): Par (3): high (4): fanc (5): expensive\n",
      "-------\n",
      "Seq: Then, Peter and Edward had a long argument, and afterwards Peter said to Edward\n",
      "Target: Peter\n",
      "(1):ADD (2): higher (3): high (4): next (5): fanc\n",
      "-------\n",
      "Seq: Then, Charlie and Ryan were thinking about going to the restaurant. Charlie wanted to give a bone to Ryan\n",
      "Target: Ryan\n",
      "(1):warn (2): shy (3): named (4): from (5): imported\n",
      "-------\n",
      "Seq: Then, Charlie and Ryan were thinking about going to the restaurant. Charlie wanted to give a bone to Ryan\n",
      "Target: Charlie\n",
      "(1): and (2): silver (3):, (4): Silver (5): between\n",
      "-------\n",
      "Seq: Then, Charlie and Ryan were thinking about going to the restaurant. Charlie wanted to give a bone to Ryan\n",
      "Target: Charlie\n",
      "(1): and (2): Silver (3):, (4): bin (5):aut\n",
      "-------\n",
      "Seq: After Clark and Alex went to the office, Clark gave a drink to Alex\n",
      "Target: Alex\n",
      "(1): single (2):Inside (3):, (4): intellectual (5): Silver\n",
      "-------\n",
      "Seq: After Clark and Alex went to the office, Clark gave a drink to Alex\n",
      "Target: Clark\n",
      "(1): intellect (2): boss (3): refined (4): excell (5): supervisor\n",
      "-------\n",
      "Seq: After Clark and Alex went to the office, Clark gave a drink to Alex\n",
      "Target: Clark\n",
      "(1): boss (2): high (3): stiffness (4): intellect (5): expensive\n",
      "-------\n",
      "Seq: Then, Russell and Peter had a lot of fun at the school. Russell gave a computer to Peter\n",
      "Target: Peter\n",
      "(1):ADD (2): Par (3): individual (4): higher (5): expensive\n",
      "-------\n",
      "Seq: Then, Russell and Peter had a lot of fun at the school. Russell gave a computer to Peter\n",
      "Target: Russell\n",
      "(1): Rec (2): travel (3): requiring (4):Runner (5):Rec\n",
      "-------\n",
      "Seq: Then, Russell and Peter had a lot of fun at the school. Russell gave a computer to Peter\n",
      "Target: Russell\n",
      "(1): Rec (2): travel (3): requiring (4):Runner (5):ator\n",
      "-------\n",
      "Seq: Then, Stephen and River went to the house. River gave a ring to Stephen\n",
      "Target: Stephen\n",
      "(1):. (2):, (3):- (4):\n",
      " (5)::\n",
      "-------\n",
      "Seq: Then, Stephen and River went to the house. River gave a ring to Stephen\n",
      "Target: River\n",
      "(1): direct (2):\n",
      " (3): and (4): directly (5):.\n",
      "-------\n",
      "Seq: Then, Stephen and River went to the house. River gave a ring to Stephen\n",
      "Target: River\n",
      "(1):\n",
      " (2): direct (3): water (4): and (5):.\n",
      "-------\n",
      "Seq: Then, Justin and Daniel were thinking about going to the station. Daniel wanted to give a necklace to Justin\n",
      "Target: Justin\n",
      "(1):- (2): only (3):, (4): decreasing (5): close\n",
      "-------\n",
      "Seq: Then, Justin and Daniel were thinking about going to the station. Daniel wanted to give a necklace to Justin\n",
      "Target: Daniel\n",
      "(1): post (2): after (3): resolutions (4): decom (5):named\n",
      "-------\n",
      "Seq: Then, Justin and Daniel were thinking about going to the station. Daniel wanted to give a necklace to Justin\n",
      "Target: Daniel\n",
      "(1): post (2): decom (3): resolutions (4):- (5): named\n",
      "-------\n",
      "Seq: When Tyler and Steven got a ring at the school, Steven decided to give it to Tyler\n",
      "Target: Tyler\n",
      "(1): shift (2):transition (3): Millenn (4): moving (5): transitions\n",
      "-------\n",
      "Seq: When Tyler and Steven got a ring at the school, Steven decided to give it to Tyler\n",
      "Target: Steven\n",
      "(1):expensive (2): intellectual (3): expensive (4): intellect (5): male\n",
      "-------\n",
      "Seq: When Tyler and Steven got a ring at the school, Steven decided to give it to Tyler\n",
      "Target: Steven\n",
      "(1):expensive (2): intellectual (3): expensive (4): intellect (5): intellectuals\n",
      "-------\n",
      "Seq: Then, Jay and Collins had a long argument, and afterwards Jay said to Collins\n",
      "Target: Collins\n",
      "(1):dual (2): dual (3): addition (4): additive (5): added\n",
      "-------\n",
      "Seq: Then, Jay and Collins had a long argument, and afterwards Jay said to Collins\n",
      "Target: Jay\n",
      "(1): intellectual (2):. (3):, (4): high (5):-\n",
      "-------\n",
      "Seq: Then, Jay and Collins had a long argument, and afterwards Jay said to Collins\n",
      "Target: Jay\n",
      "(1): intellectual (2):. (3):, (4): Ph (5): high\n",
      "-------\n",
      "Seq: Then, Jack and Eva had a lot of fun at the house. Jack gave a necklace to Eva\n",
      "Target: Eva\n",
      "(1): woman (2): before (3): reverse (4):- (5): up\n",
      "-------\n",
      "Seq: Then, Jack and Eva had a lot of fun at the house. Jack gave a necklace to Eva\n",
      "Target: Jack\n",
      "(1): single (2): fast (3): straight (4): man (5): quick\n",
      "-------\n",
      "Seq: Then, Jack and Eva had a lot of fun at the house. Jack gave a necklace to Eva\n",
      "Target: Jack\n",
      "(1): straight (2): fast (3): single (4): quick (5): man\n",
      "-------\n",
      "Seq: Then, Blake and Emily had a lot of fun at the hospital. Emily gave a ring to Blake\n",
      "Target: Blake\n",
      "(1): vigorous (2): high (3):high (4): rugged (5): brute\n",
      "-------\n",
      "Seq: Then, Blake and Emily had a lot of fun at the hospital. Emily gave a ring to Blake\n",
      "Target: Emily\n",
      "(1):- (2): woman (3): and (4):, (5): un\n",
      "-------\n",
      "Seq: Then, Blake and Emily had a lot of fun at the hospital. Emily gave a ring to Blake\n",
      "Target: Emily\n",
      "(1):- (2): and (3): woman (4):, (5): women\n",
      "-------\n",
      "Seq: After Martin and John went to the office, John gave a drink to Martin\n",
      "Target: Martin\n",
      "(1): general (2): mortal (3): personal (4): individual (5): and\n",
      "-------\n",
      "Seq: After Martin and John went to the office, John gave a drink to Martin\n",
      "Target: John\n",
      "(1):\n",
      " (2):. (3):, (4): and (5):-\n",
      "-------\n",
      "Seq: After Martin and John went to the office, John gave a drink to Martin\n",
      "Target: John\n",
      "(1):\n",
      " (2):. (3):, (4):- (5): and\n",
      "-------\n",
      "Seq: Then, Blake and Laura were thinking about going to the school. Blake wanted to give a computer to Laura\n",
      "Target: Laura\n",
      "(1): softer (2): gentle (3): et (4): soft (5):oses\n",
      "-------\n",
      "Seq: Then, Blake and Laura were thinking about going to the school. Blake wanted to give a computer to Laura\n",
      "Target: Blake\n",
      "(1): vigorous (2): high (3):high (4): rugged (5): brute\n",
      "-------\n",
      "Seq: Then, Blake and Laura were thinking about going to the school. Blake wanted to give a computer to Laura\n",
      "Target: Blake\n",
      "(1): rugged (2): vigorous (3): brute (4): rough (5): high\n",
      "-------\n",
      "Seq: Then, Robert and John had a lot of fun at the school. John gave a necklace to Robert\n",
      "Target: Robert\n",
      "(1):, (2):. (3):\n",
      " (4): and (5): (\n",
      "-------\n",
      "Seq: Then, Robert and John had a lot of fun at the school. John gave a necklace to Robert\n",
      "Target: John\n",
      "(1):\n",
      " (2):. (3):, (4): and (5):-\n",
      "-------\n",
      "Seq: Then, Robert and John had a lot of fun at the school. John gave a necklace to Robert\n",
      "Target: John\n",
      "(1):\n",
      " (2):. (3):, (4): and (5):-\n",
      "-------\n",
      "Seq: After Peter and Andrew went to the office, Peter gave a kiss to Andrew\n",
      "Target: Andrew\n",
      "(1):- (2): personal (3): individual (4): for (5): Freeman\n",
      "-------\n",
      "Seq: After Peter and Andrew went to the office, Peter gave a kiss to Andrew\n",
      "Target: Peter\n",
      "(1): higher (2):ADD (3): elite (4): high (5): expensive\n",
      "-------\n",
      "Seq: After Peter and Andrew went to the office, Peter gave a kiss to Andrew\n",
      "Target: Peter\n",
      "(1):ADD (2): high (3): expensive (4): higher (5): added\n",
      "-------\n",
      "Seq: Then, Kyle and George went to the house. Kyle gave a necklace to George\n",
      "Target: George\n",
      "(1): expansion (2): Major (3):ood (4):Named (5):ister\n",
      "-------\n",
      "Seq: Then, Kyle and George went to the house. Kyle gave a necklace to George\n",
      "Target: Kyle\n",
      "(1):Partition (2):LIM (3): split (4): gradually (5): intellectual\n",
      "-------\n",
      "Seq: Then, Kyle and George went to the house. Kyle gave a necklace to George\n",
      "Target: Kyle\n",
      "(1):Partition (2): split (3):LIM (4):length (5):Limit\n",
      "-------\n",
      "Seq: After Frank and River went to the restaurant, Frank gave a basketball to River\n",
      "Target: River\n",
      "(1): direct (2):\n",
      " (3): and (4): directly (5): real\n",
      "-------\n",
      "Seq: After Frank and River went to the restaurant, Frank gave a basketball to River\n",
      "Target: Frank\n",
      "(1):\n",
      " (2): direct (3): straight (4): and (5): before\n",
      "-------\n",
      "Seq: After Frank and River went to the restaurant, Frank gave a basketball to River\n",
      "Target: Frank\n",
      "(1):\n",
      " (2): and (3):, (4): before (5):-\n",
      "-------\n",
      "Seq: After Jamie and Peter went to the restaurant, Jamie gave a computer to Peter\n",
      "Target: Peter\n",
      "(1): individual (2):ADD (3): refined (4): higher (5): finer\n",
      "-------\n",
      "Seq: After Jamie and Peter went to the restaurant, Jamie gave a computer to Peter\n",
      "Target: Jamie\n",
      "(1):second (2): second (3): segmented (4): sequentially (5): twice\n",
      "-------\n",
      "Seq: After Jamie and Peter went to the restaurant, Jamie gave a computer to Peter\n",
      "Target: Jamie\n",
      "(1): segmented (2): split (3):Split (4): second (5):split\n",
      "-------\n",
      "Seq: After Victoria and Luke went to the hospital, Victoria gave a computer to Luke\n",
      "Target: Luke\n",
      "(1):children (2): kid (3): satell (4): kids (5): Part\n",
      "-------\n",
      "Seq: After Victoria and Luke went to the hospital, Victoria gave a computer to Luke\n",
      "Target: Victoria\n",
      "(1): highest (2): upgraded (3): impos (4): expensive (5): costly\n",
      "-------\n",
      "Seq: After Victoria and Luke went to the hospital, Victoria gave a computer to Luke\n",
      "Target: Victoria\n",
      "(1): expensive (2): costly (3): impos (4): heavy (5):dual\n",
      "-------\n",
      "Seq: When Stephen and John got a snack at the office, Stephen decided to give it to John\n",
      "Target: John\n",
      "(1):\n",
      " (2):. (3):, (4): and (5):-\n",
      "-------\n",
      "Seq: When Stephen and John got a snack at the office, Stephen decided to give it to John\n",
      "Target: Stephen\n",
      "(1):first (2): first (3):expensive (4):. (5):Method\n",
      "-------\n",
      "Seq: When Stephen and John got a snack at the office, Stephen decided to give it to John\n",
      "Target: Stephen\n",
      "(1):. (2):M (3):\n",
      " (4): formal (5):-\n",
      "-------\n",
      "Seq: Then, Prince and Jonathan were working at the station. Prince decided to give a bone to Jonathan\n",
      "Target: Jonathan\n",
      "(1):quick (2):SHORT (3): commute (4): rapid (5): msgid\n",
      "-------\n",
      "Seq: Then, Prince and Jonathan were working at the station. Prince decided to give a bone to Jonathan\n",
      "Target: Prince\n",
      "(1):next (2): next (3): future (4):Next (5): movement\n",
      "-------\n",
      "Seq: Then, Prince and Jonathan were working at the station. Prince decided to give a bone to Jonathan\n",
      "Target: Prince\n",
      "(1):hard (2):next (3): future (4): next (5):Nob\n",
      "-------\n",
      "Seq: Then, Ford and Anna had a long argument, and afterwards Ford said to Anna\n",
      "Target: Anna\n",
      "(1): lady (2): soft (3): hesit (4):LEN (5): softer\n",
      "-------\n",
      "Seq: Then, Ford and Anna had a long argument, and afterwards Ford said to Anna\n",
      "Target: Ford\n",
      "(1): man (2): rigid (3): Mr (4): gone (5): under\n",
      "-------\n",
      "Seq: Then, Ford and Anna had a long argument, and afterwards Ford said to Anna\n",
      "Target: Ford\n",
      "(1): man (2): Mr (3): gone (4): rigid (5): straight\n",
      "-------\n",
      "Seq: Then, Sarah and Dean were thinking about going to the hospital. Dean wanted to give a basketball to Sarah\n",
      "Target: Sarah\n",
      "(1):- (2): and (3): under (4):\n",
      " (5): the\n",
      "-------\n",
      "Seq: Then, Sarah and Dean were thinking about going to the hospital. Dean wanted to give a basketball to Sarah\n",
      "Target: Dean\n",
      "(1):. (2): requiring (3): higher (4): elev (5): best\n",
      "-------\n",
      "Seq: Then, Sarah and Dean were thinking about going to the hospital. Dean wanted to give a basketball to Sarah\n",
      "Target: Dean\n",
      "(1):. (2): requiring (3): hierarchical (4): best (5): Rec\n",
      "-------\n",
      "Seq: Then, Charles and Connor had a lot of fun at the store. Charles gave a basketball to Connor\n",
      "Target: Connor\n",
      "(1): straight (2): mach (3): sec (4):OY (5): second\n",
      "-------\n",
      "Seq: Then, Charles and Connor had a lot of fun at the store. Charles gave a basketball to Connor\n",
      "Target: Charles\n",
      "(1): main (2): major (3): and (4):Main (5): central\n",
      "-------\n",
      "Seq: Then, Charles and Connor had a lot of fun at the store. Charles gave a basketball to Connor\n",
      "Target: Charles\n",
      "(1): main (2): and (3): sil (4): major (5): central\n",
      "-------\n",
      "Seq: Then, William and Matthew were working at the garden. William decided to give a snack to Matthew\n",
      "Target: Matthew\n",
      "(1): next (2): prolong (3): Sequ (4): prolonged (5): Mr\n",
      "-------\n",
      "Seq: Then, William and Matthew were working at the garden. William decided to give a snack to Matthew\n",
      "Target: William\n",
      "(1):\n",
      " (2):, (3):. (4): and (5):-\n",
      "-------\n",
      "Seq: Then, William and Matthew were working at the garden. William decided to give a snack to Matthew\n",
      "Target: William\n",
      "(1):\n",
      " (2):, (3):. (4): and (5):-\n",
      "-------\n",
      "Seq: After Marcus and Michael went to the house, Marcus gave a snack to Michael\n",
      "Target: Michael\n",
      "(1): straight (2): Iv (3): duo (4): next (5):achine\n",
      "-------\n",
      "Seq: After Marcus and Michael went to the house, Marcus gave a snack to Michael\n",
      "Target: Marcus\n",
      "(1): terminating (2): afterwards (3): fibrobl (4): terminate (5): termination\n",
      "-------\n",
      "Seq: After Marcus and Michael went to the house, Marcus gave a snack to Michael\n",
      "Target: Marcus\n",
      "(1): ladder (2): next (3): afterwards (4):pow (5): cumbersome\n",
      "-------\n",
      "Seq: After Charles and Edward went to the store, Charles gave a kiss to Edward\n",
      "Target: Edward\n",
      "(1): expensive (2): costly (3): further (4):expensive (5):.\n",
      "-------\n",
      "Seq: After Charles and Edward went to the store, Charles gave a kiss to Edward\n",
      "Target: Charles\n",
      "(1): heavy (2): over (3): massive (4): major (5): Heavy\n",
      "-------\n",
      "Seq: After Charles and Edward went to the store, Charles gave a kiss to Edward\n",
      "Target: Charles\n",
      "(1): main (2): major (3): and (4): central (5): heavy\n",
      "-------\n",
      "Seq: Then, Daniel and Jessica had a lot of fun at the station. Daniel gave a ring to Jessica\n",
      "Target: Jessica\n",
      "(1):- (2): and (3): second (4):\n",
      " (5):.\n",
      "-------\n",
      "Seq: Then, Daniel and Jessica had a lot of fun at the station. Daniel gave a ring to Jessica\n",
      "Target: Daniel\n",
      "(1): resolutions (2): post (3): elim (4): reduction (5):-\n",
      "-------\n",
      "Seq: Then, Daniel and Jessica had a lot of fun at the station. Daniel gave a ring to Jessica\n",
      "Target: Daniel\n",
      "(1): resolutions (2): decom (3): split (4): post (5):-\n",
      "-------\n",
      "Seq: Then, Edward and William were working at the station. William decided to give a bone to Edward\n",
      "Target: Edward\n",
      "(1): costly (2): expensive (3):. (4): further (5): formal\n",
      "-------\n",
      "Seq: Then, Edward and William were working at the station. William decided to give a bone to Edward\n",
      "Target: William\n",
      "(1):\n",
      " (2):, (3):. (4): and (5):-\n",
      "-------\n",
      "Seq: Then, Edward and William were working at the station. William decided to give a bone to Edward\n",
      "Target: William\n",
      "(1):\n",
      " (2):, (3):. (4): and (5):-\n",
      "-------\n",
      "Seq: Then, Peter and Joshua had a lot of fun at the office. Peter gave a snack to Joshua\n",
      "Target: Joshua\n",
      "(1): straight (2): fully (3): high (4):high (5):nikov\n",
      "-------\n",
      "Seq: Then, Peter and Joshua had a lot of fun at the office. Peter gave a snack to Joshua\n",
      "Target: Peter\n",
      "(1):ADD (2): Par (3): high (4): fanc (5): expensive\n",
      "-------\n",
      "Seq: Then, Peter and Joshua had a lot of fun at the office. Peter gave a snack to Joshua\n",
      "Target: Peter\n",
      "(1):ADD (2): high (3): expensive (4): higher (5):high\n",
      "-------\n",
      "Seq: Then, Rose and Thomas were thinking about going to the house. Rose wanted to give a snack to Thomas\n",
      "Target: Thomas\n",
      "(1): and (2): ( (3):, (4):- (5): still\n",
      "-------\n",
      "Seq: Then, Rose and Thomas were thinking about going to the house. Rose wanted to give a snack to Thomas\n",
      "Target: Rose\n",
      "(1):\n",
      " (2): reverse (3): up (4): high (5): Red\n",
      "-------\n",
      "Seq: Then, Rose and Thomas were thinking about going to the house. Rose wanted to give a snack to Thomas\n",
      "Target: Rose\n",
      "(1):\n",
      " (2): women (3): Red (4): woman (5): high\n",
      "Copy circuit for head 12.2 (sign=1) : Top 5 accuracy: 0.6666666666666667%\n"
     ]
    }
   ],
   "source": [
    "list_of_heads = [(9, 4), (12, 2)]\n",
    "copy_scores = compute_copy_score(model, list_of_heads, ioi_dataset=ioi_dataset, batch_size=12, verbose=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After computing copy scores - Memory Allocated: 10.61 GB, Memory Reserved: 26.36 GB\n"
     ]
    }
   ],
   "source": [
    "print_gpu_memory_usage(\"After computing copy scores\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from utils.head_metrics import BatchIOIDataset, collate_fn\n",
    "# ioi_dataset.__class__ = BatchIOIDataset\n",
    "# ioi_dataloader = DataLoader(ioi_dataset, batch_size=20, collate_fn=partial(collate_fn, device=model.cfg.device))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch 0 - Memory Allocated: 10.61 GB, Memory Reserved: 24.87 GB\n",
      "Batch 1 - Memory Allocated: 10.61 GB, Memory Reserved: 24.87 GB\n"
     ]
    }
   ],
   "source": [
    "from transformer_lens import ActivationCache\n",
    "from typing import List\n",
    "\n",
    "def concatenate_activation_caches(caches: List[ActivationCache]) -> ActivationCache:\n",
    "    # input to ActivationCache should be a dictionary where the keys are individual keys of\n",
    "    # caches[0] and the values are concatenated across all batches via the first dimension\n",
    "    return ActivationCache({k: torch.cat([c[k] for c in caches], dim=0) for k in caches[0].keys()}, model.cfg.model_name, True)\n",
    "\n",
    "def run_with_cache_batched(model: HookedTransformer, input_ids: torch.Tensor, batch_size: int = 20):\n",
    "    results = []\n",
    "    caches = []\n",
    "    total_size = input_ids.shape[0]\n",
    "    for i in range(0, total_size, batch_size):\n",
    "        # Adjust batch end index to avoid going out of bounds\n",
    "        batch_input = input_ids[i:min(i+batch_size, total_size)]\n",
    "        batch_logits, batch_cache = model.run_with_cache(batch_input)\n",
    "        # send to CPU\n",
    "        batch_logits = batch_logits.cpu()\n",
    "        batch_cache = batch_cache.to('cpu')\n",
    "        print_gpu_memory_usage(f\"Batch {i//batch_size}\")\n",
    "        results.append(batch_logits)\n",
    "        caches.append(batch_cache)\n",
    "    return torch.cat(results, dim=0), concatenate_activation_caches(caches)\n",
    "\n",
    "logits_from_batched, cache_from_batched = run_with_cache_batched(model, ioi_dataset.toks.long(), batch_size=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total size: 23.15 GB\n"
     ]
    },
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n",
      "\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n",
      "\u001b[1;31mClick <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. \n",
      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
     ]
    }
   ],
   "source": [
    "running_size = 0\n",
    "for k in cache_from_batched.keys():\n",
    "    #print(f\"Key: {k}, Size: {tensor_memory_size(cache_from_batched[k]) / (1024 ** 3):.2f} GB\")\n",
    "    #print(tensor_memory_size(cache_from_batched[k]))\n",
    "    running_size += tensor_memory_size(cache_from_batched[k])\n",
    "    #print(f\"New running size: {running_size} bytes\")\n",
    "print(f\"Total size: {running_size / (1024 ** 3):.2f} GB\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15052800"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "running_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "ename": "OutOfMemoryError",
     "evalue": "CUDA out of memory. Tried to allocate 460.00 MiB. GPU 0 has a total capacty of 47.54 GiB of which 8.38 MiB is free. Process 1312631 has 308.00 MiB memory in use. Process 2252695 has 47.22 GiB memory in use. Of the allocated memory 44.73 GiB is allocated by PyTorch, and 2.18 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOutOfMemoryError\u001b[0m                          Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[15], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m logits, cache \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_with_cache\u001b[49m\u001b[43m(\u001b[49m\u001b[43mioi_dataset\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtoks\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlong\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/mnt/ssd-1/ctigges/miniconda/envs/cot/lib/python3.8/site-packages/transformer_lens/HookedTransformer.py:648\u001b[0m, in \u001b[0;36mHookedTransformer.run_with_cache\u001b[0;34m(self, return_cache_object, remove_batch_dim, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m    631\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrun_with_cache\u001b[39m(\n\u001b[1;32m    632\u001b[0m     \u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39mmodel_args, return_cache_object\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, remove_batch_dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs\n\u001b[1;32m    633\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    640\u001b[0m     Union[ActivationCache, Dict[\u001b[38;5;28mstr\u001b[39m, torch\u001b[38;5;241m.\u001b[39mTensor]],\n\u001b[1;32m    641\u001b[0m ]:\n\u001b[1;32m    642\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Wrapper around `run_with_cache` in HookedRootModule.\u001b[39;00m\n\u001b[1;32m    643\u001b[0m \n\u001b[1;32m    644\u001b[0m \u001b[38;5;124;03m    If return_cache_object is True, this will return an ActivationCache object, with a bunch of\u001b[39;00m\n\u001b[1;32m    645\u001b[0m \u001b[38;5;124;03m    useful HookedTransformer specific methods, otherwise it will return a dictionary of\u001b[39;00m\n\u001b[1;32m    646\u001b[0m \u001b[38;5;124;03m    activations as in HookedRootModule.\u001b[39;00m\n\u001b[1;32m    647\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 648\u001b[0m     out, cache_dict \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_with_cache\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    649\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mremove_batch_dim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mremove_batch_dim\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\n\u001b[1;32m    650\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    651\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m return_cache_object:\n\u001b[1;32m    652\u001b[0m         cache \u001b[38;5;241m=\u001b[39m ActivationCache(\n\u001b[1;32m    653\u001b[0m             cache_dict, \u001b[38;5;28mself\u001b[39m, has_batch_dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;129;01mnot\u001b[39;00m remove_batch_dim\n\u001b[1;32m    654\u001b[0m         )\n",
      "File \u001b[0;32m/mnt/ssd-1/ctigges/miniconda/envs/cot/lib/python3.8/site-packages/transformer_lens/hook_points.py:467\u001b[0m, in \u001b[0;36mHookedRootModule.run_with_cache\u001b[0;34m(self, names_filter, device, remove_batch_dim, incl_bwd, reset_hooks_end, clear_contexts, *model_args, **model_kwargs)\u001b[0m\n\u001b[1;32m    457\u001b[0m cache_dict, fwd, bwd \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_caching_hooks(\n\u001b[1;32m    458\u001b[0m     names_filter, incl_bwd, device, remove_batch_dim\u001b[38;5;241m=\u001b[39mremove_batch_dim\n\u001b[1;32m    459\u001b[0m )\n\u001b[1;32m    461\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhooks(\n\u001b[1;32m    462\u001b[0m     fwd_hooks\u001b[38;5;241m=\u001b[39mfwd,\n\u001b[1;32m    463\u001b[0m     bwd_hooks\u001b[38;5;241m=\u001b[39mbwd,\n\u001b[1;32m    464\u001b[0m     reset_hooks_end\u001b[38;5;241m=\u001b[39mreset_hooks_end,\n\u001b[1;32m    465\u001b[0m     clear_contexts\u001b[38;5;241m=\u001b[39mclear_contexts,\n\u001b[1;32m    466\u001b[0m ):\n\u001b[0;32m--> 467\u001b[0m     model_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    468\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m incl_bwd:\n\u001b[1;32m    469\u001b[0m         model_out\u001b[38;5;241m.\u001b[39mbackward()\n",
      "File \u001b[0;32m/mnt/ssd-1/ctigges/miniconda/envs/cot/lib/python3.8/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/mnt/ssd-1/ctigges/miniconda/envs/cot/lib/python3.8/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m/mnt/ssd-1/ctigges/miniconda/envs/cot/lib/python3.8/site-packages/transformer_lens/HookedTransformer.py:569\u001b[0m, in \u001b[0;36mHookedTransformer.forward\u001b[0;34m(self, input, return_type, loss_per_token, prepend_bos, padding_side, start_at_layer, tokens, shortformer_pos_embed, attention_mask, stop_at_layer, past_kv_cache)\u001b[0m\n\u001b[1;32m    564\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m shortformer_pos_embed \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    565\u001b[0m         shortformer_pos_embed \u001b[38;5;241m=\u001b[39m shortformer_pos_embed\u001b[38;5;241m.\u001b[39mto(\n\u001b[1;32m    566\u001b[0m             devices\u001b[38;5;241m.\u001b[39mget_device_for_block_index(i, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcfg)\n\u001b[1;32m    567\u001b[0m         )\n\u001b[0;32m--> 569\u001b[0m     residual \u001b[38;5;241m=\u001b[39m \u001b[43mblock\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    570\u001b[0m \u001b[43m        \u001b[49m\u001b[43mresidual\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    571\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;66;43;03m# Cache contains a list of HookedTransformerKeyValueCache objects, one for each\u001b[39;49;00m\n\u001b[1;32m    572\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;66;43;03m# block\u001b[39;49;00m\n\u001b[1;32m    573\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpast_kv_cache_entry\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_kv_cache\u001b[49m\u001b[43m[\u001b[49m\u001b[43mi\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m    574\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mpast_kv_cache\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mis\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mnot\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\n\u001b[1;32m    575\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    576\u001b[0m \u001b[43m        \u001b[49m\u001b[43mshortformer_pos_embed\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mshortformer_pos_embed\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    577\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    578\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# [batch, pos, d_model]\u001b[39;00m\n\u001b[1;32m    580\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m stop_at_layer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    581\u001b[0m     \u001b[38;5;66;03m# When we stop at an early layer, we end here rather than doing further computation\u001b[39;00m\n\u001b[1;32m    582\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m residual\n",
      "File \u001b[0;32m/mnt/ssd-1/ctigges/miniconda/envs/cot/lib/python3.8/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/mnt/ssd-1/ctigges/miniconda/envs/cot/lib/python3.8/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m/mnt/ssd-1/ctigges/miniconda/envs/cot/lib/python3.8/site-packages/transformer_lens/components.py:1445\u001b[0m, in \u001b[0;36mTransformerBlock.forward\u001b[0;34m(self, resid_pre, shortformer_pos_embed, past_kv_cache_entry, attention_mask)\u001b[0m\n\u001b[1;32m   1437\u001b[0m     key_input \u001b[38;5;241m=\u001b[39m attn_in\n\u001b[1;32m   1438\u001b[0m     value_input \u001b[38;5;241m=\u001b[39m attn_in\n\u001b[1;32m   1440\u001b[0m attn_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhook_attn_out(\n\u001b[1;32m   1441\u001b[0m     \u001b[38;5;66;03m# hook the residual stream states that are used to calculate the\u001b[39;00m\n\u001b[1;32m   1442\u001b[0m     \u001b[38;5;66;03m# queries, keys and values, independently.\u001b[39;00m\n\u001b[1;32m   1443\u001b[0m     \u001b[38;5;66;03m# Then take the layer norm of these inputs, and pass these to the attention module.\u001b[39;00m\n\u001b[1;32m   1444\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mattn(\n\u001b[0;32m-> 1445\u001b[0m         query_input\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mln1\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquery_input\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1446\u001b[0m         \u001b[38;5;241m+\u001b[39m (\u001b[38;5;241m0.0\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m shortformer_pos_embed \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m shortformer_pos_embed),\n\u001b[1;32m   1447\u001b[0m         key_input\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mln1(key_input)\n\u001b[1;32m   1448\u001b[0m         \u001b[38;5;241m+\u001b[39m (\u001b[38;5;241m0.0\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m shortformer_pos_embed \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m shortformer_pos_embed),\n\u001b[1;32m   1449\u001b[0m         value_input\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mln1(value_input),\n\u001b[1;32m   1450\u001b[0m         past_kv_cache_entry\u001b[38;5;241m=\u001b[39mpast_kv_cache_entry,\n\u001b[1;32m   1451\u001b[0m         attention_mask\u001b[38;5;241m=\u001b[39mattention_mask,\n\u001b[1;32m   1452\u001b[0m     )\n\u001b[1;32m   1453\u001b[0m )  \u001b[38;5;66;03m# [batch, pos, d_model]\u001b[39;00m\n\u001b[1;32m   1454\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcfg\u001b[38;5;241m.\u001b[39mattn_only \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcfg\u001b[38;5;241m.\u001b[39mparallel_attn_mlp:\n\u001b[1;32m   1455\u001b[0m     resid_mid \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhook_resid_mid(\n\u001b[1;32m   1456\u001b[0m         resid_pre \u001b[38;5;241m+\u001b[39m attn_out\n\u001b[1;32m   1457\u001b[0m     )  \u001b[38;5;66;03m# [batch, pos, d_model]\u001b[39;00m\n",
      "File \u001b[0;32m/mnt/ssd-1/ctigges/miniconda/envs/cot/lib/python3.8/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/mnt/ssd-1/ctigges/miniconda/envs/cot/lib/python3.8/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m/mnt/ssd-1/ctigges/miniconda/envs/cot/lib/python3.8/site-packages/transformer_lens/components.py:266\u001b[0m, in \u001b[0;36mLayerNormPre.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    263\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcfg\u001b[38;5;241m.\u001b[39mdtype \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m [torch\u001b[38;5;241m.\u001b[39mfloat32, torch\u001b[38;5;241m.\u001b[39mfloat64]:\n\u001b[1;32m    264\u001b[0m     x \u001b[38;5;241m=\u001b[39m x\u001b[38;5;241m.\u001b[39mto(torch\u001b[38;5;241m.\u001b[39mfloat32)\n\u001b[0;32m--> 266\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[43mx\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mx\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmean\u001b[49m\u001b[43m(\u001b[49m\u001b[43maxis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkeepdim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# [batch, pos, length]\u001b[39;00m\n\u001b[1;32m    267\u001b[0m scale: Union[\n\u001b[1;32m    268\u001b[0m     Float[torch\u001b[38;5;241m.\u001b[39mTensor, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbatch pos 1\u001b[39m\u001b[38;5;124m\"\u001b[39m],\n\u001b[1;32m    269\u001b[0m     Float[torch\u001b[38;5;241m.\u001b[39mTensor, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbatch pos head_index 1\u001b[39m\u001b[38;5;124m\"\u001b[39m],\n\u001b[1;32m    270\u001b[0m ] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhook_scale((x\u001b[38;5;241m.\u001b[39mpow(\u001b[38;5;241m2\u001b[39m)\u001b[38;5;241m.\u001b[39mmean(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m, keepdim\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m) \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39meps)\u001b[38;5;241m.\u001b[39msqrt())\n\u001b[1;32m    271\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhook_normalized(x \u001b[38;5;241m/\u001b[39m scale)\u001b[38;5;241m.\u001b[39mto(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcfg\u001b[38;5;241m.\u001b[39mdtype)\n",
      "\u001b[0;31mOutOfMemoryError\u001b[0m: CUDA out of memory. Tried to allocate 460.00 MiB. GPU 0 has a total capacty of 47.54 GiB of which 8.38 MiB is free. Process 1312631 has 308.00 MiB memory in use. Process 2252695 has 47.22 GiB memory in use. Of the allocated memory 44.73 GiB is allocated by PyTorch, and 2.18 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF"
     ]
    }
   ],
   "source": [
    "logits, cache = model.run_with_cache(ioi_dataset.toks.long())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([100, 21, 768])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cache['hook_embed'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in cache.keys():\n",
    "    if cache[k].shape[0] != 100:\n",
    "        print(k, cache[k].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.allclose(logits, logits_from_batched, atol=1e-01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in cache.keys():\n",
    "    if not torch.allclose(cache[k], cache_from_batched[k], atol=1e-01):\n",
    "        print(k, cache[k].shape, cache_from_batched[k].shape)\n",
    "        # if 'attn_scores' not in k:\n",
    "        for p in range(100):\n",
    "            if not torch.allclose(logits[p][ioi_dataset.word_idx['end'][p]], logits_from_batched[p][ioi_dataset.word_idx['end'][p]], atol=1e-02):\n",
    "                print(f\"index {p}\")\n",
    "                print(ioi_dataset.ioi_prompts[p])\n",
    "                print(logits[p][ioi_dataset.word_idx['end'][p]][:5], logits_from_batched[p][ioi_dataset.word_idx['end'][p]][:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cot",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}