test_extract_defs_with_NER.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a980f048-41df-4700-adfe-1624b077b70d",
   "metadata": {},
   "source": [
    "# Compare results from the Token Classification with other sources\n",
    "Getting the data from the repo: https://github.com/JUrban/extract-defs/tree/main/lm-inputs/defsCT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e119767f-ea2b-4e8c-9d4a-6b819b7d6077",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/luis/code/arxivDownload/venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import os\n",
    "from os.path import join\n",
    "import json\n",
    "\n",
    "from transformers import (AutoTokenizer,\n",
    "                         TFAutoModelForTokenClassification\n",
    ",)\n",
    "\n",
    "import ner.llm_utils as llu\n",
    "\n",
    "cfg = {'checkpoint': 'bert-base-uncased',\n",
    "      'max_length': 150, # check mp_infer_HFTrans_ner.py\n",
    "      }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "dc8b1e9f-48a4-4228-9395-6a2c86c5a332",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 300 definitions to compare.\n"
     ]
    }
   ],
   "source": [
    "xdefs_root = join(os.path.expanduser('~'),'zornMedia/extract-defs')\n",
    "xdefs_inputs = join(xdefs_root, 'lm-inputs/defsCT')\n",
    "xdefs_outputs = join(xdefs_root, 'lm-outputs/defsCT')\n",
    "xdefs_inputs_filelst = sorted(os.listdir(xdefs_inputs))\n",
    "xdefs_outputs_filelst = sorted(os.listdir(xdefs_outputs))\n",
    "\n",
    "with open(join(xdefs_outputs, xdefs_outputs_filelst[4]), 'r') as fobj:\n",
    "    xdefs_out_lst = fobj.readlines()\n",
    "with open(join(xdefs_inputs, xdefs_inputs_filelst[4]), 'r') as fobj:\n",
    "    xdefs_in_lst = fobj.readlines()\n",
    "\n",
    "compare_lst = []\n",
    "for file_index in range(6):\n",
    "# path to the json comparison files\n",
    "    data_compare_path = join(os.path.curdir, \n",
    "                             f'data/2/compare_{file_index:0>3}.json')\n",
    "    with open(data_compare_path, 'r') as fobj:\n",
    "        compare_lst += (json.load(fobj))\n",
    "print(f'There are {len(compare_lst)} definitions to compare.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "67f74a43-a428-4eaa-b9e1-10ca28948f00",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['taxotopy retract']\n"
     ]
    }
   ],
   "source": [
    "# Example to grab 'Definition 8.1: DEFINES \"taxotopic\"\\n'\n",
    "# and 'Definition 7.2: DEFINES \"taxotopic\", \"S-taxotopic\", \"taxotopy equivalent\", \"S-taxotopy equivalent\", \"null-taxotopic\"\\n'\n",
    "#reg_expr = re.compile('Definition[\\s\\d\\.\\:]+DEFINES\\s+\"(.+?)\"+')\n",
    "reg_expr = re.compile('\"(.+?)\"+')\n",
    "def get_term(out_str):\n",
    "    # The out_str is the output string produced by the LLM\n",
    "    Defin = reg_expr.findall(out_str)\n",
    "    return Defin\n",
    "\n",
    "print(get_term(xdefs_out_lst[3]))\n",
    "\n",
    "reg_expr2 = re.compile(\n",
    "    '(?:Definition\\s+[\\d\\.]+\\s+(.+)|[\\d\\.]+\\s+Definition\\.?\\s+(.+))')\n",
    "#reg_expr2 = re.compile('Definition\\s+[\\d\\.]+\\s+(.+)')\n",
    "#reg_expr2 = re.compile('[\\d\\.]+\\s+(.+)')\n",
    "def get_text(in_str):\n",
    "    Defin = reg_expr2.findall(in_str)\n",
    "    return ''.join(Defin[0]) if len(Defin)>0 else in_str\n",
    "#get_text(xdefs_in_lst[-3])    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7ddca3b3-96a2-4f81-a17d-4d24059e1594",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 20/300 = 0.067 exact matches.\n",
      "          78/300 = 0.260 first word matches.\n",
      "          49/300 = 0.163 last word matches.\n",
      "There are 121/300 = 0.403 subword matches.\n"
     ]
    }
   ],
   "source": [
    "# Comparisons\n",
    "# Exact Matches\n",
    "\n",
    "def exact_match(list1, list2):\n",
    "    if list1 is None or list2 is None:\n",
    "        return False\n",
    "    return any([l1 == l2 for l1 in list1 for l2 in list2])\n",
    "\n",
    "def first_word_match(list1, list2):\n",
    "    if list1 is None or list2 is None:\n",
    "        return False\n",
    "    return exact_match([l.split()[0] for l in list1],\n",
    "                       [l.split()[0] for l in list2])\n",
    "    \n",
    "def last_word_match(list1, list2):\n",
    "    if list1 is None or list2 is None:\n",
    "        return False\n",
    "    return exact_match([l.split()[-1] for l in list1],\n",
    "                       [l.split()[-1] for l in list2])\n",
    "\n",
    "def subword_match(list1, list2):\n",
    "    if list1 is None or list2 is None:\n",
    "        return False\n",
    "    # join all the phrases\n",
    "    List1 = ' '.join(list1)\n",
    "    List2 = ' '.join(list2)\n",
    "    return exact_match(List1.split(), List2.split())\n",
    "\n",
    "exact_match_count = 0\n",
    "word_1st_match_count = 0\n",
    "last_word_match_count = 0\n",
    "subword_match_count = 0\n",
    "for comp_dict in compare_lst:\n",
    "    if exact_match(comp_dict['my-term'], comp_dict['extract-defs-term']):\n",
    "        exact_match_count += 1\n",
    "    if first_word_match(comp_dict['my-term'], comp_dict['extract-defs-term']):\n",
    "        word_1st_match_count += 1\n",
    "    if last_word_match(comp_dict['my-term'], comp_dict['extract-defs-term']):\n",
    "        last_word_match_count += 1\n",
    "    if subword_match(comp_dict['my-term'], comp_dict['extract-defs-term']):\n",
    "        subword_match_count += 1\n",
    "print(f\"\"\"There are {exact_match_count}/{len(compare_lst)} = {exact_match_count/len(compare_lst):.3f} exact matches.\"\"\")\n",
    "print(f\"\"\"          {word_1st_match_count}/{len(compare_lst)} = {word_1st_match_count/len(compare_lst):.3f} first word matches.\"\"\")\n",
    "print(f\"\"\"          {last_word_match_count}/{len(compare_lst)} = {last_word_match_count/len(compare_lst):.3f} last word matches.\"\"\")\n",
    "print(f\"\"\"There are {subword_match_count}/{len(compare_lst)} = {subword_match_count/len(compare_lst):.3f} subword matches.\"\"\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "75cedb84-9bf5-4fcf-b081-fb2382b5f4dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "f2b7c1df-958c-4f26-a2f5-ab41eeeecb09",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'A dg-vector bundle over a formal pointed dg-manifold _inline_math_ is given by a linearly compact vector bundle _inline_math_ over _inline_math_ such that the corresponding module _inline_math_ carries a differential _inline_math_ so that _inline_math_ becomes a dg-module over the dg-algebra _inline_math_ and _inline_math_ vanishes on _inline_math_.'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_expr3 = re.compile('\\$.+?\\$')\n",
    "def remove_latex_formulas(text):\n",
    "    return re.sub(reg_expr3,'_inline_math_', text)\n",
    "remove_latex_formulas(get_text(xdefs_in_lst[24]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1485e976-e715-4686-b9ff-d25d839b9c38",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer\\\n",
    "          .from_pretrained(cfg['checkpoint'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ec624d8b-816b-460b-bf1d-b3d97eb4041f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-05 03:42:05.173600: I external/local_xla/xla/tsl/cuda/cudart_stub.cc:32] Could not find cuda drivers on your machine, GPU will not be used.\n",
      "2024-08-05 03:42:05.175761: I external/local_xla/xla/tsl/cuda/cudart_stub.cc:32] Could not find cuda drivers on your machine, GPU will not be used.\n",
      "2024-08-05 03:42:05.183593: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-08-05 03:42:05.197704: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-08-05 03:42:05.201749: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-08-05 03:42:05.212345: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-08-05 03:42:06.020875: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'input_ids': <tf.Tensor: shape=(1, 31), dtype=int32, numpy=\n",
       "array([[  101,  6210,  1023,  1012,  1017,  1037,  4359,  8040,  2828,\n",
       "         1002,  1050,  1002,  1011,  3850,  2094,  1999,  1037, 19490,\n",
       "        18847, 16975,  4696,  1002,  1058,  1002,  2003,  1037,  3074,\n",
       "         1002,  1037,  1035,   102]], dtype=int32)>, 'token_type_ids': <tf.Tensor: shape=(1, 31), dtype=int32, numpy=\n",
       "array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=int32)>, 'attention_mask': <tf.Tensor: shape=(1, 31), dtype=int32, numpy=\n",
       "array([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "        1, 1, 1, 1, 1, 1, 1, 1, 1]], dtype=int32)>}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer(xdefs_in_lst[15][:100],\n",
    "         return_tensors='tf',\n",
    "         is_split_into_words=False,\n",
    "         padding=True,\n",
    "         truncation=True,\n",
    "         max_length=cfg['max_length'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c07e0474-16d9-4c4f-a946-cc3f8a459cd9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Definition 9.3 A reduced SC type $n$-operad in a symmetric monoidal category $V$ is a collection $A_{T}\\\\in V$ where $T$ runs over all coloured pruned $n$-trees such that $A_{U^{1}_{n}}=A_{U^{2}_{n}}=I$ and $A_{z^{n}U_{0}}=I,$ where $U^{1}_{n}$ is a linear tree with its unique tip coloured by $1$ and $U^{2}_{n}$ is a linear tree with its tip coloured by $2.$ This collection is equipped with a morphism $m_{\\\\sigma}:A_{S}\\\\otimes A_{T_{1}^{(p)}}\\\\otimes\\\\ldots\\\\otimes A_{T_{k}^{(p)}}% \\\\rightarrow A_{T}$ for every coloured morphism of trees $\\\\sigma:T\\\\rightarrow S$ between coloured pruned $n$-trees. They must satisfy the obvious associativity and unitarity conditions. For an unbased reduced SC-operad we use collections without degenerate trees and define multiplication only with respect to surjections of coloured trees.\\n'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xdefs_in_lst[15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "3ba82545-c2c2-49b8-8680-73e3baca05a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Getzler-Jones $n$-operad']"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_term(xdefs_out_lst[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "1aca6114-d4cd-4682-ad91-eb937736673d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"in_text\": \"hola como estan?\", \"out_prompt\": \"bien\", \"out_finetune\": \"todo bien\"}'"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out_dict = {'in_text': 'hola como estan?',\n",
    "            'out_prompt': 'bien',\n",
    "            'out_finetune': 'todo bien'}\n",
    "\n",
    "json.dumps(out_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "d526533e-a09b-45b3-9e2e-5adf63e34c4b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.path.isdir('/home/luis/zornMedia/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "6d0e220e-1f22-441e-8369-c457796adeab",
   "metadata": {},
   "outputs": [
    {
     "ename": "AssertionError",
     "evalue": "hola",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[47], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhola\u001b[39m\u001b[38;5;124m'\u001b[39m\n",
      "\u001b[0;31mAssertionError\u001b[0m: hola"
     ]
    }
   ],
   "source": [
    "assert False, 'hola'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "98a94d65-0066-4413-89ba-8f59ce45e2c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "ll = []\n",
    "ll[0] if len(ll)>0 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "26a2d3a4-0ddd-4e2e-bffa-407edd9e6d96",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(66, 66)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concat_tokens=[['[CLS]', 'The', 'Get', '##zle', '##r', '-', 'Jones', '_', 'in', '##line', '_', 'math', '_', '-', 'opera', '##d', '_', 'in', '##line', '_', 'math', '_', '\\\\', 'b', '##f', 'G', '##J', '_', 'in', '##line', '_', 'math', '_', 'is', 'the', 'image', 'of', '_', 'in', '##line', '_', 'math', '_', 'This', 'is', 'a', 'reduced', '_', 'in', '##line', '_', 'math', '_', '-', 'terminal', '_', 'in', '##line', '_', 'math', '_', '-', 'opera', '##d', '.', '[SEP]']]\n",
    "ccc =  ['O', 'O', 'B-DFNDUM', 'I-DFNDUM', 'I-DFNDUM', 'I-DFNDUM', 'I-DFNDUM', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-DFNDUM', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n",
    "len(concat_tokens[0]), len(ccc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "3e1fe490-f9d5-49c7-9e95-6f33adebca95",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'special_tokens'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[59], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mllu\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcrop_terms\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconcat_tokens\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mccc\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/code/arxivDownload/ner/llm_utils.py:148\u001b[0m, in \u001b[0;36mcrop_terms\u001b[0;34m(tokens, preds, **kwargs)\u001b[0m\n\u001b[1;32m    146\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcrop_terms\u001b[39m(tokens, preds, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m    147\u001b[0m     words, preds \u001b[38;5;241m=\u001b[39m get_words_back(tokens, preds\u001b[38;5;241m=\u001b[39mpreds, \n\u001b[0;32m--> 148\u001b[0m                       special_tokens\u001b[38;5;241m=\u001b[39m\u001b[43mkwargs\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mspecial_tokens\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m)\n\u001b[1;32m    149\u001b[0m     golds \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mgolds\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m    151\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m golds \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "\u001b[0;31mKeyError\u001b[0m: 'special_tokens'"
     ]
    }
   ],
   "source": [
    "llu.crop_terms(concat_tokens[0], ccc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "64009840-1026-4b2d-9e62-ba671986fdc3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'_004'"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f\"_{4:0>3}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63e174a1-e50f-4a3f-8fea-88189de45f69",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}