OECD/generate_pubtator_input.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "84a00149",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This script is used to generate input files for PubTator3 API\n",
    "# Check scrapper.ipynb to see the example about how to scrape data from OECD webpage\n",
    "import csv\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "85d0b9fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read the CSV file\n",
    "input_file = os.getcwd() + '/extracted_data.csv'\n",
    "\n",
    "# Open the CSV file and read its content\n",
    "with open(input_file, mode='r', encoding='utf-8') as file:\n",
    "    reader = csv.DictReader(file)\n",
    "    # Filter rows to include only those starting with \"Test No\"\n",
    "    filtered_rows = [row for row in reader if row['Title'].startswith('Test No')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "0637664b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/442E.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/487.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/439.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/444A.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/442D.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/473.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/487.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/473.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/487.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/428.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/473.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/435.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/435.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/476.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/498.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/439.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/432.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/439.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/431.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/430.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/431.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/431.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/476.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/490.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/476.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/431.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/490.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/430.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/431.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/430.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/479.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/455.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/455.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/319B.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/319A.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/455.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/482.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/496.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/491.txt.\n",
      "Extracted data has been combined and written into /Users/tsunghanhsieh/BioNER/OECD/Pubtator_Input/493.txt.\n"
     ]
    }
   ],
   "source": [
    "# Save file for PubTator3 API\n",
    "# Iterate over the filtered rows and save each as a separate text file\n",
    "\n",
    "\n",
    "# Generate output text\n",
    "for row in filtered_rows:\n",
    "    # Extract test tile\n",
    "    test_title = row['Title']\n",
    "    \n",
    "    # Extract the test number from the Title column\n",
    "    test_no = row['Title'].split(':')[0].split()[-1]\n",
    "    \n",
    "    # Set the file name\n",
    "    output_file = os.getcwd() + \"/Pubtator_Input/\" + test_no + \".txt\"\n",
    "    \n",
    "    # Get the content from the Description column\n",
    "    content = row['Description']\n",
    "    \n",
    "    # Save the content to the file\n",
    "    with open(output_file, 'w', encoding='utf-8') as file:\n",
    "        file.write(test_no + 'n')\n",
    "        file.write(test_title + 'n')\n",
    "        file.write(output_text + '\\n\\n')\n",
    "    print(f\"Extracted data has been combined and written into {output_file}.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23b26b0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note the output may contain duplicated information.\n",
    "# Need some manual curation after PubTator3 API annotation."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}