From 218072c2597c0a3fe22395d5f34aac3f79c594ad Mon Sep 17 00:00:00 2001 From: alisoncallahan Date: Wed, 20 Apr 2022 15:48:19 -0700 Subject: [PATCH 1/3] First commit of TAC 2017 dataloader. VERY MUCH in progress. --- biodatasets/tac2017/tac2017.py | 300 +++++++++++++++++++++++++++++++++ 1 file changed, 300 insertions(+) create mode 100644 biodatasets/tac2017/tac2017.py diff --git a/biodatasets/tac2017/tac2017.py b/biodatasets/tac2017/tac2017.py new file mode 100644 index 00000000..64b3f8d8 --- /dev/null +++ b/biodatasets/tac2017/tac2017.py @@ -0,0 +1,300 @@ +# coding=utf-8 +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This template serves as a starting point for contributing a dataset to the BigScience Biomedical repo. + +When modifying it for your dataset, look for TODO items that offer specific instructions. + +Full documentation on writing dataset loading scripts can be found here: +https://huggingface.co/docs/datasets/add_dataset.html + +To create a dataset loading script you will create a class and implement 3 methods: + * `_info`: Establishes the schema for the dataset, and returns a datasets.DatasetInfo object. + * `_split_generators`: Downloads and extracts data for each split (e.g. train/val/test) or associate local data with each split. + * `_generate_examples`: Creates examples from data on disk that conform to each schema defined in `_info`. + +TODO: Before submitting your script, delete this doc string and replace it with a description of your dataset. + +[bigbio_schema_name] = (kb, pairs, qa, text, t2t, entailment) +""" + +import os +from typing import List, Tuple, Dict + +#from lxml import etree +import xml.etree.ElementTree as ET + +import datasets +from utils import schemas +from utils.configs import BigBioConfig +from utils.constants import Tasks + +# TODO: Add BibTeX citation +_CITATION = """\ +@article{, + author = {}, + title = {}, + journal = {}, + volume = {}, + year = {}, + url = {}, + doi = {}, + biburl = {}, + bibsource = {} +} +""" + +_DATASETNAME = "tac2017" + +_DESCRIPTION = """\ +This dataset is designed for extraction of ADRs from prescription drug labels. +""" +_HOMEPAGE = "https://bionlp.nlm.nih.gov/tac2017adversereactions/" + +_LICENSE = "None provided." + +# TODO: Add links to the urls needed to download your dataset files. +# For local datasets, this variable can be an empty dictionary. + +# For publicly available datasets you will most likely end up passing these URLs to dl_manager in _split_generators. +# In most cases the URLs will be the same for the source and bigbio config. +# However, if you need to access different files for each config you can have multiple entries in this dict. +# This can be an arbitrarily nested dict/list of URLs (see below in `_split_generators` method) +_URLS = { + "tac2017": "https://bionlp.nlm.nih.gov/tac2017adversereactions/train_xml.tar.gz", +} + +_SUPPORTED_TASKS = [ + Tasks.NAMED_ENTITY_DISAMBIGUATION, + Tasks.NAMED_ENTITY_RECOGNITION, + Tasks.RELATION_EXTRACTION +] + +_SOURCE_VERSION = "1.0.0" + +_BIGBIO_VERSION = "1.0.0" + +class Tac2017Dataset(datasets.GeneratorBasedBuilder): + """TODO: Short description of my dataset.""" + + SOURCE_VERSION = datasets.Version(_SOURCE_VERSION) + BIGBIO_VERSION = datasets.Version(_BIGBIO_VERSION) + + # You will be able to load the "source" or "bigbio" configurations with + # ds_source = datasets.load_dataset('my_dataset', name='source') + # ds_bigbio = datasets.load_dataset('my_dataset', name='bigbio') + + # For local datasets you can make use of the `data_dir` and `data_files` kwargs + # https://huggingface.co/docs/datasets/add_dataset.html#downloading-data-files-and-organizing-splits + # ds_source = datasets.load_dataset('my_dataset', name='source', data_dir="/path/to/data/files") + # ds_bigbio = datasets.load_dataset('my_dataset', name='bigbio', data_dir="/path/to/data/files") + + # TODO: For each dataset, implement Config for Source and BigBio; + # If dataset contains more than one subset (see examples/bioasq.py) implement for EACH of them. + # Each of them should contain: + # - name: should be unique for each dataset config eg. bioasq10b_(source|bigbio)_[bigbio_schema_name] + # - version: option = (SOURCE_VERSION|BIGBIO_VERSION) + # - description: one line description for the dataset + # - schema: options = (source|bigbio_[bigbio_schema_name]) + # - subset_id: subset id is the canonical name for the dataset (eg. bioasq10b) + # where [bigbio_schema_name] = (kb, pairs, qa, text, t2t, entailment) + + BUILDER_CONFIGS = [ + BigBioConfig( + name="tac2017_source", + version=SOURCE_VERSION, + description="TAC 2017 source schema", + schema="source", + subset_id="tac2017", + ), + BigBioConfig( + name="tac2017_bigbio_[bigbio_schema_name]", + version=BIGBIO_VERSION, + description="TAC 2017 BigBio schema", + schema="bigbio_kb", + subset_id="tac2017", + ), + ] + + DEFAULT_CONFIG_NAME = "tac2017_source" + + def _info(self) -> datasets.DatasetInfo: + + # Create the source schema; this schema will keep all keys/information/labels as close to the original dataset as possible. + + # You can arbitrarily nest lists and dictionaries. + # For iterables, use lists over tuples or `datasets.Sequence` + + if self.config.schema == "source": + # TODO: Create your source schema here + features = datasets.Features( + { + "labels" : { + "drug": datasets.Value("string"), + "text": { + "section" : { + "name": datasets.Value("string"), + "id": datasets.Value("string"), + "text": datasets.Value("string"), + }, + "mentions" : { + "id": datasets.Value("string"), + "type": datasets.Value("string"), + "section": datasets.Value("string"), + "start": datasets.Value("int32"), + "len": datasets.Value("int32"), + "str":datasets.Value("string") + }, + "relations": { + "id": datasets.Value("string"), + "type": datasets.Value("string"), + "arg1": datasets.Value("string"), + "arg2": datasets.Value("string") + }, + "reactions": { + "id": datasets.Value("string"), + "str": datasets.Value("string"), + "normalization": { + "id": datasets.Value("string"), + "meddra_pt": datasets.Value("string"), + "meddra_pt_id": datasets.Value("string"), + "meddra_llt": datasets.Value("string"), + "meddra_llt_id": datasets.Value("string") + }, + } + } + } + } + ) + + # EX: Arbitrary NER type dataset + # features = datasets.Features( + # { + # "doc_id": datasets.Value("string"), + # "text": datasets.Value("string"), + # "entities": [ + # { + # "offsets": [datasets.Value("int64")], + # "text": datasets.Value("string"), + # "type": datasets.Value("string"), + # "entity_id": datasets.Value("string"), + # } + # ], + # } + # ) + + # Choose the appropriate bigbio schema for your task and copy it here. You can find information on the schemas in the CONTRIBUTING guide. + + # In rare cases you may get a dataset that supports multiple tasks requiring multiple schemas. In that case you can define multiple bigbio configs with a bigbio_[bigbio_schema_name] format. + + # For example bigbio_kb, bigbio_t2t + elif self.config.schema == "bigbio_kb": + # e.g. features = schemas.kb_features + # TODO: Choose your big-bio schema here + raise NotImplementedError() + + return datasets.DatasetInfo( + description=_DESCRIPTION, + features=features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager) -> List[datasets.SplitGenerator]: + """Returns SplitGenerators.""" + # TODO: This method is tasked with downloading/extracting the data and defining the splits depending on the configuration + + # If you need to access the "source" or "bigbio" config choice, that will be in self.config.name + + # LOCAL DATASETS: You do not need the dl_manager; you can ignore this argument. Make sure `gen_kwargs` in the return gets passed the right filepath + + # PUBLIC DATASETS: Assign your data-dir based on the dl_manager. + + # dl_manager is a datasets.download.DownloadManager that can be used to download and extract URLs; many examples use the download_and_extract method; see the DownloadManager docs here: https://huggingface.co/docs/datasets/package_reference/builder_classes.html#datasets.DownloadManager + + # dl_manager can accept any type of nested list/dict and will give back the same structure with the url replaced with the path to local files. + + # TODO: KEEP if your dataset is PUBLIC; remove if not + # urls = _URLS[_DATASETNAME] + # data_dir = dl_manager.download_and_extract(urls) + # Not all datasets have predefined canonical train/val/test splits. + # If your dataset has no predefined splits, use datasets.Split.TRAIN for all of the data. + if self.config.data_dir is None: + raise ValueError("This is a local dataset. Please pass the data_dir kwarg to load_dataset.") + else: + data_dir = self.config.data_dir + + return [ + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + # Whatever you put in gen_kwargs will be passed to _generate_examples + gen_kwargs={ + "files": os.listdir(data_dir), + "split": "train", + }, + ), + ] + + # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` + + # TODO: change the args of this function to match the keys in `gen_kwargs`. You may add any necessary kwargs. + + def _generate_examples(self, files, split: str) -> Tuple[int, Dict]: + """Yields examples as (key, example) tuples.""" + # TODO: This method handles input defined in _split_generators to yield (key, example) tuples from the dataset. + + # The `key` is for legacy reasons (tfds) and is not important in itself, but must be unique for each example. + + # NOTE: For local datasets you will have access to self.config.data_dir and self.config.data_files + + if self.config.schema == "source": + for file in files: + fpath = os.path.join(self.config.data_dir, file) + with open(fpath) as xml_file: + tree = ET.parse(xml_file) + label = tree.getroot() + #print(label.attrib["drug"]) + uid = 0 + for child in label: + example = {} + if child.tag == "Text": + text = child + for section in text: + example["section"] = section + #print(section.attrib["id"]) + #print(section.text) + yield uid, example + uid +=1 + + # # TODO: yield (key, example) tuples in the original dataset schema + # for key, example in thing: + # yield key, example + + elif self.config.schema == "bigbio_kb": + # TODO: yield (key, example) tuples in the bigbio schema + for key, example in thing: + yield key, example + + +# This template is based on the following template from the datasets package: +# https://github.com/huggingface/datasets/blob/master/templates/new_dataset_script.py + + +# This allows you to run your dataloader with `python [dataset_name].py` during development +# TODO: Remove this before making your PR +if __name__ == "__main__": + datasets.load_dataset(__file__) From 13f5244dbb5b1eb4b0e1b630ca4754618a16ac72 Mon Sep 17 00:00:00 2001 From: alisoncallahan Date: Thu, 21 Apr 2022 15:03:51 -0700 Subject: [PATCH 2/3] updated data downloading. still working on schema and example generator --- biodatasets/tac2017/tac2017.py | 133 ++++++++++++++++++++++++++++----- 1 file changed, 114 insertions(+), 19 deletions(-) diff --git a/biodatasets/tac2017/tac2017.py b/biodatasets/tac2017/tac2017.py index 64b3f8d8..4c83ef17 100644 --- a/biodatasets/tac2017/tac2017.py +++ b/biodatasets/tac2017/tac2017.py @@ -223,31 +223,126 @@ def _split_generators(self, dl_manager) -> List[datasets.SplitGenerator]: # LOCAL DATASETS: You do not need the dl_manager; you can ignore this argument. Make sure `gen_kwargs` in the return gets passed the right filepath # PUBLIC DATASETS: Assign your data-dir based on the dl_manager. + train_fpaths = { + 'train_xml/ADCETRIS.xml', + 'train_xml/BEPREVE.xml', + 'train_xml/CLEVIPREX.xml', + 'train_xml/DYSPORT.xml', + 'train_xml/FERRIPROX.xml', + 'train_xml/ILARIS.xml', + 'train_xml/KALYDECO.xml', + 'train_xml/ONFI.xml', + 'train_xml/SAPHRIS.xml', + 'train_xml/TIVICAY.xml', + 'train_xml/VIZAMYL.xml', + 'train_xml/ZYKADIA.xml', + 'train_xml/ADREVIEW.xml', + 'train_xml/BESIVANCE.xml', + 'train_xml/COARTEM.xml', + 'train_xml/EDARBI.xml', + 'train_xml/FIRAZYR.xml', + 'train_xml/IMBRUVICA.xml', + 'train_xml/KYPROLIS.xml', + 'train_xml/OTEZLA.xml', + 'train_xml/SIMPONI.xml', + 'train_xml/TOVIAZ.xml', + 'train_xml/VORAXAZE.xml', + 'train_xml/ZYTIGA.xml', + 'train_xml/AFINITOR.xml', + 'train_xml/BLINCYTO.xml', + 'train_xml/COMETRIQ.xml', + 'train_xml/ELIQUIS.xml', + 'train_xml/FULYZAQ.xml', + 'train_xml/INLYTA.xml', + 'train_xml/LUMIZYME.xml', + 'train_xml/PICATO.xml', + 'train_xml/SIRTURO.xml', + 'train_xml/TREANDA.xml', + 'train_xml/XALKORI.xml', + 'train_xml/AMPYRA.xml', + 'train_xml/BOSULIF.xml', + 'train_xml/DALVANCE.xml', + 'train_xml/ENTEREG.xml', + 'train_xml/GADAVIST.xml', + 'train_xml/INTELENCE.xml', + 'train_xml/MULTAQ.xml', + 'train_xml/POTIGA.xml', + 'train_xml/STENDRA.xml', + 'train_xml/TRULICITY.xml', + 'train_xml/XEOMIN.xml', + 'train_xml/AMYVID.xml', + 'train_xml/BREO.xml', + 'train_xml/DATSCAN.xml', + 'train_xml/EOVIST.xml', + 'train_xml/GILENYA.xml', + 'train_xml/INVOKANA.xml', + 'train_xml/NATAZIA.xml', + 'train_xml/PRADAXA.xml', + 'train_xml/STRIBILD.xml', + 'train_xml/TUDORZA.xml', + 'train_xml/XIAFLEX.xml', + 'train_xml/APTIOM.xml', + 'train_xml/CARBAGLU.xml', + 'train_xml/DIFICID.xml', + 'train_xml/ERWINAZE.xml', + 'train_xml/GILOTRIF.xml', + 'train_xml/JARDIANCE.xml', + 'train_xml/NESINA.xml', + 'train_xml/PRISTIQ.xml', + 'train_xml/TAFINLAR.xml', + 'train_xml/ULESFIA.xml', + 'train_xml/XTANDI.xml', + 'train_xml/ARCAPTA.xml', + 'train_xml/CERDELGA.xml', + 'train_xml/DOTAREM.xml', + 'train_xml/EYLEA.xml', + 'train_xml/GRANIX.xml', + 'train_xml/JEVTANA.xml', + 'train_xml/NEURACEQ.xml', + 'train_xml/PROLIA.xml', + 'train_xml/TANZEUM.xml', + 'train_xml/ULORIC.xml', + 'train_xml/YERVOY.xml', + 'train_xml/BELEODAQ.xml', + 'train_xml/CHOLINE.xml', + 'train_xml/DUAVEE.xml', + 'train_xml/FANAPT.xml', + 'train_xml/HALAVEN.xml', + 'train_xml/JUBLIA.xml', + 'train_xml/NORTHERA.xml', + 'train_xml/PROMACTA.xml', + 'train_xml/TECFIDERA.xml', + 'train_xml/VICTRELIS.xml', + 'train_xml/ZERBAXA.xml', + 'train_xml/BENLYSTA.xml', + 'train_xml/CIMZIA.xml', + 'train_xml/DUREZOL.xml', + 'train_xml/FARXIGA.xml', + 'train_xml/HORIZANT.xml', + 'train_xml/KALBITOR.xml', + 'train_xml/NULOJIX.xml', + 'train_xml/QUTENZA.xml', + 'train_xml/TEFLARO.xml', + 'train_xml/VIMIZIM.xml', + 'train_xml/ZYDELIG.xml' + } # dl_manager is a datasets.download.DownloadManager that can be used to download and extract URLs; many examples use the download_and_extract method; see the DownloadManager docs here: https://huggingface.co/docs/datasets/package_reference/builder_classes.html#datasets.DownloadManager # dl_manager can accept any type of nested list/dict and will give back the same structure with the url replaced with the path to local files. - # TODO: KEEP if your dataset is PUBLIC; remove if not - # urls = _URLS[_DATASETNAME] - # data_dir = dl_manager.download_and_extract(urls) - # Not all datasets have predefined canonical train/val/test splits. - # If your dataset has no predefined splits, use datasets.Split.TRAIN for all of the data. - if self.config.data_dir is None: - raise ValueError("This is a local dataset. Please pass the data_dir kwarg to load_dataset.") - else: - data_dir = self.config.data_dir - + urls = _URLS[_DATASETNAME] + data_dir = dl_manager.download_and_extract(urls) return [ datasets.SplitGenerator( name=datasets.Split.TRAIN, - # Whatever you put in gen_kwargs will be passed to _generate_examples - gen_kwargs={ - "files": os.listdir(data_dir), + gen_kwargs= { + "files": [os.path.join(data_dir, path) for path in train_fpaths], "split": "train", }, ), - ] + ] + # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` @@ -260,11 +355,11 @@ def _generate_examples(self, files, split: str) -> Tuple[int, Dict]: # The `key` is for legacy reasons (tfds) and is not important in itself, but must be unique for each example. # NOTE: For local datasets you will have access to self.config.data_dir and self.config.data_files + print(files) if self.config.schema == "source": for file in files: - fpath = os.path.join(self.config.data_dir, file) - with open(fpath) as xml_file: + with open(file) as xml_file: tree = ET.parse(xml_file) label = tree.getroot() #print(label.attrib["drug"]) @@ -275,9 +370,9 @@ def _generate_examples(self, files, split: str) -> Tuple[int, Dict]: text = child for section in text: example["section"] = section - #print(section.attrib["id"]) - #print(section.text) - yield uid, example + print(section.attrib["id"]) + print(section.text) + #yield uid, example uid +=1 # # TODO: yield (key, example) tuples in the original dataset schema From 8c22b2c4a390e28176ba61e734d0034fe267c324 Mon Sep 17 00:00:00 2001 From: alisoncallahan Date: Fri, 29 Apr 2022 15:16:25 -0700 Subject: [PATCH 3/3] Major update to add example generators for source and kb schemas. There is some weirdness with text offsets and how special characters are rendered (I think), so tests do not pass. --- biodatasets/tac2017/tac2017.py | 855 ++++++++++++++++++++++----------- 1 file changed, 571 insertions(+), 284 deletions(-) diff --git a/biodatasets/tac2017/tac2017.py b/biodatasets/tac2017/tac2017.py index 4c83ef17..ced122e8 100644 --- a/biodatasets/tac2017/tac2017.py +++ b/biodatasets/tac2017/tac2017.py @@ -14,49 +14,40 @@ # limitations under the License. """ -This template serves as a starting point for contributing a dataset to the BigScience Biomedical repo. - -When modifying it for your dataset, look for TODO items that offer specific instructions. - -Full documentation on writing dataset loading scripts can be found here: -https://huggingface.co/docs/datasets/add_dataset.html - -To create a dataset loading script you will create a class and implement 3 methods: - * `_info`: Establishes the schema for the dataset, and returns a datasets.DatasetInfo object. - * `_split_generators`: Downloads and extracts data for each split (e.g. train/val/test) or associate local data with each split. - * `_generate_examples`: Creates examples from data on disk that conform to each schema defined in `_info`. - -TODO: Before submitting your script, delete this doc string and replace it with a description of your dataset. - -[bigbio_schema_name] = (kb, pairs, qa, text, t2t, entailment) +Drug labels (prescribing information or package inserts) describe what a particular medicine +is supposed to do, who should or should not take it, how to use it, and specific safety concerns. +The US Food and Drug Administration (FDA) publishes regulations governing the content and format +of this information to provide recommendations for applicants developing labeling for new drugs +and revising labeling for already approved drugs. One of the major aspects of drug information +are safety concerns in the form of Adverse Drug Reactions (ADRs). In this evaluation, we are +focusing on extraction of ADRs from the prescription drug labels. """ import os -from typing import List, Tuple, Dict - -#from lxml import etree import xml.etree.ElementTree as ET +from typing import Dict, List, Tuple import datasets + from utils import schemas from utils.configs import BigBioConfig from utils.constants import Tasks -# TODO: Add BibTeX citation _CITATION = """\ @article{, - author = {}, - title = {}, - journal = {}, + author = {Kirk Roberts and + Dina Demner-Fushman and + Joseph M. Tonning}, + title = {Overview of the TAC 2017 Adverse Reaction Extraction from Drug Labels Track}, + journal = {Proceedings of the Text Analysis Conference (TAC) 2017, November 13-14 2017, Gaithersburg MD USA}, volume = {}, - year = {}, - url = {}, + year = {2017}, + url = {https://tac.nist.gov/publications/2017/additional.papers/TAC2017.ADR_overview.proceedings.pdf}, doi = {}, biburl = {}, bibsource = {} } """ - _DATASETNAME = "tac2017" _DESCRIPTION = """\ @@ -66,52 +57,23 @@ _LICENSE = "None provided." -# TODO: Add links to the urls needed to download your dataset files. -# For local datasets, this variable can be an empty dictionary. - -# For publicly available datasets you will most likely end up passing these URLs to dl_manager in _split_generators. -# In most cases the URLs will be the same for the source and bigbio config. -# However, if you need to access different files for each config you can have multiple entries in this dict. -# This can be an arbitrarily nested dict/list of URLs (see below in `_split_generators` method) _URLS = { "tac2017": "https://bionlp.nlm.nih.gov/tac2017adversereactions/train_xml.tar.gz", } -_SUPPORTED_TASKS = [ - Tasks.NAMED_ENTITY_DISAMBIGUATION, - Tasks.NAMED_ENTITY_RECOGNITION, - Tasks.RELATION_EXTRACTION -] +_SUPPORTED_TASKS = [Tasks.NAMED_ENTITY_DISAMBIGUATION, Tasks.NAMED_ENTITY_RECOGNITION, Tasks.RELATION_EXTRACTION] _SOURCE_VERSION = "1.0.0" _BIGBIO_VERSION = "1.0.0" + class Tac2017Dataset(datasets.GeneratorBasedBuilder): - """TODO: Short description of my dataset.""" + """The TAC 2017 dataset is designed for extraction of ADRs from prescription drug labels.""" SOURCE_VERSION = datasets.Version(_SOURCE_VERSION) BIGBIO_VERSION = datasets.Version(_BIGBIO_VERSION) - # You will be able to load the "source" or "bigbio" configurations with - # ds_source = datasets.load_dataset('my_dataset', name='source') - # ds_bigbio = datasets.load_dataset('my_dataset', name='bigbio') - - # For local datasets you can make use of the `data_dir` and `data_files` kwargs - # https://huggingface.co/docs/datasets/add_dataset.html#downloading-data-files-and-organizing-splits - # ds_source = datasets.load_dataset('my_dataset', name='source', data_dir="/path/to/data/files") - # ds_bigbio = datasets.load_dataset('my_dataset', name='bigbio', data_dir="/path/to/data/files") - - # TODO: For each dataset, implement Config for Source and BigBio; - # If dataset contains more than one subset (see examples/bioasq.py) implement for EACH of them. - # Each of them should contain: - # - name: should be unique for each dataset config eg. bioasq10b_(source|bigbio)_[bigbio_schema_name] - # - version: option = (SOURCE_VERSION|BIGBIO_VERSION) - # - description: one line description for the dataset - # - schema: options = (source|bigbio_[bigbio_schema_name]) - # - subset_id: subset id is the canonical name for the dataset (eg. bioasq10b) - # where [bigbio_schema_name] = (kb, pairs, qa, text, t2t, entailment) - BUILDER_CONFIGS = [ BigBioConfig( name="tac2017_source", @@ -121,7 +83,7 @@ class Tac2017Dataset(datasets.GeneratorBasedBuilder): subset_id="tac2017", ), BigBioConfig( - name="tac2017_bigbio_[bigbio_schema_name]", + name="tac2017_bigbio_kb", version=BIGBIO_VERSION, description="TAC 2017 BigBio schema", schema="bigbio_kb", @@ -133,263 +95,588 @@ class Tac2017Dataset(datasets.GeneratorBasedBuilder): def _info(self) -> datasets.DatasetInfo: - # Create the source schema; this schema will keep all keys/information/labels as close to the original dataset as possible. - - # You can arbitrarily nest lists and dictionaries. - # For iterables, use lists over tuples or `datasets.Sequence` - if self.config.schema == "source": - # TODO: Create your source schema here features = datasets.Features( { - "labels" : { - "drug": datasets.Value("string"), - "text": { - "section" : { - "name": datasets.Value("string"), - "id": datasets.Value("string"), - "text": datasets.Value("string"), - }, - "mentions" : { + "drug": datasets.Value("string"), + "text": { + "sections": [ + { "id": datasets.Value("string"), - "type": datasets.Value("string"), - "section": datasets.Value("string"), - "start": datasets.Value("int32"), - "len": datasets.Value("int32"), - "str":datasets.Value("string") - }, - "relations": { + "name": datasets.Value("string"), + "section_text": datasets.Value("string"), + } + ], + }, + "mentions": [ + { + "id": datasets.Value("string"), + "source_id": datasets.Value("string"), + "type": datasets.Value("string"), + "section_id": datasets.Value("string"), + "start": datasets.Value("int32"), + "len": datasets.Value("int32"), + "str": datasets.Value("string"), + } + ], + "relations": [ + { + "id": datasets.Value("string"), + "type": datasets.Value("string"), + "arg1": datasets.Value("string"), + "arg2": datasets.Value("string"), + } + ], + "reactions": [ + { + "id": datasets.Value("string"), + "str": datasets.Value("string"), + "normalization": { "id": datasets.Value("string"), - "type": datasets.Value("string"), - "arg1": datasets.Value("string"), - "arg2": datasets.Value("string") + "meddra_pt": datasets.Value("string"), + "meddra_pt_id": datasets.Value("string"), + "meddra_llt": datasets.Value("string"), + "meddra_llt_id": datasets.Value("string"), }, - "reactions": { - "id": datasets.Value("string"), - "str": datasets.Value("string"), - "normalization": { - "id": datasets.Value("string"), - "meddra_pt": datasets.Value("string"), - "meddra_pt_id": datasets.Value("string"), - "meddra_llt": datasets.Value("string"), - "meddra_llt_id": datasets.Value("string") - }, - } } - } + ], } ) - # EX: Arbitrary NER type dataset - # features = datasets.Features( - # { - # "doc_id": datasets.Value("string"), - # "text": datasets.Value("string"), - # "entities": [ - # { - # "offsets": [datasets.Value("int64")], - # "text": datasets.Value("string"), - # "type": datasets.Value("string"), - # "entity_id": datasets.Value("string"), - # } - # ], - # } - # ) - - # Choose the appropriate bigbio schema for your task and copy it here. You can find information on the schemas in the CONTRIBUTING guide. - - # In rare cases you may get a dataset that supports multiple tasks requiring multiple schemas. In that case you can define multiple bigbio configs with a bigbio_[bigbio_schema_name] format. - - # For example bigbio_kb, bigbio_t2t elif self.config.schema == "bigbio_kb": - # e.g. features = schemas.kb_features - # TODO: Choose your big-bio schema here - raise NotImplementedError() + features = schemas.kb_features return datasets.DatasetInfo( - description=_DESCRIPTION, - features=features, - homepage=_HOMEPAGE, - license=_LICENSE, - citation=_CITATION, + description=_DESCRIPTION, features=features, homepage=_HOMEPAGE, license=_LICENSE, citation=_CITATION, ) def _split_generators(self, dl_manager) -> List[datasets.SplitGenerator]: """Returns SplitGenerators.""" - # TODO: This method is tasked with downloading/extracting the data and defining the splits depending on the configuration - - # If you need to access the "source" or "bigbio" config choice, that will be in self.config.name - - # LOCAL DATASETS: You do not need the dl_manager; you can ignore this argument. Make sure `gen_kwargs` in the return gets passed the right filepath - - # PUBLIC DATASETS: Assign your data-dir based on the dl_manager. train_fpaths = { - 'train_xml/ADCETRIS.xml', - 'train_xml/BEPREVE.xml', - 'train_xml/CLEVIPREX.xml', - 'train_xml/DYSPORT.xml', - 'train_xml/FERRIPROX.xml', - 'train_xml/ILARIS.xml', - 'train_xml/KALYDECO.xml', - 'train_xml/ONFI.xml', - 'train_xml/SAPHRIS.xml', - 'train_xml/TIVICAY.xml', - 'train_xml/VIZAMYL.xml', - 'train_xml/ZYKADIA.xml', - 'train_xml/ADREVIEW.xml', - 'train_xml/BESIVANCE.xml', - 'train_xml/COARTEM.xml', - 'train_xml/EDARBI.xml', - 'train_xml/FIRAZYR.xml', - 'train_xml/IMBRUVICA.xml', - 'train_xml/KYPROLIS.xml', - 'train_xml/OTEZLA.xml', - 'train_xml/SIMPONI.xml', - 'train_xml/TOVIAZ.xml', - 'train_xml/VORAXAZE.xml', - 'train_xml/ZYTIGA.xml', - 'train_xml/AFINITOR.xml', - 'train_xml/BLINCYTO.xml', - 'train_xml/COMETRIQ.xml', - 'train_xml/ELIQUIS.xml', - 'train_xml/FULYZAQ.xml', - 'train_xml/INLYTA.xml', - 'train_xml/LUMIZYME.xml', - 'train_xml/PICATO.xml', - 'train_xml/SIRTURO.xml', - 'train_xml/TREANDA.xml', - 'train_xml/XALKORI.xml', - 'train_xml/AMPYRA.xml', - 'train_xml/BOSULIF.xml', - 'train_xml/DALVANCE.xml', - 'train_xml/ENTEREG.xml', - 'train_xml/GADAVIST.xml', - 'train_xml/INTELENCE.xml', - 'train_xml/MULTAQ.xml', - 'train_xml/POTIGA.xml', - 'train_xml/STENDRA.xml', - 'train_xml/TRULICITY.xml', - 'train_xml/XEOMIN.xml', - 'train_xml/AMYVID.xml', - 'train_xml/BREO.xml', - 'train_xml/DATSCAN.xml', - 'train_xml/EOVIST.xml', - 'train_xml/GILENYA.xml', - 'train_xml/INVOKANA.xml', - 'train_xml/NATAZIA.xml', - 'train_xml/PRADAXA.xml', - 'train_xml/STRIBILD.xml', - 'train_xml/TUDORZA.xml', - 'train_xml/XIAFLEX.xml', - 'train_xml/APTIOM.xml', - 'train_xml/CARBAGLU.xml', - 'train_xml/DIFICID.xml', - 'train_xml/ERWINAZE.xml', - 'train_xml/GILOTRIF.xml', - 'train_xml/JARDIANCE.xml', - 'train_xml/NESINA.xml', - 'train_xml/PRISTIQ.xml', - 'train_xml/TAFINLAR.xml', - 'train_xml/ULESFIA.xml', - 'train_xml/XTANDI.xml', - 'train_xml/ARCAPTA.xml', - 'train_xml/CERDELGA.xml', - 'train_xml/DOTAREM.xml', - 'train_xml/EYLEA.xml', - 'train_xml/GRANIX.xml', - 'train_xml/JEVTANA.xml', - 'train_xml/NEURACEQ.xml', - 'train_xml/PROLIA.xml', - 'train_xml/TANZEUM.xml', - 'train_xml/ULORIC.xml', - 'train_xml/YERVOY.xml', - 'train_xml/BELEODAQ.xml', - 'train_xml/CHOLINE.xml', - 'train_xml/DUAVEE.xml', - 'train_xml/FANAPT.xml', - 'train_xml/HALAVEN.xml', - 'train_xml/JUBLIA.xml', - 'train_xml/NORTHERA.xml', - 'train_xml/PROMACTA.xml', - 'train_xml/TECFIDERA.xml', - 'train_xml/VICTRELIS.xml', - 'train_xml/ZERBAXA.xml', - 'train_xml/BENLYSTA.xml', - 'train_xml/CIMZIA.xml', - 'train_xml/DUREZOL.xml', - 'train_xml/FARXIGA.xml', - 'train_xml/HORIZANT.xml', - 'train_xml/KALBITOR.xml', - 'train_xml/NULOJIX.xml', - 'train_xml/QUTENZA.xml', - 'train_xml/TEFLARO.xml', - 'train_xml/VIMIZIM.xml', - 'train_xml/ZYDELIG.xml' + "train_xml/ADCETRIS.xml", + "train_xml/BEPREVE.xml", + "train_xml/CLEVIPREX.xml", + "train_xml/DYSPORT.xml", + "train_xml/FERRIPROX.xml", + "train_xml/ILARIS.xml", + "train_xml/KALYDECO.xml", + "train_xml/ONFI.xml", + "train_xml/SAPHRIS.xml", + "train_xml/TIVICAY.xml", + "train_xml/VIZAMYL.xml", + "train_xml/ZYKADIA.xml", + "train_xml/ADREVIEW.xml", + "train_xml/BESIVANCE.xml", + "train_xml/COARTEM.xml", + "train_xml/EDARBI.xml", + "train_xml/FIRAZYR.xml", + "train_xml/IMBRUVICA.xml", + "train_xml/KYPROLIS.xml", + "train_xml/OTEZLA.xml", + "train_xml/SIMPONI.xml", + "train_xml/TOVIAZ.xml", + "train_xml/VORAXAZE.xml", + "train_xml/ZYTIGA.xml", + "train_xml/AFINITOR.xml", + "train_xml/BLINCYTO.xml", + "train_xml/COMETRIQ.xml", + "train_xml/ELIQUIS.xml", + "train_xml/FULYZAQ.xml", + "train_xml/INLYTA.xml", + "train_xml/LUMIZYME.xml", + "train_xml/PICATO.xml", + "train_xml/SIRTURO.xml", + "train_xml/TREANDA.xml", + "train_xml/XALKORI.xml", + "train_xml/AMPYRA.xml", + "train_xml/BOSULIF.xml", + "train_xml/DALVANCE.xml", + "train_xml/ENTEREG.xml", + "train_xml/GADAVIST.xml", + "train_xml/INTELENCE.xml", + "train_xml/MULTAQ.xml", + "train_xml/POTIGA.xml", + "train_xml/STENDRA.xml", + "train_xml/TRULICITY.xml", + "train_xml/XEOMIN.xml", + "train_xml/AMYVID.xml", + "train_xml/BREO.xml", + "train_xml/DATSCAN.xml", + "train_xml/EOVIST.xml", + "train_xml/GILENYA.xml", + "train_xml/INVOKANA.xml", + "train_xml/NATAZIA.xml", + "train_xml/PRADAXA.xml", + "train_xml/STRIBILD.xml", + "train_xml/TUDORZA.xml", + "train_xml/XIAFLEX.xml", + "train_xml/APTIOM.xml", + "train_xml/CARBAGLU.xml", + "train_xml/DIFICID.xml", + "train_xml/ERWINAZE.xml", + "train_xml/GILOTRIF.xml", + "train_xml/JARDIANCE.xml", + "train_xml/NESINA.xml", + "train_xml/PRISTIQ.xml", + "train_xml/TAFINLAR.xml", + "train_xml/ULESFIA.xml", + "train_xml/XTANDI.xml", + "train_xml/ARCAPTA.xml", + "train_xml/CERDELGA.xml", + "train_xml/DOTAREM.xml", + "train_xml/EYLEA.xml", + "train_xml/GRANIX.xml", + "train_xml/JEVTANA.xml", + "train_xml/NEURACEQ.xml", + "train_xml/PROLIA.xml", + "train_xml/TANZEUM.xml", + "train_xml/ULORIC.xml", + "train_xml/YERVOY.xml", + "train_xml/BELEODAQ.xml", + "train_xml/CHOLINE.xml", + "train_xml/DUAVEE.xml", + "train_xml/FANAPT.xml", + "train_xml/HALAVEN.xml", + "train_xml/JUBLIA.xml", + "train_xml/NORTHERA.xml", + "train_xml/PROMACTA.xml", + "train_xml/TECFIDERA.xml", + "train_xml/VICTRELIS.xml", + "train_xml/ZERBAXA.xml", + "train_xml/BENLYSTA.xml", + "train_xml/CIMZIA.xml", + "train_xml/DUREZOL.xml", + "train_xml/FARXIGA.xml", + "train_xml/HORIZANT.xml", + "train_xml/KALBITOR.xml", + "train_xml/NULOJIX.xml", + "train_xml/QUTENZA.xml", + "train_xml/TEFLARO.xml", + "train_xml/VIMIZIM.xml", + "train_xml/ZYDELIG.xml", } - # dl_manager is a datasets.download.DownloadManager that can be used to download and extract URLs; many examples use the download_and_extract method; see the DownloadManager docs here: https://huggingface.co/docs/datasets/package_reference/builder_classes.html#datasets.DownloadManager - - # dl_manager can accept any type of nested list/dict and will give back the same structure with the url replaced with the path to local files. - urls = _URLS[_DATASETNAME] data_dir = dl_manager.download_and_extract(urls) return [ datasets.SplitGenerator( name=datasets.Split.TRAIN, - gen_kwargs= { - "files": [os.path.join(data_dir, path) for path in train_fpaths], - "split": "train", - }, + gen_kwargs={"files": [os.path.join(data_dir, path) for path in train_fpaths], "split": "train"}, ), - ] - - - # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` - - # TODO: change the args of this function to match the keys in `gen_kwargs`. You may add any necessary kwargs. + ] + + def _generate_example_sections(self, uid, source_sections_tree): + """ + Parse sections XML + + Parameters + ---------- + uid : int + unique identifier being updated with each execution + source_sections_tree : etree object + XML of drug label sections + + Returns + ---------- + int + updated unique identifier + dict + drug label sections information + """ + + sections = [] + + for source_section in source_sections_tree: + source_section_id = source_section.attrib["id"] + source_section_name = source_section.attrib["name"] + source_section_text = source_section.text + section = {"id": source_section_id, "name": source_section_name, "section_text": source_section_text} + sections.append(section) + uid += 1 + + return (uid, sections) + + def _generate_example_mentions(self, uid, source_mentions_tree): + """ + Parse mentions XML + + Parameters + ---------- + uid : int + unique identifier being updated with each execution + source_mentions_tree : etree object + XML of drug label ADR mentions + + Returns + ---------- + int + updated unique identifier + dict + ADR mentions information + """ + mentions = [] + for source_mention in source_mentions_tree: + source_mention_id = source_mention.attrib["id"] + source_mention_section_id = source_mention.attrib["section"] + source_mention_type = source_mention.attrib["type"] + source_mention_start = source_mention.attrib["start"] + source_mention_len = source_mention.attrib["len"] + source_mention_str = source_mention.attrib["str"] + + if "," in source_mention_start: + source_mention_starts = source_mention_start.split(",") + source_mention_lens = source_mention_len.split(",") + i = 0 + for source_mention_start in source_mention_starts: + mention = { + "id": str(uid), + "source_id": source_mention_id, + "type": source_mention_type, + "section_id": source_mention_section_id, + "start": int(source_mention_start), + "len": int(source_mention_lens[i]), + "str": source_mention_str, + } + mentions.append(mention) + uid += 1 + i+=1 + else: + mention = { + "id": str(uid), + "source_id": source_mention_id, + "type": source_mention_type, + "section_id": source_mention_section_id, + "start": int(source_mention_start), + "len": int(source_mention_len), + "str": source_mention_str, + } + mentions.append(mention) + uid += 1 + return (uid, mentions) + + def _generate_example_relations(self, uid, source_relations_tree): + """ + Parse relations XML + + Parameters + ---------- + uid : int + unique identifier being updated with each execution + source_relations_tree : etree object + XML of drug label ADR relations + + Returns + ---------- + int + updated unique identifier + dict + drug label relations information + """ + relations = [] + for source_relation in source_relations_tree: + source_relation_id = source_relation.attrib["id"] + source_relation_type = source_relation.attrib["type"] + source_relation_arg1 = source_relation.attrib["arg1"] + source_relation_arg2 = source_relation.attrib["arg2"] + + relation = { + "id": source_relation_id, + "type": source_relation_type, + "arg1": source_relation_arg1, + "arg2": source_relation_arg2, + } + relations.append(relation) + uid += 1 + return (uid, relations) + + def _generate_example_reactions(self, uid, source_reactions_tree): + """ + Parse reactions XML + + Parameters + ---------- + uid : int + unique identifier being updated with each execution + source_reactions_tree : etree object + XML of drug label reaction normalizations to MedDRA terms + + Returns + ---------- + int + updated unique identifier + dict + reactions normalization information + """ + reactions = [] + for source_reaction in source_reactions_tree: + source_reaction_id = source_reaction.attrib["id"] + source_reaction_str = source_reaction.attrib["str"] + + reaction = {"id": source_reaction_id, "str": source_reaction_str} + + for source_reaction_normalization in source_reaction: + source_reaction_normalization_id = source_reaction_normalization.attrib["id"] + + normalization = {"id": source_reaction_normalization_id} + + if "meddra_pt" in source_reaction_normalization.attrib: + source_reaction_normalization_meddra_pt = source_reaction_normalization.attrib["meddra_pt"] + source_reaction_normalization_meddra_pt_id = source_reaction_normalization.attrib["meddra_pt_id"] + normalization["meddra_pt"] = source_reaction_normalization_meddra_pt + normalization["meddra_pt_id"] = source_reaction_normalization_meddra_pt_id + else: + normalization["meddra_pt"] = "" + normalization["meddra_pt_id"] = "" + + if "meddra_llt" in source_reaction_normalization.attrib: + source_reaction_normalization_meddra_llt = source_reaction_normalization.attrib["meddra_llt"] + source_reaction_normalization_meddra_llt_id = source_reaction_normalization.attrib["meddra_llt_id"] + normalization["meddra_llt"] = source_reaction_normalization_meddra_llt + normalization["meddra_llt_id"] = source_reaction_normalization_meddra_llt_id + else: + normalization["meddra_llt"] = "" + normalization["meddra_llt_id"] = "" + reaction["normalization"] = normalization + uid += 1 + reactions.append(reaction) + uid += 1 + return (uid, reactions) + + def _generate_example_kb_passages(self, uid, drug_name, source_sections_tree): + """ + Parse sections XML into passages for KB schema + + Parameters + ---------- + uid : int + unique identifier being updated with each execution + source_sections_tree : etree object + XML of drug label sections + + Returns + ---------- + int + updated unique identifier + dict + KB schema passages information + """ + passages = [] + + overall_text = "" + + for source_section in source_sections_tree: + passage_id = drug_name + "_" + source_section.attrib["id"] + passage_type = source_section.attrib["name"] + passage_text = source_section.text + + passage_offsets = (len(overall_text), len(passage_text)+len(overall_text)) + passage = {"id": passage_id, "type": passage_type, "text": [passage_text], "offsets": [passage_offsets]} + passages.append(passage) + uid += 1 + overall_text = overall_text+passage_text + + return (uid, passages) + + def _generate_example_kb_entities(self, uid, drug_name, source_mentions_tree, normalizations, passages): + """ + Parse mentions XML into entities for KB schema, including normalizations from source "reactions" data + + Parameters + ---------- + uid : int + unique identifier being updated with each execution + source_mentions_tree : etree object + XML of drug label ADR mentions + + Returns + ---------- + int + updated unique identifier + dict + KB schema entities information + """ + entities = [] + for source_mention in source_mentions_tree: + entity_type = source_mention.attrib["type"] + entity_text = source_mention.attrib["str"] + entity_id = source_mention.attrib["id"] + passage = source_mention.attrib["section"] + + passage_num = int(passage[1:]) + + pretext = "" + + if passage_num>1: + texts = [passage["text"][0] for passage in passages for i in range(0,passage_num-1) if passage["id"] == "S"+str(i)] + pretext = "".join(text for text in texts) + + relevant_normalizations = [ + normalization for normalization in normalizations if entity_text == normalization["str"] + ] + + source_mention_start = source_mention.attrib["start"] + source_mention_len = source_mention.attrib["len"] + + if "," in source_mention_start: + source_mention_starts = source_mention_start.split(",") + source_mention_lens = source_mention_len.split(",") + + for source_mention_start in source_mention_starts: + entity = { + "id": drug_name + "_" + entity_id, + "type": entity_type, + "text": [entity_text], + "offsets": [ + [len(pretext) + int(source_mention_start), len(pretext) + int(source_mention_start) + int(source_mention_lens[0])] + ], + "normalized": [ + {"db_name": "meddra", "db_id": rn["meddra_id"]} for rn in relevant_normalizations + ], + } + entities.append(entity) + uid += 1 + else: + entity = { + "id": drug_name + "_" + entity_id, + "type": entity_type, + "text": [entity_text], + "offsets": [[len(pretext) + int(source_mention_start), len(pretext) + int(source_mention_start) + int(source_mention_len)]], + "normalized": [{"db_name": "meddra", "db_id": rn["meddra_id"]} for rn in relevant_normalizations], + } + entities.append(entity) + uid += 1 + return (uid, entities) + + def _generate_kb_entity_normalizations(self, uid, source_reactions_tree): + """ + Parse reactions XML into entity normalizations for KB schema + + Parameters + ---------- + uid : int + unique identifier being updated with each execution + source_reactions_tree : etree object + XML of drug label reaction normalizations to MedDRA terms + + Returns + ---------- + int + updated unique identifier + dict + reactions normalization information for KB schema + """ + normalizations = [] + for source_reaction in source_reactions_tree: + + for source_reaction_normalization in source_reaction: + if "meddra_pt" in source_reaction_normalization.attrib: + normalized_term = {} + normalized_term["str"] = source_reaction.attrib["str"] + source_reaction_normalization_meddra_pt_id = source_reaction_normalization.attrib["meddra_pt_id"] + normalized_term["meddra_id"] = source_reaction_normalization_meddra_pt_id + normalizations.append(normalized_term) + + if "meddra_llt" in source_reaction_normalization.attrib: + normalized_term = {} + normalized_term["str"] = source_reaction.attrib["str"] + source_reaction_normalization_meddra_llt_id = source_reaction_normalization.attrib["meddra_llt_id"] + normalized_term["meddra_id"] = source_reaction_normalization_meddra_llt_id + normalizations.append(normalized_term) + return(uid, normalizations) + + def _generate_example_kb_relations(self, uid, drug_name, source_relations_tree): + """ + Parse relations XML for KB schema + + Parameters + ---------- + uid : int + unique identifier being updated with each execution + source_relations_tree : etree object + XML of drug label ADR relations + + Returns + ---------- + int + updated unique identifier + dict + drug label relations information for KB schema + """ + relations = [] + for source_relation in source_relations_tree: + source_relation_id = source_relation.attrib["id"] + source_relation_type = source_relation.attrib["type"] + source_relation_arg1 = source_relation.attrib["arg1"] + source_relation_arg2 = source_relation.attrib["arg2"] + + relation = { + "id": source_relation_id, + "type": source_relation_type, + "arg1_id": drug_name + "_" + source_relation_arg1, + "arg2_id": drug_name + "_" + source_relation_arg2, + "normalized": [], + } + uid += 1 + relations.append(relation) + return (uid, relations) def _generate_examples(self, files, split: str) -> Tuple[int, Dict]: """Yields examples as (key, example) tuples.""" - # TODO: This method handles input defined in _split_generators to yield (key, example) tuples from the dataset. - - # The `key` is for legacy reasons (tfds) and is not important in itself, but must be unique for each example. - - # NOTE: For local datasets you will have access to self.config.data_dir and self.config.data_files - print(files) - - if self.config.schema == "source": - for file in files: - with open(file) as xml_file: - tree = ET.parse(xml_file) - label = tree.getroot() - #print(label.attrib["drug"]) - uid = 0 - for child in label: - example = {} - if child.tag == "Text": - text = child - for section in text: - example["section"] = section - print(section.attrib["id"]) - print(section.text) - #yield uid, example - uid +=1 - - # # TODO: yield (key, example) tuples in the original dataset schema - # for key, example in thing: - # yield key, example + uid = 0 + + for file in files: + with open(file) as xml_file: + source_tree = ET.parse(xml_file) + source_label = source_tree.getroot() + source_drug_name = source_label.attrib["drug"] + + source_text = [source_child for source_child in source_label if source_child.tag == "Text"][0] + source_mentions = [source_child for source_child in source_label if source_child.tag == "Mentions"][0] + source_relations = [source_child for source_child in source_label if source_child.tag == "Relations"][ + 0 + ] + source_reactions = [source_child for source_child in source_label if source_child.tag == "Reactions"][ + 0 + ] + + if self.config.schema == "source": + example = {"drug": source_drug_name, "text": {}, "mentions": [], "relations": [], "reactions": []} + uid, sections = self._generate_example_sections(uid, source_text) + example["text"] = {"sections": sections} + + uid, mentions = self._generate_example_mentions(uid, source_mentions) + example["mentions"] = mentions + + uid, relations = self._generate_example_relations(uid, source_relations) + example["relations"] = relations + + uid, reactions = self._generate_example_reactions(uid, source_reactions) + example["reactions"] = reactions + yield uid, example + + elif self.config.schema == "bigbio_kb": + + example = { + "id": uid, + "document_id": source_drug_name, + "passages": [], + "entities": [], + "relations": [], + "events": [], + "coreferences": [], + } - elif self.config.schema == "bigbio_kb": - # TODO: yield (key, example) tuples in the bigbio schema - for key, example in thing: - yield key, example + uid, entity_normalizations = self._generate_kb_entity_normalizations(uid, source_reactions) + uid, passages = self._generate_example_kb_passages(uid, source_drug_name, source_text) + example["passages"] = passages -# This template is based on the following template from the datasets package: -# https://github.com/huggingface/datasets/blob/master/templates/new_dataset_script.py + uid, entities = self._generate_example_kb_entities( + uid, source_drug_name, source_mentions, entity_normalizations, passages + ) + example["entities"] = entities + uid, relations = self._generate_example_kb_relations(uid, source_drug_name, source_relations) + example["relations"] = relations -# This allows you to run your dataloader with `python [dataset_name].py` during development -# TODO: Remove this before making your PR -if __name__ == "__main__": - datasets.load_dataset(__file__) + yield uid, example