Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Cu 1yn0v9e duplicate multiprocessing methods #364

Merged
merged 3 commits into from
Nov 27, 2023
Merged

Cu 1yn0v9e duplicate multiprocessing methods #364

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b19cecf
CU-1yn0v9e: Rename and deprecate one of the multiprocessing methods;
mart-r Oct 23, 2023
bf7f756
CU-1yn0v9e: Rename and deprecate the multiprocessing_pipe method;
mart-r Oct 23, 2023
db1f6f5
CU-1yn0v9e: Fix typo in docstring; more consistent naming
mart-r Oct 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 45 additions & 2 deletions medcat/cat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1327,6 +1327,7 @@ def _save_docs_to_file(self, docs: Iterable, annotated_ids: List[str], save_dir_
pickle.dump((annotated_ids, part_counter), open(annotated_ids_path, 'wb'))
return part_counter

@deprecated(message="Use `multiprocessing_batch_char_size` instead")
def multiprocessing(self,
data: Union[List[Tuple], Iterable[Tuple]],
nproc: int = 2,
Expand All @@ -1337,9 +1338,31 @@ def multiprocessing(self,
out_split_size_chars: Optional[int] = None,
save_dir_path: str = os.path.abspath(os.getcwd()),
min_free_memory=0.1) -> Dict:
return self.multiprocessing_batch_char_size(data=data, nproc=nproc,
batch_size_chars=batch_size_chars,
only_cui=only_cui, addl_info=addl_info,
separate_nn_components=separate_nn_components,
out_split_size_chars=out_split_size_chars,
save_dir_path=save_dir_path,
min_free_memory=min_free_memory)

def multiprocessing_batch_char_size(self,
data: Union[List[Tuple], Iterable[Tuple]],
nproc: int = 2,
batch_size_chars: int = 5000 * 1000,
only_cui: bool = False,
addl_info: List[str] = [],
separate_nn_components: bool = True,
out_split_size_chars: Optional[int] = None,
save_dir_path: str = os.path.abspath(os.getcwd()),
min_free_memory=0.1) -> Dict:
r"""Run multiprocessing for inference, if out_save_path and out_split_size_chars is used this will also continue annotating
documents if something is saved in that directory.

This method batches the data based on the number of characters as specified by user.

PS: This method is unlikely to work on a Windows machine.

Args:
data:
Iterator or array with format: [(id, text), (id, text), ...]
Expand Down Expand Up @@ -1523,15 +1546,35 @@ def _multiprocessing_batch(self,

return docs

def multiprocessing_pipe(self,
@deprecated(message="Use `multiprocessing_batch_nr_of_docs` instead")
def multiprocessing_pipe(self, in_data: Union[List[Tuple], Iterable[Tuple]],
nproc: Optional[int] = None,
batch_size: Optional[int] = None,
only_cui: bool = False,
addl_info: List[str] = [],
return_dict: bool = True,
batch_factor: int = 2) -> Union[List[Tuple], Dict]:
return self.multiprocessing_batch_nr_of_docs(in_data=in_data, nproc=nproc,
batch_size=batch_size,
only_cui=only_cui,
addl_info=addl_info,
return_dict=return_dict,
batch_factor=batch_factor)

def multiprocessing_batch_nr_of_docs(self,
Copy link
Member

@tomolopolis tomolopolis Oct 30, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe rename to multiprocessing_batch_docs_size for consistency with above?

in_data: Union[List[Tuple], Iterable[Tuple]],
nproc: Optional[int] = None,
batch_size: Optional[int] = None,
only_cui: bool = False,
addl_info: List[str] = [],
return_dict: bool = True,
batch_factor: int = 2) -> Union[List[Tuple], Dict]:
"""Run multiprocessing NOT FOR TRAINING
"""Run multiprocessing NOT FOR TRAINING.

Thios method batches the data based on the number of documents as specified by the user.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

typo


PS:
This method supports Windows.

Args:
in_data (Union[List[Tuple], Iterable[Tuple]]): List with format: [(id, text), (id, text), ...]
Expand Down
8 changes: 4 additions & 4 deletions tests/test_cat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ def test_multiprocessing(self):
(2, ""),
(3, None)
]
out = self.undertest.multiprocessing(in_data, nproc=1)
out = self.undertest.multiprocessing_batch_char_size(in_data, nproc=1)

self.assertEqual(3, len(out))
self.assertEqual(1, len(out[1]['entities']))
Expand All @@ -73,7 +73,7 @@ def test_multiprocessing_pipe(self):
(2, "The dog is sitting outside the house."),
(3, "The dog is sitting outside the house."),
]
out = self.undertest.multiprocessing_pipe(in_data, nproc=2, return_dict=False)
out = self.undertest.multiprocessing_batch_nr_of_docs(in_data, nproc=2, return_dict=False)
self.assertTrue(type(out) == list)
self.assertEqual(3, len(out))
self.assertEqual(1, out[0][0])
Expand All @@ -89,7 +89,7 @@ def test_multiprocessing_pipe_with_malformed_texts(self):
(2, ""),
(3, None),
]
out = self.undertest.multiprocessing_pipe(in_data, nproc=1, batch_size=1, return_dict=False)
out = self.undertest.multiprocessing_batch_nr_of_docs(in_data, nproc=1, batch_size=1, return_dict=False)
self.assertTrue(type(out) == list)
self.assertEqual(3, len(out))
self.assertEqual(1, out[0][0])
Expand All @@ -105,7 +105,7 @@ def test_multiprocessing_pipe_return_dict(self):
(2, "The dog is sitting outside the house."),
(3, "The dog is sitting outside the house.")
]
out = self.undertest.multiprocessing_pipe(in_data, nproc=2, return_dict=True)
out = self.undertest.multiprocessing_batch_nr_of_docs(in_data, nproc=2, return_dict=True)
self.assertTrue(type(out) == dict)
self.assertEqual(3, len(out))
self.assertEqual({'entities': {}, 'tokens': []}, out[1])
Expand Down
Loading