Skip to content

Commit

Permalink
feat: catch error (#17)
Browse files Browse the repository at this point in the history
* feat: catch error

* feat: apply black
  • Loading branch information
samsja authored Aug 10, 2023
1 parent 4636fa1 commit 8e66273
Showing 1 changed file with 22 additions and 10 deletions.
32 changes: 22 additions & 10 deletions textbook/dataset_gen/dataset_gen.py
Original file line number Diff line number Diff line change
Expand Up @@ -191,19 +191,31 @@ def mass_generation(
"•",
TimeElapsedColumn(),
) as progress:

def update_progress():
progress.update(task, advance=1)

with ThreadPoolExecutor(max_workers=pool_size) as executor:
task = progress.add_task("[red]Generating...", total=len(prompts))

def map_fn(prompt):
_generation_wrapper(
prompt, get_generator, update_progress, save_dir, retries
progress_task = progress.add_task("[red]Generating...", total=len(prompts))

def update_progress():
progress.update(progress_task, advance=1)

tasks = []

for prompt in prompts:
tasks.append(
executor.submit(
_generation_wrapper,
prompt,
get_generator,
update_progress,
save_dir,
retries,
)
)

list(executor.map(map_fn, prompts))
for task in tasks:
try:
task.result()
except Exception as e:
raise e


def load_prompts(file: str, key_prompt: str = "prompt") -> List[str]:
Expand Down

0 comments on commit 8e66273

Please sign in to comment.