add argument to get_tokenizer function #103

	name: Run unittests
	on:
	push:
	branches:
	- main
	pull_request:
	branches:
	- main
	jobs:
	test:
	name: Test
	runs-on: ubuntu-latest

	steps:
	- uses: actions/checkout@v2

	- name: Set up Python
	uses: actions/setup-python@v2
	with:
	python-version: '3.10'

	- name: Set up Conda
	uses: conda-incubator/setup-miniconda@v2
	with:
	auto-update-conda: true
	environment-file: environment.yml
	activate-environment: testenv

	- name: Run unittests
	shell: bash -l {0}
	run: python3 confirm_test.py --confirm

Provide feedback