diff --git a/.env b/.env index 2c6458c..be9985f 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -JUPYTER_PORT=43603 +JUPYTER_PORT=43604 diff --git a/Dockerfile b/Dockerfile index 502a9c9..b5d1294 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,31 @@ -FROM python:3.10 +# FROM python:3.10 +FROM nvidia/cuda:11.7.0-cudnn8-runtime-ubuntu22.04 +ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update +RUN apt-get install python3.10 python3-pip -y +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 +# Install poetry +RUN python3 -m pip install poetry + +# cv2 dependencies +RUN apt-get install ffmpeg libsm6 libxext6 -y # Install opencv-python dependency -RUN apt-get install libgl1 -y +# RUN apt-get install libgl1 -y # Install poppler for pdf2image (converting pdf to images) RUN apt-get install poppler-utils -y -# Install poetry -RUN pip3 install poetry +# # Install poetry +# RUN pip3 install poetry WORKDIR /app COPY poetry.lock pyproject.toml /app/ COPY docile /app/docile +COPY baselines /app/baselines COPY README.md /app/README.md -RUN poetry install --no-interaction --with test --with doctr +# RUN poetry install --no-interaction --with test --with doctr +RUN poetry install --no-interaction --with test --with doctr --with baselines diff --git a/docile/evaluation/evaluate.py b/docile/evaluation/evaluate.py index 2bdd6d0..7fcd9bb 100644 --- a/docile/evaluation/evaluate.py +++ b/docile/evaluation/evaluate.py @@ -72,6 +72,23 @@ def from_file(cls, path: Path) -> "EvaluationResult": } return cls(matchings, dct["dataset_name"], dct["iou_threshold"]) + @classmethod + def from_files(cls, *paths: Sequence[Path]) -> "EvaluationResult": + """Load evaluation results for different tasks from multiple files at once.""" + if len(paths) == 0: + raise ValueError("At least one path must be provided") + evaluations = [cls.from_file(path) for path in paths] + if len(set(evaluation.dataset_name for evaluation in evaluations)) != 1: + raise ValueError("Cannot load evaluations on different datasets") + if len(set(evaluation.iou_threshold for evaluation in evaluations)) != 1: + raise ValueError("Cannot load evaluations that used different config (iou_threshold)") + all_matchings = {} + for evaluation in evaluations: + if not set(all_matchings.keys()).isdisjoint(evaluation.task_to_docid_to_matching): + raise ValueError("Tasks in the evaluations are not disjoint") + all_matchings.update(evaluation.task_to_docid_to_matching) + return cls(all_matchings, evaluations[0].dataset_name, evaluations[0].iou_threshold) + def get_primary_metric(self, task: str) -> float: """Return the primary metric used for DocILE'23 benchmark competition.""" metric = TASK_TO_PRIMARY_METRIC_NAME[task] diff --git a/docile/tools/dbg_my_browser_for_QA.ipynb b/docile/tools/dbg_my_browser_for_QA.ipynb new file mode 100644 index 0000000..7cf362b --- /dev/null +++ b/docile/tools/dbg_my_browser_for_QA.ipynb @@ -0,0 +1,277 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "8615055c-348b-4801-bf7a-0c87592a374d", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from pathlib import Path\n", + "from docile.dataset import Dataset\n", + "from docile.dataset import Field\n", + "from docile.tools.my_dataset_browser import MyDatasetBrowser, load_predictions\n", + "from docile.evaluation import EvaluationResult" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "253f0673-211c-4f84-a98e-5c7a5943814d", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Loading documents for docile221221-0:test: 100%|██████████| 1000/1000 [00:04<00:00, 221.36it/s]\n" + ] + } + ], + "source": [ + "DATASET_PATH = Path(\"/storage/pif_documents/dataset_exports/docile221221-0/\")\n", + "dataset = Dataset(\"test\", DATASET_PATH)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "2caa8073-d8f9-4547-afd1-6e8f8f41a567", + "metadata": {}, + "outputs": [], + "source": [ + "from docile.evaluation.evaluate import evaluate_dataset\n", + "\n", + "#intermediate_predictions = load_predictions(Path(\"/storage/table_extraction/predictions/NER/fullpage_multilabel/docile221221-0/LayoutLMv3_wr025/v2/test_intermediate_predictions.json\"))\n", + "kile_predictions = load_predictions(Path(\"/storage/table_extraction/predictions/NER/fullpage_multilabel/docile221221-0/LayoutLMv3_wr025/v2/test_predictions_KILE.json\"))\n", + "lir_predictions = load_predictions(Path(\"/storage/table_extraction/predictions/NER/fullpage_multilabel/docile221221-0/LayoutLMv3_wr025/v2/test_predictions_LIR.json\"))\n", + "\n", + "intermediate_predictions = load_predictions(Path(\"/storage/table_extraction/predictions/NER/fullpage_multilabel/docile221221-0/RoBERTa_base_gas4_wr01_stride_128_new2DposEmb/test_intermediate_predictions.json\"))\n", + "kile_predictions = load_predictions(Path(\"/storage/table_extraction/predictions/NER/fullpage_multilabel/docile221221-0/RoBERTa_base_gas4_wr01_stride_128_new2DposEmb/test_predictions_KILE.json\"))\n", + "lir_predictions = load_predictions(Path(\"/storage/table_extraction/predictions/NER/fullpage_multilabel/docile221221-0/RoBERTa_base_gas4_wr01_stride_128_new2DposEmb/test_predictions_LIR.json\"))" + ] + }, + { + "cell_type": "raw", + "id": "bcfcf9c9-c457-4872-b8cf-8314761dbcc8", + "metadata": {}, + "source": [ + "evaluation_result_KILE = evaluate_dataset(dataset, kile_predictions, {})\n", + "evaluation_result_LIR = evaluate_dataset(dataset, {}, lir_predictions)\n", + "\n", + "evaluation_result_KILE.to_file(Path(\"/storage/table_extraction/predictions/NER/fullpage_multilabel/docile221221-0/LayoutLMv3_wr025/v2/test_results_KILE.json\"))\n", + "evaluation_result_LIR.to_file(Path(\"/storage/table_extraction/predictions/NER/fullpage_multilabel/docile221221-0/LayoutLMv3_wr025/v2/test_results_LIR.json\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "ece51b3b-d627-455a-90d0-b146d7c21de7", + "metadata": {}, + "outputs": [], + "source": [ + "#EVALUATION_PATHS = [\n", + "# Path(\"/storage/table_extraction/predictions/NER/fullpage_multilabel/docile221221-0/LayoutLMv3_wr025/v2/test_results_KILE.json\"), \n", + "# Path(\"/storage/table_extraction/predictions/NER/fullpage_multilabel/docile221221-0/LayoutLMv3_wr025/v2/test_results_LIR.json\")\n", + "#]\n", + "\n", + "EVALUATION_PATHS = [\n", + " Path(\"/storage/table_extraction/predictions/NER/fullpage_multilabel/docile221221-0/RoBERTa_base_gas4_wr01_stride_128_new2DposEmb/test_results_KILE.json\"), \n", + " Path(\"/storage/table_extraction/predictions/NER/fullpage_multilabel/docile221221-0/RoBERTa_base_gas4_wr01_stride_128_new2DposEmb/test_results_LIR.json\")\n", + "]\n", + "\n", + "evaluation_results = EvaluationResult.from_files(*EVALUATION_PATHS)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "62633bd0-6f38-4ee0-805f-8fdeb7e0a34a", + "metadata": {}, + "outputs": [], + "source": [ + "#kile_predictions" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "b790b8c1-9a48-4a4e-9979-318c7356d446", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "28af6ac79d3842ec9e2d1235514cd576", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(HBox(children=(Button(icon='arrow-left', layout=Layout(flex='0 0 auto', width='auto'), style=Bu…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "callbacks = [\"Annotations_KILE\", \"Annotations_LIR\", \"Predictions_KILE\", \"Predictions_LIR\", \"Predictions_intermediate\"]\n", + "browser = MyDatasetBrowser(dataset, evaluation_results=evaluation_results, kile_predictions=kile_predictions, lir_predictions=lir_predictions, intermediate_predictions=intermediate_predictions, callbacks=callbacks)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5a103a17-104d-4513-84ad-c999779a45ec", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "6ba97aef-be4f-4fd5-980e-c68641e49a46", + "metadata": {}, + "outputs": [], + "source": [ + "kile_f1 = [browser.evaluation_results.get_metrics(\"kile\", docid=x.docid)[\"f1\"] for x in browser.dataset]" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "df7ad5ac-d42a-48a0-b490-a7fb8b78fee2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA280lEQVR4nO3deXxU5d3///dMSCYJZCXJhIRAEJFFdgJpRFxqalyKtbW9KVJBavWhRUVz3xZwgZt629C6lLai1AW9f1+roK1bFbHcUURqBFmioAIiSxBICIQwIYFJMnN+fwRGAwlkwsycWV7Px4PHI3POdWY+c6E5b65znetYDMMwBAAAYBKr2QUAAIDIRhgBAACmIowAAABTEUYAAICpCCMAAMBUhBEAAGAqwggAADAVYQQAAJiqi9kFdITb7dbevXuVkJAgi8VidjkAAKADDMNQXV2dsrKyZLW2P/4REmFk7969ysnJMbsMAADQCbt371bPnj3b3R8SYSQhIUFSy5dJTEw0uRoAANARDodDOTk5nvN4e0IijJy4NJOYmEgYAQAgxJxpigUTWAEAgKkIIwAAwFSEEQAAYCrCCAAAMBVhBAAAmIowAgAATEUYAQAApiKMAAAAUxFGAACAqbwOIytXrtT48eOVlZUli8Wi119//YzHrFixQiNHjpTNZtO5556r559/vhOlAgCAcOR1GKmvr9ewYcO0YMGCDrXfsWOHrr76al166aUqLy/XXXfdpV/96ld69913vS4WAACEH6+fTXPllVfqyiuv7HD7hQsXqk+fPnr00UclSQMHDtSqVav0xz/+UUVFRd5+PAAACDN+f1BeWVmZCgsLW20rKirSXXfd1e4xTqdTTqfT89rhcPirPAAAQtZ+xzH9b9lONTS6zvq9fjm2j3JS431Qlff8HkYqKytlt9tbbbPb7XI4HDp69Kji4uJOOaakpERz5871d2kAAIS0Rf/eqYUffO2T9xo/LCt8w0hnzJo1S8XFxZ7XDodDOTk5JlYEAEDwqW1olCTl90lVXm7KWb2XPTHWFyV1it/DSGZmpqqqqlptq6qqUmJiYpujIpJks9lks9n8XRoAACHN2eyWJBUOtOvmi84xuZrO8/s6IwUFBSotLW21bfny5SooKPD3RwMAENaONbXMFbFFh/ayYV5Xf+TIEZWXl6u8vFxSy6275eXlqqiokNRyiWXy5Mme9rfeequ2b9+u3/zmN9q8ebOeeOIJvfzyy7r77rt98w0AAIhQJ0ZGYrtEmVzJ2fE6jKxdu1YjRozQiBEjJEnFxcUaMWKEZs+eLUnat2+fJ5hIUp8+ffT2229r+fLlGjZsmB599FE988wz3NYLAMBZCpeREa/njFxyySUyDKPd/W2trnrJJZdow4YN3n4UAAA4DU8YibSREQAAEBw8l2lCfGQktKsHACCCMTICAABMxcgIAAAw1bGmljDCyAgAADCFs7nlMg0jIwAAwBTOphOXaRgZAQAAAXbwiFONrhOXaUL7dB7a1QMAEKFm/OMzSVJ/e4JSu8aYXM3ZIYwAABBiGhqb9d7m/ZKk+T8fLovFYnJFZ4cwAgBAiNm0xyG3IdkTbRrYI9Hscs4aYQQAgBDz5T6HJGlIdpLJlfgGYQQAgBCz40C9JKlvejeTK/ENwggAACHm6+ojkqRz0ruaXIlveP3UXgAAYI7dNQ16YfUurd91SJI0IDP054tIhBEAAELGb9/6Qsu/qJIk9e4er6E9w2POCGEEAIAQ0NDYrA+2VkuSfjm2j346qmfI39J7AmEEAIAQsGZHjRqb3cpOjtMDPxwYNkFEYgIrAAAh4aOvD0qSxp7bPayCiEQYAQAg6O13HNNTK7dLksaem2ZyNb5HGAEAIMi9tGa35+cL+oZfGGHOCAAAQeitz/Zq3fFbeE9MXP3pqJ5KT7CZWZZfEEYAAAgyu2sadMdLG2QYrbdPKcg1pR5/I4wAABBgTS73afe/Ub5HhiH1y+imy8+3S5L6pHXTkDBZV+RkhBEAAALE7Tb0i2dXe+6MOZNfjeujCaN7+bkq8zGBFQCAAFmxdX+Hg0h2cpyuHNLDzxUFB0ZGAAAIkJVbD0iSrh2epbnXDD5t2662KHWJiowxA8IIAAA+8v7m/br75XIdbXS1ub/x+FyRS/pnKCk+OpClBTXCCAAAPnCsyaXZb25SbUPTadslxHbRBed2D1BVoYEwAgCAD0x8+mPtrjkqSXruxtHqZ+/WZrvUrjGKj+H0+130BgAAZ6HJ5dYnO2q0oaJWknRJ/3RddF66oqzh9fwYfyKMAABwFma/8bleWlMhSTrP3k3PTx1jckWhJzKm6QIA4Cfvba6SJPVKjdevLznX5GpCEyMjAAB00hMrtqnK4ZQkvXXnhUqM5Q6ZzmBkBACATqh3Nuvx97ZJkvL7pBJEzgIjIwAAeKHJ5dZTK7fri30ONTS6ZE+0afEt3zO7rJBGGAEAwAtLN+7Tw+9u8bz+wSC7LBbunDkbhBEAALzw0baWZ8vk90nVheemaWJ++D/Izt8IIwAAdNAHW6u1ZO1uSdJtl/TVJf0zTK4oPDCBFQCADvrj8q2SpJT4aI3rl25yNeGDMAIAQAccqm/Up9/USpL+v1/ms8KqD3GZBgCANrxRvkflu2s9r/fWHpVhSP3tCRrSM8m8wsIQYQQAgJNU1zl115JyGcap+y4dwDwRXyOMAABwkoqaehmGlBwfrUnfuVsmPqaLfpHf28TKwhNhBACAk3xz6Kgk6Tx7gu4pGmByNeGPMAIAwHH/7+NdenrldjmONUmSeibHmVxRZCCMAABw3PP/3qGKmgbP6xG9U0ysJnIQRgAAOO7EE3gX/mKk+qZ307kZ3UyuKDIQRgAAkHTE2awjzmZJ0rh+6epq4xQZKPQ0ACBsVdc59fxHO9TQ6Dpj2yPHWoJIgq0LQSTA6G0AQNh6csXXWvTvHV4d06t7vJ+qQXsIIwCAsLV6R8sTdn84tId6dyBkWC0WXTm4h7/LwkkIIwCAsPTRtgP6fK9DkvTADwfJnhhrckVoDw/KAwCEnYbGZs149TNJ0i++14sgEuQYGQEAhKz3N+/XF/scp2x/df032l1zVFlJsZpxBSuoBjvCCAAgJO2pPaqb/vcTudt4mN0Jt116rhJiowNXFDqFMAIACDmHjzZp8rOr5TakzMRYXXxe+iltMhJtmpCXY0J18BZhBAAQcl5Zu1tfV9dLkq4dka2ZV3IpJpR1agLrggULlJubq9jYWOXn52vNmjWnbT9//nz1799fcXFxysnJ0d13361jx451qmAAAHYdbHl+THSURbdefI7J1eBseR1GlixZouLiYs2ZM0fr16/XsGHDVFRUpP3797fZ/sUXX9TMmTM1Z84cffnll3r22We1ZMkS3XvvvWddPAAgMlU6Wv5BO2f8+UqOjzG5Gpwtry/TPPbYY7r55ps1depUSdLChQv19ttva9GiRZo5c+Yp7T/66CONHTtW119/vSQpNzdXEydO1OrVq8+ydABAODjW5NKKLfs7tGT7CVur6iS1zBdB6PMqjDQ2NmrdunWaNWuWZ5vValVhYaHKysraPOaCCy7QCy+8oDVr1mjMmDHavn27li5dqhtuuKHdz3E6nXI6nZ7XDsept20BAMLD0yu369HlWzt1bI9kwkg48CqMHDhwQC6XS3a7vdV2u92uzZs3t3nM9ddfrwMHDujCCy+UYRhqbm7WrbfeetrLNCUlJZo7d643pQEAQtSJdULOs3dTZlJch487L6ObBvVI9FdZCCC/302zYsUK/e53v9MTTzyh/Px8bdu2TdOnT9eDDz6oBx54oM1jZs2apeLiYs9rh8OhnBxuzwKAcPTNoaOSpHuKBugHg+xnaI1w5FUYSUtLU1RUlKqqqlptr6qqUmZmZpvHPPDAA7rhhhv0q1/9SpI0ZMgQ1dfX65ZbbtF9990nq/XUObQ2m002m82b0gAAIWr3oZY7Y3qmdHxUBOHFqzASExOjUaNGqbS0VNdee60kye12q7S0VLfffnubxzQ0NJwSOKKioiRJhnGaZfMAAGHnwBGnFq3aoaNNLZNVnc1u1TY0KTrK0qGn6iI8eX2Zpri4WFOmTFFeXp7GjBmj+fPnq76+3nN3zeTJk5Wdna2SkhJJ0vjx4/XYY49pxIgRnss0DzzwgMaPH+8JJQCAyDDvnc36+7pvTtn+/QEZio9hHc5I5fXf/IQJE1RdXa3Zs2ersrJSw4cP17JlyzyTWisqKlqNhNx///2yWCy6//77tWfPHqWnp2v8+PF66KGHfPctAABB72ijS//8dK8k6dL+6RqU1TL5NCYqSv8xuqeZpcFkFiMErpU4HA4lJSXp8OHDSkxk5jQABIPSL6v03//8XM4md4faN7sN1dQ3Kjs5TqtmXCqLxeLnCmG2jp6/GRMDAHTKMx/u0O6ao14f98NhPQgiaIUwAgDw2qH6Rq3ecVCS9NyNo5WR2LE7IGOirOqb3s2fpSEEEUYAAF6Z+Y/PtPiT3ZKkc9K76pL+6Yx04Kx06qm9AIDI5HIbem3DHs/rWy/qSxDBWWNkBADQYUs37pOzuWXC6udzi9TVxmkEZ4+REQBAhz21crskqV9GN4IIfIYwAgDosP11xyRJt17c1+RKEE4IIwCADmlyubW/zilJGndemsnVIJwwxgYAEeJwQ5Oe/fcO1R1r6tTxx5pcMgwpOsqitK48zBS+QxgBgAjx8trd+nPpV2f9Pr27d5XVyh008B3CCABEiN2HGiRJY3JTNbpPSqfewyKLis7P9GVZAGEEACJFlaNl8un4YT10Q0GuucUA38EEVgCIAC63oTU7aiRJGYmxJlcDtEYYAYAIcOsL63SooWXiqp0wgiBDGAGAMLfg/W1a/kWVJOn8rEQN7JFgckVAa8wZAYAw94/130iShvZM0pu3X2hyNcCpCCMAEKLeKN+j/3n7SzW53KdtV3v88szzU8cEoizAa4QRAAhRz3y4Q9XHV0Q9kzG5qUrtGuPnioDOIYwAQAg6fLRJm/YeliT947YLlBQXfdr2vbvHB6IsoFMIIwAQgrZU1skwpOzkOI3q3bkFzIBgQRgBgCDV5HLr6Q+3t3kpZtv+I5Kk8+zdAl0W4HOEEQAIUsu/qNIflm05bZvzs5ICVA3gP4QRAAhSm/c5JLXckjuuX9op++NjumhSfq9AlwX4HGEEAIJQ2dcH9ef3tkmSrhmWpV+NO8fkigD/YQVWAAhCf1y+1fPz8Jxk8woBAoCREQAIkKdWfq1/fV7VobafflMrSSr5yRDl5ab6sSrAfIQRAAiAY00uzXtns9xGx4/JTo7Tz0fn+K8oIEgQRgDAz2rqG/Xx9oNyG1JSXLR+f92QDh03LCdZFovFz9UB5iOMAIAf7a09qkseWaHG5pbnxwzskaArBvcwuSoguBBGAMCP1u46pMZmt2KirEpPsGlSfm+zSwKCDmEEAPzoq6o6SdJ1o7JV8pOhJlcDBCdu7QUAP9peXS9J6pvOsu1AewgjAOBHB460PFfGnhhrciVA8CKMAIAfHWpolCSlxMeYXAkQvAgjAOBHhxqaJEkpXaNNrgQIXoQRAPATwzB0qJ6REeBMuJsGQERat6tGb39WKUNeLInqJZfbUPPxJVcJI0D7CCMAItKMf2zUtv1HAvJZKfHRiouJCshnAaGIMAIg4hiGoYqaBknSjRfkqqvNv0Hhon7pfn1/INQRRgBEnNqGJs/y7DOvHKDYaEYtADMxgRVAxKl0HJPUcvmEIAKYjzACIOLsPn6JhoXIgOBAGAEQcf71RZUkaWTvFJMrASAxZwRABHE2u/SzhWX67JvDkqTrRmabXBEAiTACIAI4m1166oPt2lJV5wki4/qlaWQvRkaAYEAYARD23tiwV48u3+p5ffWQHvrLxBGyWCwmVgXgBMIIgJD3dfURrdt5qN39b366V5KU3ydVF52Xrkn5vWS1EkSAYEEYARDSXG5DE5/6WPvrnGdsO3Vsrq4Y3CMAVQHwBmEEQEj79Jta7a9zKi46SgV9u7fbrkdSrL4/wB7AygB0FGEEQEhbvb1GknTxeelaeMMok6sB0BmEEQBBqcnl1lMrt+vAkdNffln11QFJ0oheyQGoCoA/EEYABKV3NlXq4Xe3dLj9CG7TBUIWYQRAUPpkR8vll9G5KRrTJ/W0bXumxGt0LmEECFWEEQCm2lpVp6+qjpyy/YOt1ZKkqWP76Koh3AEDhDPCCADTVNc5Nf4vq+Rsdre5Py46Sheflx7gqgAEGmEEgCkeeXeL/rZ6l5zNbqV2jVG/jG6t9lss0k9G9lRXG7+mgHDXqf/LFyxYoIcffliVlZUaNmyY/vKXv2jMmDHttq+trdV9992nV199VTU1Nerdu7fmz5+vq666qtOFAwhdhmFo0b93qKHRJUmaflk/Tbkg19yiAJjG6zCyZMkSFRcXa+HChcrPz9f8+fNVVFSkLVu2KCMj45T2jY2N+sEPfqCMjAz9/e9/V3Z2tnbt2qXk5GRf1A8gBNU2NHmCyLK7xqm/PcHkigCYyesw8thjj+nmm2/W1KlTJUkLFy7U22+/rUWLFmnmzJmntF+0aJFqamr00UcfKTo6WpKUm5t7dlUDCBrNLrd2HqyXYXT8mK+r6yVJad1sGpCZ6KfKAIQKr8JIY2Oj1q1bp1mzZnm2Wa1WFRYWqqysrM1j3nzzTRUUFGjatGl64403lJ6eruuvv14zZsxQVFRUm8c4nU45nd8udORwOLwpE0AA3fjcJ1q17UCnjs1OifNxNQBCkVdh5MCBA3K5XLLbWz/fwW63a/PmzW0es337dr333nuaNGmSli5dqm3btunXv/61mpqaNGfOnDaPKSkp0dy5c70pDYAJDMPQmp0t64Ekx0fLaun4k3CjrBb9fHSOv0oDEEL8Pk3d7XYrIyNDTz31lKKiojRq1Cjt2bNHDz/8cLthZNasWSouLva8djgcysnhlxYQbA7WN6qx2S2LRfrkvkJFR1nNLglACPIqjKSlpSkqKkpVVVWttldVVSkzM7PNY3r06KHo6OhWl2QGDhyoyspKNTY2KiYm5pRjbDabbDabN6UBMMGugy1zP9K72QgiADrNq98eMTExGjVqlEpLSz3b3G63SktLVVBQ0OYxY8eO1bZt2+R2f7uo0datW9WjR482gwiA0HHP3z+TJPVIZu4HgM7z+jJNcXGxpkyZory8PI0ZM0bz589XfX295+6ayZMnKzs7WyUlJZKk2267TY8//rimT5+uO+64Q1999ZV+97vf6c477/TtNwHgN6+s3a0v9p06kXx3TYMk6arBbY+MAkBHeB1GJkyYoOrqas2ePVuVlZUaPny4li1b5pnUWlFRIav12wGXnJwcvfvuu7r77rs1dOhQZWdna/r06ZoxY4bvvgUAv9lc6fCMgLSld/d43XLROQGsCEC4sRiGN6sDmMPhcCgpKUmHDx9WYiJrEgCB4nYb6v/AO2pyGRqQmaDLBrZe2NAii64YnKnB2UkmVQggmHX0/M1DHwC065tDR9Xkavn3ym+u6K/vD7Cf4QgA8B5hBIAk6YizWc98uF2HjzZ5tu2tPSpJGtQjkSACwG8IIwAktUxSnf9/X7W5bwiXYQD4EWEEgCRpa1WdJOl756RqVO8Uz3ZblyhNYKVUAH5EGAGgTXsO66U1uyVJE0bn6McjeppcEYBIwpKJADTr1Y2en3mKLoBAY2QEiABfVx/RofrGNve5DenL4wuaFf/gPA3sQRgBEFiEESDMffT1AV3/9OoztkuOj9Yd3z83ABUBQGuEESCM1TY06j9f/lRSS9hIiW/7eVAWSdfn95LFYglgdQDQgjAChLEZ//hM+w4fkyT99keDdc2wLJMrAoBTEUaAEPfH5Vv19sZ9be7beaBeklRwTnddPohFywAEJ8IIEMKONbn0+Pvb5HK3/4ip/vYEvXhzPpdgAAQtwggQIr6qqtO7n1fqu7mjpr5RLreh1K4xemLSyDaPG9gjkSACIKgRRoAQcefics8tuCcb2jNJ3zune4ArAgDfIIwAQa7Z5da0F9fry30OWSzSz0fnqOX+lxbRURZNyu9tXoEAcJYII0CQe/hfW/Tu51WSpLzeKSr5yVCTKwIA32I5eCCIvbe5Sn/9YLskqW96V/31hjyTKwIA3yOMAEHsxIJlkrT4lgKldm170TIACGWEESBINbvcOtTQJEl65dYCpSfYTK4IAPyDMAIEqXqny/PzsJ7J5hUCAH5GGAGC1JHGZklSTJRVMV34XxVA+OI3HBCk6p0tYaSrLcrkSgDAvwgjQJA64gkj3IEPILwRRoAgdWJkpBthBECYI4wAQaqekREAEYLfckAHrdt1SG9/tk+G2n9Cri99XV0viZERAOGP33JAB9376kZtqaoL+OeyvgiAcEcYATrowBGnJGnimF5K7RodkM+MiYrST/N6BuSzAMAshBGgg+qOtczhmHZpX/VMiTe5GgAIH0xgBTrA2exSo8stSUqIDcyoCABECsII0AFHjo+KSEwoBQBfI4wAHXDiEk3XmChFWS0mVwMA4YUwAnTAidVQuUQDAL5HGAE64OPtByVJ3WK5RAMAvkYYAc5g9faD+p+3v5Qk2RNZ8wMAfI0wApyGYRh64I1Nntf3XjXQxGoAIDwx5gycxqY9Dm2tOiJJemf6OA3skWhyRQAQfhgZAU6jdHOVJKnofDtBBAD8hJER4DsMw9CC97dp2/6W0ZCPt9dIki4bYDezLAAIa4QR4Ds27jmsR/61tdW26CiLLhmQblJFABD+CCOIeI5jTVq365AMw9Cqr1pu4R2QmaCfjmp5QN3g7CRlJMSaWSIAhDXCCCLenS9t0Iot1a22XTogQ78ad45JFQFAZCGMIGIZhqH//WinJ4gMyU6SxSIlxHbRz0fnmFwdAEQOwggi1hMrvtbD726RJA3OTtQ/77jQ5IoAIDIRRhC2Djc0acpza7S39mib+2sbmiRJ56R31aIbRweyNADAdxBGELZWbN2v8t21p21jT7Rp6Z3jFBsdFZiiAACnIIwgLL2+YY/+t2ynJOlHw7N0y0VtT0bNSY0niACAyQgjCDuffVOru5aUe15f1C9d52clmVcQAOC0CCMIO6+u3yNJio226r8u76/xw7JMrggAcDqEEYSso40uPfPhdtU0NLbavvyLlufJ/P66ofrR8GwzSgMAeIEwgpD19sZ9enT51jb3WSzSqN4pAa4IANAZhBGErF0H6yVJw3om6cJ+aa32nZ+VpJ4p8WaUBQDwEmEEIWtv7TFJ0uXnZ2rapeeaXA0AoLMIIwgJzmaXnvqg9fyQ1TtaHmrXI4mH2AFAKCOMICS8tn5Pu/NDctO6BrgaAIAvEUYQ1N7bXKX7X9ukg/UtIyLj+qVpaM9v1wzJSYnXiJxkk6oDAPiCtTMHLViwQLm5uYqNjVV+fr7WrFnToeMWL14si8Wia6+9tjMfiwj0j/V7tPfwMTmb3epitWj2DwfpnqIBnj8/H9NLFovF7DIBAGfB6zCyZMkSFRcXa86cOVq/fr2GDRumoqIi7d+//7TH7dy5U//1X/+lcePGdbpYRJ59xx9yN2f8IK2+9zL1syeYXBEAwNe8DiOPPfaYbr75Zk2dOlWDBg3SwoULFR8fr0WLFrV7jMvl0qRJkzR37lydc07bzwgB2rLvcMsdMyN6pah7N5vJ1QAA/MGrOSONjY1at26dZs2a5dlmtVpVWFiosrKydo/77W9/q4yMDN1000368MMPz/g5TqdTTqfT89rhcHhTJkKQYRi65f+t08qt1a22O5vdkqQs7pgBgLDlVRg5cOCAXC6X7HZ7q+12u12bN29u85hVq1bp2WefVXl5eYc/p6SkRHPnzvWmNIS4g/WNnmXcT3aevZvSGBUBgLDl17tp6urqdMMNN+jpp59WWlramQ84btasWSouLva8djgcysnJ8UeJCBJ7DrXMDUlPsOm1X1/Qap89MVZWK5NUASBceRVG0tLSFBUVpaqq1v+CraqqUmZm5intv/76a+3cuVPjx4/3bHO7W4bdu3Tpoi1btqhv376nHGez2WSz8S/hSLLn+ETVnJQ4lnEHgAjj1QTWmJgYjRo1SqWlpZ5tbrdbpaWlKigoOKX9gAEDtHHjRpWXl3v+XHPNNbr00ktVXl7OaAc8dh1skCRlE0QAIOJ4fZmmuLhYU6ZMUV5ensaMGaP58+ervr5eU6dOlSRNnjxZ2dnZKikpUWxsrAYPHtzq+OTkZEk6ZTsii2EYmvvPL7S+4pAkaXdNSxgZ9p0FzQAAkcHrMDJhwgRVV1dr9uzZqqys1PDhw7Vs2TLPpNaKigpZrZ1aSw0RZHfNUT3/0c5W22KirLpi8KmX+wAA4c1iGIZhdhFn4nA4lJSUpMOHDysxMdHscuADK7dWa/KiNcpJjdNvr2kZJevVPV5907uZXBkAwFc6ev7m2TTwmWNNLj29svWTdduzbf8RSVJ/e6IuHZDh79IAAEGMMAKfebN8b7tP1m3PeXZGQgAg0hFG4DMbdrdMRs3vk6q83JQzto+P6aLrx/Tyd1kAgCBHGIHPbKiolSTdeEGurhzSw9xiAAAhgzCCTqs8fEyz39gkx7EmGYa0ubJOFos0pk+q2aUBAEIIYQSdtviTCv3rpOfJ5PXm6boAAO8QRuC1xWsqtKWqTh9saXnC7sQxvTT23O6yWizKZ1QEAOAlwgi8sm5XjWa+urHVtuvH9NIQVk4FAHQSYQQd5nIbuu7JMknS+VmJuqR/unK7dyWIAADOCmEEHVZT/+1iZrN/OEj553Q3sRoAQLjgITLosMNHmyRJibFdCCIAAJ8hjKDDDh9tGRlJjo8xuRIAQDghjKDDahtaRkaS46NNrgQAEE4II+iwE5dpkuIIIwAA32ECa4RatmmfVu+o8eqYL/c5JBFGAAC+RRiJQEcbXbrjpQ1qchmdOj4zMdbHFQEAIhlhJAJV1znV5DIUE2XVzRf18erYuOgoTRjNk3YBAL5DGIlA1UeckqT0BJvuKRpgcjUAgEjHBNYIdPB4GElL4IF2AADzEUYi0IEjLeuFpHdjvRAAgPkIIxFo7a6Wu2jSE5iICgAwH3NGIojLbWjeO1/q1fV7JEnXjcw2uSIAABgZiSjLNlXq6Q93SJLyeqcoLzfV5IoAACCMRIx6Z7OmvbhekhQdZdGj/zHM5IoAAGhBGIkQa3Z+u9rqq7eNVe/uXU2sBgCAbzFnJMw1u9x6f0u1lm7cJ0ka1y9NQ3ommVwVAADfIoyEuSVrd+u+1zZ5Xuf3YZ4IACC4EEbC3L+3HZAknWfvpvPsCfqPvByTKwIAoDXCSJjbUFErSfrtjwbre+d0N7cYAADawATWMOY41qR9h49JkgZlJZpcDQAAbWNkJAy99dlerdt1SLUNTZKkjASbEmOjTa4KAIC2EUbCjONYk6YvLpfLbXi2MSoCAAhmhJEws7f2qFxuQ11jonTj2FxFWa368QiWfQcABC/CSJipcjglSTmp8bqnaIDJ1QAAcGaEkTBw8IhTe2tbJqpuqDgkScpM4om8AIDQQBgJcdV1Tn3/kRWqcza32m5PIIwAAEIDYSSEOZtdeuTdLZ4gknV8NCQuJko/Hsk8EQBAaCCMhCjDMPSTJz7S53sdkqSHfzpUP2N1VQBACGLRsxD1yc5DniBy8Xnp3DEDAAhZjIyEqE17DkuSrjg/UwtvGGVyNQAAdB4jIyGqqq7l7pms5DiTKwEA4OwQRkLU/uPridgTbSZXAgDA2eEyTZBpdrn1zKodqnIcO227NTtqJEn2RG7hBQCENsJIkFn5VbXmvbO5w+17dY/3YzUAAPgfYSTI7Dl0VJJ0bkY3FZ1vP23bninxGpGTHICqAADwH8JIkKk+0ihJyu+TyrNlAAARgQmsQebAkZaJqWndmJgKAIgMhJEg4nYbenF1hSQpLYEwAgCIDISRIPLmp3s9P/dOZWIqACAyEEaCxAsf79JdS8olSdnJcbrw3DRzCwIAIEAII0HAMAw98+F2z+uFvxglq9ViYkUAAAQOd9OYpNnl1hf7HGpsdqvK4dTOgw2K6WLVhgd+oK42/loAAJGDs55JHvnXVi384OtW28bkphJEAAARhzOfSZZu3CepZX5ITBerbF2suuWic0yuCgCAwCOMmKDKcUwVNQ2Kslr0r7svYjQEABDRmMBqgq+rj0iSeqXGE0QAABGvU2FkwYIFys3NVWxsrPLz87VmzZp22z799NMaN26cUlJSlJKSosLCwtO2D3d1x5pUvORTSVKftK4mVwMAgPm8DiNLlixRcXGx5syZo/Xr12vYsGEqKirS/v3722y/YsUKTZw4Ue+//77KysqUk5Ojyy+/XHv27Dnr4kPRy2u/UaXjmCTpPHuCydUAAGA+i2EYhjcH5Ofna/To0Xr88cclSW63Wzk5Obrjjjs0c+bMMx7vcrmUkpKixx9/XJMnT+7QZzocDiUlJenw4cNKTEz0ptygc88rn+qVdd+oa0yUPpzxfaV2jTG7JAAA/KKj52+vJiw0NjZq3bp1mjVrlmeb1WpVYWGhysrKOvQeDQ0NampqUmpqarttnE6nnE6n57XD4fCmzKCy40C95v7zcx051ixJ2lJVJ0n6/U+HEkQAAJCXYeTAgQNyuVyy2+2tttvtdm3evLlD7zFjxgxlZWWpsLCw3TYlJSWaO3euN6UFrX+s+0YrtlS32ma1SMN6JptTEAAAQSagt3LMmzdPixcv1ooVKxQbG9tuu1mzZqm4uNjz2uFwKCcnJxAl+lxNQ6MkafywLF09JFOS1Cu1q3J4EB4AAJK8DCNpaWmKiopSVVVVq+1VVVXKzMw87bGPPPKI5s2bp//7v//T0KFDT9vWZrPJZrN5U1rQqj0eRkb1StYVg3uYXA0AAMHHq7tpYmJiNGrUKJWWlnq2ud1ulZaWqqCgoN3j/vCHP+jBBx/UsmXLlJeX1/lqQ9Ch+iZJUgrzQwAAaJPXl2mKi4s1ZcoU5eXlacyYMZo/f77q6+s1depUSdLkyZOVnZ2tkpISSdLvf/97zZ49Wy+++KJyc3NVWVkpSerWrZu6devmw68SnA4dHxlJjieMAADQFq/DyIQJE1RdXa3Zs2ersrJSw4cP17JlyzyTWisqKmS1fjvg8uSTT6qxsVE//elPW73PnDlz9N///d9nV30Qcza79Pt3tmhzZcvdMynx0SZXBABAcPJ6nREzhOI6I6+u/0bFL7estBpltejjWZcpPSE85sEAANARfllnBB1zrMnlCSLfH5Chm8edQxABAKAdhBE/WLfrkOfnaZeeq1G9U0ysBgCA4MZTe/1g18EGSVLf9K4EEQAAzoCRER851uTSMx9u18H6Rm2oqJUkjeuXbm5RAACEAMKIj7yzaZ8e+dfWVtvOzQj/W5cBADhbhBEfOXEL74heybqgb3clx8XoupE9Ta4KAIDgRxjxke3V9ZKkH4/I1uSCXHOLAQAghDCB1Ud2HWwJI7ndu5pcCQAAoYUw4iPVdU5Jkj2x/acRAwCAUxFGfKDZ5dahhpYH4nXvxjNoAADwBmHEB2rqWx6GZ7VIKTwQDwAArzCB9Sy9Ub5HH2ypliSldrUpymoxuSIAAEILYeQs7K5p0PTF5Z7XPVPizCsGAIAQRRg5Cx9sbRkR6ZPWVT8c2kNXDelhckUAAIQewkgnNbvcemx5y4qr143M1u3f72dyRQAAhCYmsHbSm5/u9Uxcvfi8DJOrAQAgdBFGOql8d60kydbFqsHZieYWAwBACOMyTQet3Fqt97fs97w+8fP/XDtYFgt30AAA0FmEkQ5wuQ1N+9t61TmbT9k3sAejIgAAnA3CSAfsOHBEdc5mxUZbddOFfTzbc7t31eDsJBMrAwAg9BFGOmDTHock6fysJN1TNMDkagAACC9MYO2Az/celiQNzuKSDAAAvkYYOYPFayr09Ic7JLWMjAAAAN8ijJzBC6t3eX4u6NvdxEoAAAhPhJEz2HPoqCTphZvylZMab3I1AACEH8LIaVTXOXWooUmSNDSHSzQAAPgDYaQdRxtdGv3Q/0mSEmK7KDE22uSKAAAIT4SRdmw/cMTz8+SC3iZWAgBAeCOMtOPEXJEh2awtAgCAPxFG2rGntiWMZCfHmVwJAADhjTDSjhMjI9kphBEAAPyJMNIORkYAAAgMwkg7PGGEkREAAPyKMNIOz2UaRkYAAPArwkgbjja6dLC+UZLUk5ERAAD8ijDShhOXaLrGRCkpjsXOAADwJ8JIG/Z+Z76IxWIxuRoAAMIbYaQN3EkDAEDgEEbawBojAAAEDmGkDd+OjMSbXAkAAOGPMNIGRkYAAAgcwkgbmDMCAEDgEEbaUN/YLElKiuticiUAAIQ/wkgbmprdkqToKLoHAAB/42zbhiaXIUmK6UL3AADgb5xtT2IYhhpdjIwAABAonG1PcmJURCKMAAAQCJxtT9J0fFREkmIIIwAA+B1n25N8N4xER/FcGgAA/I0wcpIT80UsFinKShgBAMDfCCMn8dxJE2Xlib0AAAQAYeQkjcfXGGG+CAAAgcEZ9yQn5oxEs8YIAAABwRn3JI2e1Ve5RAMAQCAQRk7SxIJnAAAEFGfck7AUPAAAgdWpM+6CBQuUm5ur2NhY5efna82aNadt/8orr2jAgAGKjY3VkCFDtHTp0k4VGwgnRkaYwAoAQGB4fcZdsmSJiouLNWfOHK1fv17Dhg1TUVGR9u/f32b7jz76SBMnTtRNN92kDRs26Nprr9W1116rTZs2nXXx/tDIE3sBAAgor8+4jz32mG6++WZNnTpVgwYN0sKFCxUfH69Fixa12f5Pf/qTrrjiCt1zzz0aOHCgHnzwQY0cOVKPP/74WRfvD98+JI8JrAAABEIXbxo3NjZq3bp1mjVrlmeb1WpVYWGhysrK2jymrKxMxcXFrbYVFRXp9ddfb/dznE6nnE6n57XD4fCmzA57dtUOfXOoodW2nQfqJTEyAgBAoHgVRg4cOCCXyyW73d5qu91u1+bNm9s8prKyss32lZWV7X5OSUmJ5s6d601pnfL2Z3u1vqK2zX0p8TF+/3wAAOBlGAmUWbNmtRpNcTgcysnJ8fnnXDeqpwr6dj9lexerVT8Zme3zzwMAAKfyKoykpaUpKipKVVVVrbZXVVUpMzOzzWMyMzO9ai9JNptNNpvNm9I6ZVJ+b79/BgAAOD2vJkbExMRo1KhRKi0t9Wxzu90qLS1VQUFBm8cUFBS0ai9Jy5cvb7c9AACILF5fpikuLtaUKVOUl5enMWPGaP78+aqvr9fUqVMlSZMnT1Z2drZKSkokSdOnT9fFF1+sRx99VFdffbUWL16stWvX6qmnnvLtNwEAACHJ6zAyYcIEVVdXa/bs2aqsrNTw4cO1bNkyzyTViooKWa3fDrhccMEFevHFF3X//ffr3nvvVb9+/fT6669r8ODBvvsWAAAgZFkMwzDMLuJMHA6HkpKSdPjwYSUmJppdDgAA6ICOnr9ZTAMAAJiKMAIAAExFGAEAAKYijAAAAFMRRgAAgKkIIwAAwFSEEQAAYCrCCAAAMBVhBAAAmMrr5eDNcGKRWIfDYXIlAACgo06ct8+02HtIhJG6ujpJUk5OjsmVAAAAb9XV1SkpKand/SHxbBq32629e/cqISFBFovFZ+/rcDiUk5Oj3bt388wbP6OvA4N+Dgz6OTDo58DxV18bhqG6ujplZWW1eojuyUJiZMRqtapnz55+e//ExET+Qw8Q+jow6OfAoJ8Dg34OHH/09elGRE5gAisAADAVYQQAAJgqosOIzWbTnDlzZLPZzC4l7NHXgUE/Bwb9HBj0c+CY3dchMYEVAACEr4geGQEAAOYjjAAAAFMRRgAAgKkIIwAAwFQRHUYWLFig3NxcxcbGKj8/X2vWrDG7pJBRUlKi0aNHKyEhQRkZGbr22mu1ZcuWVm2OHTumadOmqXv37urWrZuuu+46VVVVtWpTUVGhq6++WvHx8crIyNA999yj5ubmQH6VkDJv3jxZLBbdddddnm30s+/s2bNHv/jFL9S9e3fFxcVpyJAhWrt2rWe/YRiaPXu2evToobi4OBUWFuqrr75q9R41NTWaNGmSEhMTlZycrJtuuklHjhwJ9FcJWi6XSw888ID69OmjuLg49e3bVw8++GCrZ5fQz52zcuVKjR8/XllZWbJYLHr99ddb7fdVv3722WcaN26cYmNjlZOToz/84Q9nX7wRoRYvXmzExMQYixYtMj7//HPj5ptvNpKTk42qqiqzSwsJRUVFxnPPPWds2rTJKC8vN6666iqjV69expEjRzxtbr31ViMnJ8coLS011q5da3zve98zLrjgAs/+5uZmY/DgwUZhYaGxYcMGY+nSpUZaWpoxa9YsM75S0FuzZo2Rm5trDB061Jg+fbpnO/3sGzU1NUbv3r2NG2+80Vi9erWxfft249133zW2bdvmaTNv3jwjKSnJeP31141PP/3UuOaaa4w+ffoYR48e9bS54oorjGHDhhkff/yx8eGHHxrnnnuuMXHiRDO+UlB66KGHjO7duxtvvfWWsWPHDuOVV14xunXrZvzpT3/ytKGfO2fp0qXGfffdZ7z66quGJOO1115rtd8X/Xr48GHDbrcbkyZNMjZt2mS89NJLRlxcnPHXv/71rGqP2DAyZswYY9q0aZ7XLpfLyMrKMkpKSkysKnTt37/fkGR88MEHhmEYRm1trREdHW288sornjZffvmlIckoKyszDKPlfxyr1WpUVlZ62jz55JNGYmKi4XQ6A/sFglxdXZ3Rr18/Y/ny5cbFF1/sCSP0s+/MmDHDuPDCC9vd73a7jczMTOPhhx/2bKutrTVsNpvx0ksvGYZhGF988YUhyfjkk088bd555x3DYrEYe/bs8V/xIeTqq682fvnLX7ba9pOf/MSYNGmSYRj0s6+cHEZ81a9PPPGEkZKS0up3x4wZM4z+/fufVb0ReZmmsbFR69atU2FhoWeb1WpVYWGhysrKTKwsdB0+fFiSlJqaKklat26dmpqaWvXxgAED1KtXL08fl5WVaciQIbLb7Z42RUVFcjgc+vzzzwNYffCbNm2arr766lb9KdHPvvTmm28qLy9PP/vZz5SRkaERI0bo6aef9uzfsWOHKisrW/V1UlKS8vPzW/V1cnKy8vLyPG0KCwtltVq1evXqwH2ZIHbBBReotLRUW7dulSR9+umnWrVqla688kpJ9LO/+Kpfy8rKdNFFFykmJsbTpqioSFu2bNGhQ4c6XV9IPCjP1w4cOCCXy9Xql7Mk2e12bd682aSqQpfb7dZdd92lsWPHavDgwZKkyspKxcTEKDk5uVVbu92uyspKT5u2/g5O7EOLxYsXa/369frkk09O2Uc/+8727dv15JNPqri4WPfee68++eQT3XnnnYqJidGUKVM8fdVWX363rzMyMlrt79Kli1JTU+nr42bOnCmHw6EBAwYoKipKLpdLDz30kCZNmiRJ9LOf+KpfKysr1adPn1Pe48S+lJSUTtUXkWEEvjVt2jRt2rRJq1atMruUsLN7925Nnz5dy5cvV2xsrNnlhDW32628vDz97ne/kySNGDFCmzZt0sKFCzVlyhSTqwsfL7/8sv72t7/pxRdf1Pnnn6/y8nLdddddysrKop8jWERepklLS1NUVNQpdxxUVVUpMzPTpKpC0+2336633npL77//vnr27OnZnpmZqcbGRtXW1rZq/90+zszMbPPv4MQ+tFyG2b9/v0aOHKkuXbqoS5cu+uCDD/TnP/9ZXbp0kd1up599pEePHho0aFCrbQMHDlRFRYWkb/vqdL83MjMztX///lb7m5ubVVNTQ18fd88992jmzJn6+c9/riFDhuiGG27Q3XffrZKSEkn0s7/4ql/99fskIsNITEyMRo0apdLSUs82t9ut0tJSFRQUmFhZ6DAMQ7fffrtee+01vffee6cM240aNUrR0dGt+njLli2qqKjw9HFBQYE2btzY6j/+5cuXKzEx8ZSTQqS67LLLtHHjRpWXl3v+5OXladKkSZ6f6WffGDt27Cm3p2/dulW9e/eWJPXp00eZmZmt+trhcGj16tWt+rq2tlbr1q3ztHnvvffkdruVn58fgG8R/BoaGmS1tj71REVFye12S6Kf/cVX/VpQUKCVK1eqqanJ02b58uXq379/py/RSIrsW3ttNpvx/PPPG1988YVxyy23GMnJya3uOED7brvtNiMpKclYsWKFsW/fPs+fhoYGT5tbb73V6NWrl/Hee+8Za9euNQoKCoyCggLP/hO3nF5++eVGeXm5sWzZMiM9PZ1bTs/gu3fTGAb97Ctr1qwxunTpYjz00EPGV199Zfztb38z4uPjjRdeeMHTZt68eUZycrLxxhtvGJ999pnxox/9qM1bI0eMGGGsXr3aWLVqldGvX7+Iv+X0u6ZMmWJkZ2d7bu199dVXjbS0NOM3v/mNpw393Dl1dXXGhg0bjA0bNhiSjMcee8zYsGGDsWvXLsMwfNOvtbW1ht1uN2644QZj06ZNxuLFi434+Hhu7T0bf/nLX4xevXoZMTExxpgxY4yPP/7Y7JJChqQ2/zz33HOeNkePHjV+/etfGykpKUZ8fLzx4x//2Ni3b1+r99m5c6dx5ZVXGnFxcUZaWprxn//5n0ZTU1OAv01oOTmM0M++889//tMYPHiwYbPZjAEDBhhPPfVUq/1ut9t44IEHDLvdbthsNuOyyy4ztmzZ0qrNwYMHjYkTJxrdunUzEhMTjalTpxp1dXWB/BpBzeFwGNOnTzd69eplxMbGGuecc45x3333tbpVlH7unPfff7/N38tTpkwxDMN3/frpp58aF154oWGz2Yzs7Gxj3rx5Z127xTC+s+wdAABAgEXknBEAABA8CCMAAMBUhBEAAGAqwggAADAVYQQAAJiKMAIAAExFGAEAAKYijAAAAFMRRgAAgKkIIwAAwFSEEQAAYCrCCAAAMNX/D87wtOg05P3VAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt \n", + "task = \"kile\"\n", + "#task = \"lir\"\n", + "#metric = \"f1\"\n", + "#metric = \"AP\"\n", + "#metric = \"precision\"\n", + "metric = \"recall\"\n", + "values = [browser.evaluation_results.get_metrics(task, docid=x.docid)[metric] for x in browser.dataset]\n", + "values.sort()\n", + "plt.plot(values)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "420fe3da-474b-4852-b6a6-61dfec78acd8", + "metadata": {}, + "outputs": [], + "source": [ + "browser.evaluation_results.get_metrics(\"kile\", docid=browser.document.docid)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f7ea61c8-909c-4d0d-ac4e-24169b78f665", + "metadata": {}, + "outputs": [], + "source": [ + "#browser.document_tabs.children[browser.page].value\n", + "#browser.svg_content" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bcd2c35b-1347-4186-af69-faab8080eafe", + "metadata": {}, + "outputs": [], + "source": [ + "#dataset.documents[0].page_image(0)\n", + "#dataset.documents[0].page_image(0).size[1]\n", + "dataset.documents[0].annotation.fields[0].bbox.to_tuple()[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12d5b3fa-22e8-4a73-9f0c-306b91e66a77", + "metadata": {}, + "outputs": [], + "source": [ + "from docile.evaluation.evaluate import compute_metrics\n", + "compute_metrics(browser.evaluation_result_KILE.task_to_docid_to_matching[\"kile\"][browser.document.docid])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6d810aa4-633e-4132-a30a-57ccbaf6a1d7", + "metadata": {}, + "outputs": [], + "source": [ + "print(browser.evaluation_result_KILE.print_report(browser.document.docid))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2ce883b-98e9-4583-b8df-3e0e26f71c21", + "metadata": {}, + "outputs": [], + "source": [ + "browser.evaluation_result_KILE.get_metrics(\"kile\", docid=browser.document.docid)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02709cc1-bfdf-419b-b323-55580a58d06e", + "metadata": {}, + "outputs": [], + "source": [ + "browser.evaluation_result_LIR.get_metrics(\"lir\", docid=browser.document.docid)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docile/tools/my_dataset_browser.py b/docile/tools/my_dataset_browser.py new file mode 100644 index 0000000..ff3e6e9 --- /dev/null +++ b/docile/tools/my_dataset_browser.py @@ -0,0 +1,578 @@ +import os +import json +from typing import List, Tuple + +import ipywidgets +import plotly.graph_objects as go +from IPython.display import clear_output, display + +from docile.dataset import BBox, Dataset, Field +from docile.dataset import KILE_FIELDTYPES, LIR_FIELDYPES +import numpy as np +from base64 import standard_b64encode +from io import BytesIO +from matplotlib import cm +from matplotlib.colors import to_rgba +import matplotlib.pyplot as plt + +from docile.evaluation.evaluate import evaluate_dataset +from docile.evaluation import EvaluationResult +from pathlib import Path + + +def pilimage_to_b64(pilimage): + mem_f = BytesIO() + pilimage.save(mem_f, format="png") + encoded = standard_b64encode(mem_f.getvalue()) + return encoded + + +def pilimage_to_svg(pilimage): + return [ + '', + ] + + +# class Callback: +# @property +# def name(self): +# return getattr(self, "_name", self.__class__.__name__) + +fieldtypes = ["background"] + KILE_FIELDTYPES + LIR_FIELDYPES +fieldtype_to_id = {key: i for i, key in enumerate(fieldtypes)} + + +def bbox_str(bbox): + if bbox: + return f"{bbox[0]:>#04.1f}, {bbox[1]:>#04.1f}, {bbox[2]:>#04.1f}, {bbox[3]:>#04.1f}" + else: + return f"" + + +def get_color(fieldtype, N=len(fieldtypes)): + cmap = (plt.get_cmap("tab20", N).colors[:, 0:3]*255).astype(np.uint8) + cmap[0] = 0 + return cmap[fieldtype_to_id[fieldtype]] + + +def get_style(fieldtype): + # color = get_color(field.fieldtype if field.fieldtype is not None else "") + color = get_color(fieldtype if fieldtype is not None else "") + return f"stroke:rgb{tuple(color)};stroke-width:1;fill-opacity:0.25;fill:rgb{tuple(color)}" + + +def show_fields(fields, img): + WIDTH, HEIGHT = img.size + items = [(x.bbox.to_absolute_coords(WIDTH, HEIGHT).to_tuple(), x.fieldtype, x.text) for x in fields if x is not None] + svg_bboxes = "\n".join( + [ + f'{item[2]} | {item[1].replace("line_item_", "") if item[1] else "background"} | ({bbox_str(item[0])})' + for item in items + ] + ) + return f"{svg_bboxes}" + + +def get_legend(WIDTH=1980, HEIGHT=100, PER_LINE=5): + svg_legend = [] + for i, ft in enumerate(fieldtypes): + color = get_color(ft) + svg_legend.append( + f""" + + + {ft if ft else "background"} + +""" + ) + + svg_legend = "\n".join(svg_legend) + + to_display=f""" + +{svg_legend} + +""" +# + return to_display + + +class MyDatasetBrowser: + def __init__( + self, + dataset: Dataset, + evaluation_results: EvaluationResult = None, + kile_predictions: dict = None, + lir_predictions: dict = None, + intermediate_predictions: dict = None, + # display_grid: bool = False, + callbacks: List = None, + render_size: tuple = (1920, 1080), + random_seed: int = None, + sort_by: str = "kile", + ) -> None: + + self.evaluation_results = evaluation_results + + if evaluation_results is not None: + sorted_documents = sorted( + dataset.documents, + key=lambda doc: evaluation_results.get_metrics(sort_by, docid=doc.docid)["AP"], + ) + sorted_dataset = Dataset.from_documents("sorted", sorted_documents) + self.dataset = sorted_dataset + else: + self.dataset = dataset + + self.document_idx = 0 + self.document_idxs = {doc.docid: idx for idx, doc in enumerate(self.dataset.documents)} + self._document = None + self._page = 0 + self.kile_predictions = kile_predictions if kile_predictions is not None else {} + self.lir_predictions = lir_predictions if lir_predictions is not None else {} + self.intermediate_predictions = intermediate_predictions if intermediate_predictions is not None else {} + self.RENDER_W = render_size[0] + self.RENDER_H = render_size[1] + self.SVG_RENDER_WIDTH = self.RENDER_W + self._rng = np.random.default_rng(seed=random_seed) + self.context = {} + + # try: + # self.evaluation_result_KILE = evaluate_dataset(dataset, kile_predictions, {}) + # except Exception as ex: + # self.evaluation_result_KILE = None + # try: + # self.evaluation_result_LIR = evaluate_dataset(dataset, {}, lir_predictions) + # except Exception as ex: + # self.evaluation_result_LIR = None + + + # ---- GUI ---- + # -- navigation -- + narrow_layout = ipywidgets.Layout(flex="0 0 auto", width="auto") + wide_layout = ipywidgets.Layout(flex="1 1 auto", width="auto") + + previous_button = ipywidgets.Button( + disabled=False, button_style="", icon="arrow-left", layout=narrow_layout + ) + random_button = ipywidgets.Button( + disabled=False, button_style="", icon="gift", layout=narrow_layout + ) + next_button = ipywidgets.Button( + disabled=False, button_style="", icon="arrow-right", layout=narrow_layout + ) + self.redraw_button = ipywidgets.Button( + disabled=False, button_style="", icon="retweet", layout=narrow_layout + ) + self.document_search_text = ipywidgets.Text( + value=str(self.document_idx), + disabled=False, + continuous_update=False, + layout=wide_layout, + ) + + self.zoom_state = {"width": "auto", "height": "100%"} + + zoom_slider = ipywidgets.FloatSlider( + value=1.0, + min=1.0, + max=8.0, + step=0.25, + description="Zoom:", + disabled=False, + continuous_update=True, + readout=False, + layout=wide_layout, + ) + fit_width_button = ipywidgets.Button( + description="\u2194", + disabled=False, + button_style="", + tooltip="Fit width", + layout=narrow_layout, + ) + fit_height_button = ipywidgets.Button( + description="\u2195", + disabled=False, + button_style="", + tooltip="Fit height", + layout=narrow_layout, + ) + + save_button = ipywidgets.Button( + disabled=False, button_style="", icon="save", layout=narrow_layout + ) + + navigation_bar = ipywidgets.HBox( + ( + previous_button, + random_button, + next_button, + self.redraw_button, + self.document_search_text, + zoom_slider, + fit_width_button, + fit_height_button, + save_button, + ), + layout=ipywidgets.Layout( + display="flex", flex_flow="row nowrap", align_items="stretch", width="auto" + ), + ) + + # -- status -- + self.status_text = ipywidgets.HTML() + self.error_text = ipywidgets.HTML() + + status_bar = ipywidgets.VBox((self.status_text, self.error_text)) + + # -- config -- + overlays_toggle = [] + self.callbacks = {} + + for callback in callbacks or []: + # button = ipywidgets.ToggleButton(value=False, description=callback.name) + button = ipywidgets.ToggleButton(value=False, description=callback) + button.observe(lambda _: self.redraw_page(), "value") + + # self.callbacks[callback.name] = (callback, button) + self.callbacks[callback] = (callback, button) + overlays_toggle.append(button) + + self.overlays_toggle = ipywidgets.HBox( + overlays_toggle, + layout=ipywidgets.Layout(width="auto", flex_flow="row wrap", display="flex"), + ) + + # -- main view -- + height = 70 + self.document_tabs = ipywidgets.Tab(layout=ipywidgets.Layout(height=f"{height}vh")) + + # -- log text -- + self.log_text = ipywidgets.Output() + + # -- stats -- + self.statistics_text = ipywidgets.HTML() + + # -- whole layout -- + self.layout = ipywidgets.VBox( + ( + navigation_bar, status_bar, self.overlays_toggle, self.document_tabs, + self.statistics_text, self.log_text + ) + ) + + # attach callbacks + previous_button.on_click( + lambda _: self.change_document(document_idx=(self.document_idx - 1)) + ) + random_button.on_click( + lambda _: self.change_document( + document_idx=self._rng.integers(len(self.document_idxs)) + ) + ) + next_button.on_click(lambda _: self.change_document(document_idx=(self.document_idx + 1))) + self.redraw_button.on_click(lambda _: self.redraw_document()) + + self.document_search_text.observe( + lambda change: self._choose_document(change["new"]), "value" + ) + + zoom_slider.observe( + lambda change: self.change_zoom( + width="auto", height="{}%".format(int(100 * change["new"])) + ), + "value", + ) + fit_width_button.on_click(lambda _: self.change_zoom(width="100%", height="auto")) + fit_height_button.on_click(lambda _: self.change_zoom(width="auto", height="100%")) + save_button.on_click(lambda _: self.save_view()) + + self.document_tabs.observe(lambda change: self.redraw_page(), "selected_index") + + # init + display(self.layout) + self.redraw_document() + + @property + def document(self): + self._document = self._change_document(self.dataset.documents[self.document_idx], self._document) + return self._document + + @property + def page(self): + self._page = self._change_page( + self.document, self._page, self.document_tabs.selected_index + ) + return self._page + + @staticmethod + def _change_document(new_document, document): + if document is None: + document = new_document + elif document.docid != new_document.docid: + document = new_document + return document + + @staticmethod + def _change_page(document, page, new_page_n): + if page != new_page_n: + page = new_page_n + return page + + def change_zoom(self, width=None, height=None): + if width is not None: + self.zoom_state["width"] = width + if height is not None: + self.zoom_state["height"] = height + + for child in self.document_tabs.children: + child.layout.width = self.zoom_state["width"] + child.layout.height = self.zoom_state["height"] + + def _choose_document(self, query): + try: + self.change_document(document_idx=int(query)) + except ValueError: + self.change_document(document_id=query) + + def change_document(self, document_idx=None, document_id=None): + if (document_idx is not None) and (document_id is None): + document_idx = np.clip(document_idx, 0, len(self.document_idxs) - 1).tolist() + if document_idx != self.document_idx: + self.clear_context() + self.document_idx = document_idx + with self.document_search_text.hold_trait_notifications(): + self.document_search_text.value = str(self.document_idx) + self.redraw_document() + + def clear_context(self): + """Called whenever the document is about to be changed.""" + self.context.pop("error_message", None) + + def redraw_document(self): + self.redraw_button.disabled = True + + self.document_tabs.children = [ + ipywidgets.HTML() for _ in range(self.document.page_count) + ] + self.document_tabs.selected_index = 0 + + for i, _ in enumerate(self.document_tabs.children): + self.document_tabs.set_title(i, f"{i}") + + self.change_zoom() + self.redraw_page() + + self.redraw_button.disabled = False + + def redraw_page(self): + """ + Progress display related stuff. + Also is a method to handle overlay-button clicks. + """ + self.log_text.clear_output() + self.log_text.__enter__() + was_redrawing = self.redraw_button.disabled + self.redraw_button.disabled = True + + self.status_text.value = f"Rendering {self.document.docid}" + + self.error_text.value = "" + + self.svg_content = self._render_svg_content() + + height = self.document.page_image(self.page).size[1] + svg_data = f""" + + +{self.svg_content} + + """ + + self.document_tabs.children[self.page].value = svg_data + + # close log + self.log_text.__exit__(None, None, None) + + self.error_text.value = '{}'.format( + self.context.get("error_message", "") + ) + + kile_results = self.evaluation_results.get_metrics("kile", docid=self.document.docid) + lir_results = self.evaluation_results.get_metrics("lir", docid=self.document.docid) + + legend = get_legend(PER_LINE=8, HEIGHT=200) + + self.statistics_text.value = f""" +

Legend

+ {legend} +

Error Stats for {self.document.docid}_{self.page}

+

KILE task:

+ + + + + + + + + + + + + + + + + +
APf1precisionrecall
{kile_results["AP"]}{kile_results["f1"]}{kile_results["precision"]}{kile_results["recall"]}
+

LIR task:

+ + + + + + + + + + + + + + + + + +
APf1precisionrecall
{lir_results["AP"]}{lir_results["f1"]}{lir_results["precision"]}{lir_results["recall"]}
+""" + + def _render_svg_content(self): + return self._render_page_svg(self.document, self.page, self.callbacks) + + def _render_page_svg(self, document, page, callbacks): + # img = self.dataset[document].page_image(page) + img = document.page_image(page) + self.error_text.value = "" + # construct html as a list of strings, then join it (speedup) + svg_img = pilimage_to_svg(img) + overlay_elements = [] + for callback, toggle_button in callbacks.values(): + # print(f"DBG_INFO: {callback}, {toggle_button}, {toggle_button.value}") + if toggle_button.value: + if callback == "Annotations_KILE": + gt_kile_fields = self.document.annotation.fields + gt_kile_fields_page = [field for field in gt_kile_fields if field.page == page] + overlay_elements.extend(show_fields(gt_kile_fields_page, img)) + if callback == "Annotations_LIR": + gt_li_fields = self.document.annotation.li_fields + gt_li_fields_page = [field for field in gt_li_fields if field.page == page] + overlay_elements.extend(show_fields(gt_li_fields_page, img)) + if callback == "Predictions_KILE": + predictions = self.kile_predictions[self.document.docid] + predictions_page = [field for field in predictions if field.page == page] + overlay_elements.extend(show_fields(predictions_page, img)) + if callback == "Predictions_LIR": + predictions = self.lir_predictions[self.document.docid] + predictions_page = [field for field in predictions if field.page == page] + overlay_elements.extend(show_fields(predictions_page, img)) + if callback == "Predictions_intermediate": + predictions = self.intermediate_predictions[self.document.docid] + predictions_page = [field for field in predictions if field.page == page] + overlay_elements.extend(show_fields(predictions_page, img)) + return "".join(svg_img + overlay_elements) + + def save_view(self, dir_name="."): + was_disabled = self.redraw_button.disabled + self.redraw_button.disabled = True + + html_template = ( + "\n" + '{content}\n' + "\n" + ) + + os.makedirs(dir_name, exist_ok=True) + + # save html + path = os.path.join(dir_name, "{}_{}.html".format(self.document.id(), str(self.page))) + with open(path, "w") as f: + f.write( + html_template.format( + vh=900, + w=self.SVG_RENDER_WIDTH, + h=self.RENDER_H, + content=self.svg_content.encode("utf-8"), + ) + ) + + self.redraw_button.disabled = was_disabled + + +def load_predictions(fn: Path): + predictions = {} + with open(fn, "r") as json_file: + data = json.load(json_file) + for k, v in data.items(): + predictions[k] = [] + for f in v: + predictions[k].append(Field.from_dict(f)) + return predictions + + +if __name__ == "__main__": + import json + from pathlib import Path + from docile.dataset import Dataset + from docile.dataset import Field + + DATASET_PATH = Path("/storage/pif_documents/dataset_exports/docile221221-0/") + dataset = Dataset("test", DATASET_PATH) + + # PREDICTION_PATH=Path("/storage/table_extraction/predictions/NER/fullpage_multilabel/docile221221-0/LayoutLMv3_wr025/v2/test_intermediate_predictions.json") + + # docid_to_predictions = {} + # if PREDICTION_PATH.exists(): + # docid_to_predictions_raw = json.loads((PREDICTION_PATH).read_text()) + # docid_to_predictions = { + # docid: [Field.from_dict(f) for f in fields] + # for docid, fields in docid_to_predictions_raw.items() + # } + # total_predictions = sum(len(predictions) for predictions in docid_to_predictions.values()) + # print(f"Loaded {total_predictions} predictions for {len(docid_to_predictions)} documents") + # else: + # print("No predictions found.") + + EVALUATION_PATHS = [ + Path("/storage/table_extraction/predictions/NER/fullpage_multilabel/docile221221-0/LayoutLMv3_wr025/v2/test_results_KILE.json"), + Path("/storage/table_extraction/predictions/NER/fullpage_multilabel/docile221221-0/LayoutLMv3_wr025/v2/test_results_LIR.json") + ] + + evaluation_results = EvaluationResult.from_files(*EVALUATION_PATHS) + + intermediate_predictions = load_predictions(Path("/storage/table_extraction/predictions/NER/fullpage_multilabel/docile221221-0/LayoutLMv3_wr025/v2/test_intermediate_predictions.json")) + kile_predictions = load_predictions(Path("/storage/table_extraction/predictions/NER/fullpage_multilabel/docile221221-0/LayoutLMv3_wr025/v2/test_predictions_KILE.json")) + lir_predictions = load_predictions(Path("/storage/table_extraction/predictions/NER/fullpage_multilabel/docile221221-0/LayoutLMv3_wr025/v2/test_predictions_LIR.json")) + + # callbacks = ["Annotations", "Predictions"] + # browser = MyDatasetBrowser(dataset, kile_predictions=docid_to_predictions, callbacks=callbacks) + callbacks = ["Annotations_KILE", "Annotations_LIR", "Predictions_KILE", "Predictions_LIR", "Predictions_intermediate"] + sbrowser = MyDatasetBrowser(dataset, evaluation_results=evaluation_results, kile_predictions=kile_predictions, lir_predictions=lir_predictions, intermediate_predictions=intermediate_predictions, callbacks=callbacks) + + pass \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index c3d4d70..3e358d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,14 +4,19 @@ services: image: jupyter/base-notebook build: context: . + shm_size: '8gb' dockerfile: Dockerfile args: GPU_RUNTIME: nvidia + shm_size: '8gb' + group_add: + - "888" ports: - "${JUPYTER_PORT}:${JUPYTER_PORT}" command: poetry run jupyter lab --ip=0.0.0.0 --port=${JUPYTER_PORT} --allow-root volumes: - "./:/app/:cached" + - "/mnt/shared/ailabs/:/storage" deploy: resources: # consider reducing these limits @@ -20,8 +25,10 @@ services: memory: 100GB reservations: devices: - - capabilities: [gpu] + - driver: "nvidia" + capabilities: [gpu] environment: CUDA_DEVICE_ORDER: PCI_BUS_ID - CUDA_VISIBLE_DEVICES: '' + # CUDA_VISIBLE_DEVICES: '' + # CUDA_VISIBLE_DEVICES: '0,1,2,3,4,5,6,7' privileged: true