DRAFT: Browser: Qualitative Analysis

.env

@@ -1 +1 @@
-JUPYTER_PORT=43603
+JUPYTER_PORT=43604

Dockerfile

@@ -1,20 +1,31 @@
-FROM python:3.10
+# FROM python:3.10
+FROM nvidia/cuda:11.7.0-cudnn8-runtime-ubuntu22.04
 
+ARG DEBIAN_FRONTEND=noninteractive
 RUN apt-get update
+RUN apt-get install python3.10 python3-pip -y
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1
+# Install poetry
+RUN python3 -m pip install poetry
+
+# cv2 dependencies
+RUN apt-get install ffmpeg libsm6 libxext6 -y
 
 # Install opencv-python dependency
-RUN apt-get install libgl1 -y
+# RUN apt-get install libgl1 -y
 
 # Install poppler for pdf2image (converting pdf to images)
 RUN apt-get install poppler-utils -y
 
-# Install poetry
-RUN pip3 install poetry
+# # Install poetry
+# RUN pip3 install poetry
 
 WORKDIR /app
 COPY poetry.lock pyproject.toml /app/
 
 COPY docile /app/docile
+COPY baselines /app/baselines
 COPY README.md /app/README.md
 
-RUN poetry install --no-interaction --with test --with doctr
+# RUN poetry install --no-interaction --with test --with doctr
+RUN poetry install --no-interaction --with test --with doctr --with baselines

docile/evaluation/evaluate.py

@@ -72,6 +72,23 @@ def from_file(cls, path: Path) -> "EvaluationResult":
         }
         return cls(matchings, dct["dataset_name"], dct["iou_threshold"])
 
+    @classmethod
+    def from_files(cls, *paths: Sequence[Path]) -> "EvaluationResult":
+        """Load evaluation results for different tasks from multiple files at once."""
+        if len(paths) == 0:
+            raise ValueError("At least one path must be provided")
+        evaluations = [cls.from_file(path) for path in paths]
+        if len(set(evaluation.dataset_name for evaluation in evaluations)) != 1:
+            raise ValueError("Cannot load evaluations on different datasets")
+        if len(set(evaluation.iou_threshold for evaluation in evaluations)) != 1:
+            raise ValueError("Cannot load evaluations that used different config (iou_threshold)")
+        all_matchings = {}
+        for evaluation in evaluations:
+            if not set(all_matchings.keys()).isdisjoint(evaluation.task_to_docid_to_matching):
+                raise ValueError("Tasks in the evaluations are not disjoint")
+            all_matchings.update(evaluation.task_to_docid_to_matching)
+        return cls(all_matchings, evaluations[0].dataset_name, evaluations[0].iou_threshold)
+
     def get_primary_metric(self, task: str) -> float:
         """Return the primary metric used for DocILE'23 benchmark competition."""
         metric = TASK_TO_PRIMARY_METRIC_NAME[task]