ensemble_detection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "47c66200",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import os\n",
    "import cv2\n",
    "import glob\n",
    "import random\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "from ensemble_boxes import weighted_boxes_fusion,nms\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "94df8868",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def class_to_color(class_id):\n",
    "    colors = {\"abw\":(255,0,0),\"pbw\":(0,255,0),\"comp\":(0,0,255)}\n",
    "    return colors[class_id]\n",
    "\n",
    "def draw_bounding_box(img, annotation,filter_bbox=False):\n",
    "\n",
    "    img_h ,img_w,_ = img.shape\n",
    "    x_center,y_center,w,h = [annotation[1]*img_w,annotation[2]*img_h,annotation[3]*img_w,annotation[4]*img_h,]\n",
    "    x_min, y_min = int(x_center-w/2),int(y_center-h/2)\n",
    "    x_max, y_max = int(x_center+w/2),int(y_center+h/2)\n",
    "    class_id = \"abw\" if annotation[0]==0 else \"pbw\" if annotation[0]==1 else \"comp\"\n",
    "    color = class_to_color(class_id)\n",
    "\n",
    "    cv2.rectangle(img,(x_min,y_min),(x_max,y_max), color, 2)\n",
    "    if len(annotation)==6:\n",
    "        if filter_bbox:\n",
    "            if annotation[0]!=2:\n",
    "                cv2.putText(img = img,\n",
    "                  text = str(annotation[5]),\n",
    "                  org = (x_min, y_min),\n",
    "                  fontFace = cv2.FONT_HERSHEY_DUPLEX,\n",
    "                  fontScale = 0.7,\n",
    "                  color = (255,0,0),\n",
    "                  thickness = 1\n",
    "                )\n",
    "        else:\n",
    "            cv2.putText(img = img,\n",
    "                  text = str(annotation[5]),\n",
    "                  org = (x_min, y_min),\n",
    "                  fontFace = cv2.FONT_HERSHEY_DUPLEX,\n",
    "                  fontScale = 0.7,\n",
    "                  color = (255,0,0),\n",
    "                  thickness = 1\n",
    "                )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "197ca9e5",
   "metadata": {},
   "source": [
    "## WBF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "d77f15bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def get_wbf_data(img_id, annotations,dim ):\n",
    "    boxes = []\n",
    "    scores = []\n",
    "    labels = []\n",
    "    img_h,img_w =  dim\n",
    "    for annotation in annotations:\n",
    "        x_center,y_center,h,w = annotation[1:5]\n",
    "        x_center,w = x_center*img_w, w*img_w\n",
    "        y_center,h = y_center*img_h, h*img_h\n",
    "        \n",
    "        x1,y1 = x_center-w/2,y_center-h/2\n",
    "        x2,y2 = x_center+w/2,y_center+h/2\n",
    "        \n",
    "        x1,x2 = max(0,x1/img_w),min(1,x2/img_w)\n",
    "        y1,y2 = max(0,y1/img_h),min(1,y2/img_h)\n",
    "#         print([x1,y1,x2,y2])\n",
    "        boxes.append([x1,y1,x2,y2])\n",
    "        scores.append(annotation[5])     \n",
    "        labels.append(annotation[0])   \n",
    "    return boxes,scores,labels\n",
    "\n",
    "def get_yolo_data( boxes,scores,labels,dim):\n",
    "    annotations = []\n",
    "    img_h,img_w = dim\n",
    "    for i in range(len(boxes)):\n",
    "        x1,y1,x2,y2 = boxes[i]\n",
    "        x1,x2 = x1*img_w,x2*img_w\n",
    "        y1,y2 = y1*img_h,y2*img_h\n",
    "        W = x2-x1\n",
    "        H= y2-y1\n",
    "        x_center =x1+W/2\n",
    "        y_center = y1+H/2 \n",
    "        x_center /= img_w \n",
    "        y_center /=img_h\n",
    "        W /=img_w\n",
    "        H /=img_h\n",
    "        annotation = f\"{labels[i]} {' '.join(map(str,[x_center,y_center,W,H]))} {scores[i]}\"\n",
    "        annotations.append(annotation)\n",
    "    \n",
    "    return annotations\n",
    "\n",
    "def get_wbf_bbox(img_id):\n",
    "    all_boxes,all_scores,all_labels = [],[],[]\n",
    "\n",
    "    img_id  = img_id.split(\".\")[0]\n",
    "    img_h,img_w,_ = cv2.imread(f\"{raw_data}/jpg/{img_id}.jpg\").shape\n",
    "    for experiment in experiments:\n",
    "        path =f\"{experiment}/labels/{img_id}.txt\"\n",
    "#         print(path)\n",
    "        try:\n",
    "            with open(path) as fp:\n",
    "                annotations =np.array([i.split()[:] for i in fp.read().split(\"\\n\")[:-1]],dtype=float)\n",
    "                boxes,scores,labels = get_wbf_data(img_id,annotations, dim = (img_h,img_w))\n",
    "                all_boxes.append(boxes)\n",
    "                all_scores.append(scores)\n",
    "                all_labels.append(labels)\n",
    "        except FileNotFoundError: \n",
    "            all_boxes.append([])\n",
    "            all_scores.append([])\n",
    "            all_labels.append([]) \n",
    "    try:\n",
    "#         print(iou_thr,skip_box_thr)\n",
    "        boxes,scores,labels = weighted_boxes_fusion(all_boxes, all_scores, all_labels, weights=exp_weights, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n",
    "    \n",
    "    except Exception as e: \n",
    "#         print(img_id,all_boxes)\n",
    "        raise e\n",
    "    out_dict =  {\"img_id\":img_id,\"annotations\":get_yolo_data(boxes,scores,labels,dim =(img_h,img_w))}\n",
    "    return out_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba3f03dc",
   "metadata": {},
   "source": [
    "## Preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d40d59e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_counts(img_id,annotations):\n",
    "    pred_count = {}\n",
    "#     if post_process_check: \n",
    "#         preds = post_process(preds)\n",
    "    abw_count = 0\n",
    "    pbw_count =0\n",
    "    for pred in annotations: \n",
    "        if int(pred[0])==0: abw_count+=1\n",
    "        else: pbw_count+=1        \n",
    "    return {\"img_id\":img_id,\"abw\":abw_count,\"pbw\":pbw_count}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "807ff996",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_df(wbf_processed_data):\n",
    "    wbf_count_df =pd.DataFrame(columns=[\"image_id\",\"worm_type\",\"pred_count\"])\n",
    "    for wbf_img in tqdm(wbf_processed_data,desc= \"Creating DF\"):\n",
    "        img_id = wbf_img[\"img_id\"]\n",
    "        annotations = wbf_img[\"annotations\"]\n",
    "        counts = get_counts(img_id,annotations)\n",
    "        wbf_count_df.loc[len(wbf_count_df)] = [img_id+\".txt\",\"abw\",counts[\"abw\"]]\n",
    "        wbf_count_df.loc[len(wbf_count_df)] = [img_id+\".txt\",\"pbw\",counts[\"pbw\"]] \n",
    "    return wbf_count_df\n",
    "def run_wbf(img_ids):\n",
    "    print(exp_weights)\n",
    "    wbf_processed_data  = [get_wbf_bbox(img_id) for img_id in tqdm(img_ids,desc= \"Running WBF\")]\n",
    "    wbf_count_df  = create_df(wbf_processed_data)\n",
    "    return wbf_count_df,wbf_processed_data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a995dd48",
   "metadata": {},
   "source": [
    "## Compare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b98404ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "def infer(data,plot_img=False,post_process_check=False):\n",
    "    img_id= data[\"img_id\"].split(\".\")[0]\n",
    "    annotations =data[\"annotations\"]\n",
    "    if post_process_check: \n",
    "            annotations = post_process(annotations)\n",
    "    img = cv2.imread(f\"{raw_data}/jpg/{img_id}.jpg\")\n",
    "    img =cv2.cvtColor(img, cv2.COLOR_BGR2RGB) \n",
    "    if plot_img:\n",
    "        plt.imshow(img)\n",
    "        plt.show()\n",
    "    for annotation in annotations:\n",
    "        annotation = list(map(float,annotation.split()))\n",
    "#         print(annotation)\n",
    "        draw_bounding_box(img,annotation)\n",
    "    print(img.shape)\n",
    "\n",
    "    plt.imshow(img)\n",
    "    plt.title(len(annotations))\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "20e2585f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def txt2df(path,post_process_check=True):\n",
    "    files = glob.glob(path+\"/*.txt\")\n",
    "    df = pd.DataFrame(columns=[\"image_id\",\"worm_type\",\"count\"])\n",
    "    if post_process_check: print(\"post_processing\")\n",
    "    for file in files:\n",
    "        with open(file) as fp:\n",
    "            preds =np.array([i.split()[:] for i in fp.read().split(\"\\n\")[:-1]],dtype=float)\n",
    "            if post_process_check: \n",
    "                preds = post_process(preds)\n",
    "            abw = 0\n",
    "            pbw =0\n",
    "            for pred in preds: \n",
    "                if int(pred[0])==0: abw+=1\n",
    "                else: pbw+=1\n",
    "\n",
    "            df.loc[len(df)] = [os.path.basename(file),\"abw\",abw]\n",
    "            df.loc[len(df)] = [os.path.basename(file),\"pbw\",pbw]\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e67862b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SampleSubmission.csv  Train.csv  images.zip\r\n",
      "Test.csv\t      images\t images_bboxes.csv\r\n"
     ]
    }
   ],
   "source": [
    "!ls  data/raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "22156b1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# og_df = txt2df(\"detection/yolov5/detection/yolo_data/labels/validation\",post_process_check=False)\n",
    "# og_df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "b7e8fc71",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "initial_last_256_150\t\t   val_best_1536_100_nms0.3_conf0.3\r\n",
      "raw_test_best_1536_100\t\t   val_best_1536_100_nms0.3_conf0.32\r\n",
      "raw_val_best_1024_100\t\t   val_best_1536_100_nms0.3_conf0.4\r\n",
      "raw_val_best_1024_1002\t\t   val_best_1536_100_nms0.3_conf0.42\r\n",
      "test_best_1536_100_default\t   val_ensemble_best_1536_100_nms0.25_conf0.3\r\n",
      "test_best_1536_100_nms0.3_conf0.3  val_ensemble_best_1536_1024_nms0.3_conf0.3\r\n",
      "test_initial_last_1024_100\t   val_ensemble_best_1536_256_nms0.25_conf0.3\r\n",
      "test_initial_last_256_150\t   val_last_640_50\r\n",
      "test_last_640_50\t\t   val_last_highres++_100\r\n",
      "val_best_1536_100_default\t   val_lowsz_best_1536_100_nms0.3_conf0.3\r\n",
      "val_best_1536_100_nms0.25_conf0.3\r\n"
     ]
    }
   ],
   "source": [
    "!ls detection/yolov5/runs/detect/\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "7a1a9540",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1960\n",
      "1993\n",
      "[10, 1]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Running WBF: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████| 1993/1993 [00:32<00:00, 61.52it/s]\n",
      "Creating DF: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 1993/1993 [00:04<00:00, 443.48it/s]\n"
     ]
    }
   ],
   "source": [
    "iou_thr = 0.3\n",
    "skip_box_thr = 0.3\n",
    "exp_weights = [10,1]\n",
    "experiments = [\"detection/yolov5/runs/detect/raw_test_best_1536_100\",\"detection/detectron2/output1024\"]\n",
    "for i in experiments: print(len(os.listdir(i+\"/labels\")))\n",
    "img_ids = os.listdir(\"detection/yolov5/detection/yolo_data/images/test/\")\n",
    "raw_data = \"data/preproc/\"\n",
    "wbf_count_df,wbf_processed_data  = run_wbf(img_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "de843d14",
   "metadata": {},
   "outputs": [],
   "source": [
    "def post_process(wbf_processed_data,thr):\n",
    "    new_wbf = []\n",
    "    \n",
    "    for data in wbf_processed_data:\n",
    "        new_annotations =[]\n",
    "        for annotation in data[\"annotations\"]:\n",
    "            if float(annotation.split()[-1])>thr:\n",
    "                new_annotations.append(annotation)\n",
    "        new_wbf.append({\"img_id\":data[\"img_id\"],\"annotations\":new_annotations})\n",
    "    return new_wbf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "6e3083c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>worm_type</th>\n",
       "      <th>pred_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id_f0363dbf53755c5900c66b77.txt</td>\n",
       "      <td>abw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id_f0363dbf53755c5900c66b77.txt</td>\n",
       "      <td>pbw</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id_6e52c2f722c67f5b4b2589cc.txt</td>\n",
       "      <td>abw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id_6e52c2f722c67f5b4b2589cc.txt</td>\n",
       "      <td>pbw</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id_716206e74f543068344bc1f7.txt</td>\n",
       "      <td>abw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3981</th>\n",
       "      <td>id_b653f60c63ba7208b8eef848.txt</td>\n",
       "      <td>pbw</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3982</th>\n",
       "      <td>id_306566ec9a376d62e5ceb273.txt</td>\n",
       "      <td>abw</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3983</th>\n",
       "      <td>id_306566ec9a376d62e5ceb273.txt</td>\n",
       "      <td>pbw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3984</th>\n",
       "      <td>id_bfb6d7e393c07c3e6de3feb4.txt</td>\n",
       "      <td>abw</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3985</th>\n",
       "      <td>id_bfb6d7e393c07c3e6de3feb4.txt</td>\n",
       "      <td>pbw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3986 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             image_id worm_type  pred_count\n",
       "0     id_f0363dbf53755c5900c66b77.txt       abw           0\n",
       "1     id_f0363dbf53755c5900c66b77.txt       pbw          61\n",
       "2     id_6e52c2f722c67f5b4b2589cc.txt       abw           0\n",
       "3     id_6e52c2f722c67f5b4b2589cc.txt       pbw           6\n",
       "4     id_716206e74f543068344bc1f7.txt       abw           0\n",
       "...                               ...       ...         ...\n",
       "3981  id_b653f60c63ba7208b8eef848.txt       pbw          32\n",
       "3982  id_306566ec9a376d62e5ceb273.txt       abw           4\n",
       "3983  id_306566ec9a376d62e5ceb273.txt       pbw           0\n",
       "3984  id_bfb6d7e393c07c3e6de3feb4.txt       abw           5\n",
       "3985  id_bfb6d7e393c07c3e6de3feb4.txt       pbw           0\n",
       "\n",
       "[3986 rows x 3 columns]"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wbf_count_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "97d24506",
   "metadata": {},
   "outputs": [],
   "source": [
    "wbf_count_df[\"image_id_worm\"] = wbf_count_df.apply(lambda x: f\"{x['image_id'].split('.')[0]}_{x['worm_type']}\",axis=1)\n",
    "wbf_count_df = wbf_count_df[[\"image_id_worm\",\"pred_count\"]]\n",
    "wbf_count_df.columns = [\"image_id_worm\",\"number_of_worms\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "265a2601",
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_df = pd.read_csv(\"data/raw/SampleSubmission.csv\")\n",
    "# sub_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "48bc01e2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id_worm</th>\n",
       "      <th>number_of_worms</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id_00332970f80fa9a47a39516d_abw</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id_00332970f80fa9a47a39516d_pbw</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id_0035981bc3ae42eb5b57a317_abw</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id_0035981bc3ae42eb5b57a317_pbw</td>\n",
       "      <td>21.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id_005102f664b820f778291dee_abw</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5601</th>\n",
       "      <td>id_ffbcb27fa549278f47505515_pbw</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5602</th>\n",
       "      <td>id_ffc0e41e10b0c964d4a02811_abw</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5603</th>\n",
       "      <td>id_ffc0e41e10b0c964d4a02811_pbw</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5604</th>\n",
       "      <td>id_fff8c253115aacded09ad7ed_abw</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5605</th>\n",
       "      <td>id_fff8c253115aacded09ad7ed_pbw</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5606 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                        image_id_worm  number_of_worms\n",
       "0     id_00332970f80fa9a47a39516d_abw              4.0\n",
       "1     id_00332970f80fa9a47a39516d_pbw              0.0\n",
       "2     id_0035981bc3ae42eb5b57a317_abw              0.0\n",
       "3     id_0035981bc3ae42eb5b57a317_pbw             21.0\n",
       "4     id_005102f664b820f778291dee_abw              9.0\n",
       "...                               ...              ...\n",
       "5601  id_ffbcb27fa549278f47505515_pbw              0.0\n",
       "5602  id_ffc0e41e10b0c964d4a02811_abw              0.0\n",
       "5603  id_ffc0e41e10b0c964d4a02811_pbw              0.0\n",
       "5604  id_fff8c253115aacded09ad7ed_abw              0.0\n",
       "5605  id_fff8c253115aacded09ad7ed_pbw              8.0\n",
       "\n",
       "[5606 rows x 2 columns]"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_df = sub_df[[\"image_id_worm\"]].merge(wbf_count_df, on=[\"image_id_worm\"],how='left').fillna(0)\n",
    "sub_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "5ba12add",
   "metadata": {},
   "outputs": [],
   "source": [
    "# merged_df1 = wbf_count_df.merge(sub_df,on=[\"image_id\",\"worm_type\"],how=\"left\")\n",
    "# merged_df1.columns = [\"image_id\",\"worm_type\",\"pred_count\",\"count\"]\n",
    "# # merged_df1[\"diff\"] = merged_df1.apply(lambda x: abs(x[\"count\"]-x[\"pred_count\"]), axis=1)\n",
    "# merged_df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "e74aaaad",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !ls data/submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "6a0bb4e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id_worm</th>\n",
       "      <th>number_of_worms</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3699</th>\n",
       "      <td>id_a93adfa0157cd5ac1ca7484f_pbw</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        image_id_worm  number_of_worms\n",
       "3699  id_a93adfa0157cd5ac1ca7484f_pbw              0.0"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_df[sub_df[\"image_id_worm\"]==\"id_a93adfa0157cd5ac1ca7484f_pbw\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "2cb7ebd1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5606"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sub_df.image_id_worm.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "916a1be6",
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_df.to_csv(\"data/submission/weighted10:1_ensemble_d2_yolo.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "5ba92c0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import random\n",
    "# for i in random.sample(wbf_post_process,3):\n",
    "#     infer(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "15461c17",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Creating DF: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████| 648/648 [00:01<00:00, 356.99it/s]\n"
     ]
    }
   ],
   "source": [
    "wbf_post_process = post_process(wbf_processed_data,0.2)\n",
    "wbf_post_process_df = create_df(wbf_post_process)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "7bd11453",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pred_count</th>\n",
       "      <th>count</th>\n",
       "      <th>diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1296.000000</td>\n",
       "      <td>1296.000000</td>\n",
       "      <td>1296.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>9.089506</td>\n",
       "      <td>10.167438</td>\n",
       "      <td>2.108796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>24.279289</td>\n",
       "      <td>30.802581</td>\n",
       "      <td>8.848779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>313.000000</td>\n",
       "      <td>453.000000</td>\n",
       "      <td>140.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        pred_count        count         diff\n",
       "count  1296.000000  1296.000000  1296.000000\n",
       "mean      9.089506    10.167438     2.108796\n",
       "std      24.279289    30.802581     8.848779\n",
       "min       0.000000     0.000000     0.000000\n",
       "25%       0.000000     0.000000     0.000000\n",
       "50%       1.000000     1.000000     0.000000\n",
       "75%       7.000000     7.000000     1.000000\n",
       "max     313.000000   453.000000   140.000000"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df2 = wbf_post_process_df.merge(og_df,on=[\"image_id\",\"worm_type\"])\n",
    "merged_df2.columns = [\"image_id\",\"worm_type\",\"pred_count\",\"count\"]\n",
    "merged_df2[\"diff\"] = merged_df2.apply(lambda x: abs(x[\"count\"]-x[\"pred_count\"]), axis=1)\n",
    "merged_df2.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "accc7ad1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>worm_type</th>\n",
       "      <th>pred_count</th>\n",
       "      <th>count</th>\n",
       "      <th>diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id_095efeae209e071b85c680ce.txt</td>\n",
       "      <td>abw</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id_095efeae209e071b85c680ce.txt</td>\n",
       "      <td>pbw</td>\n",
       "      <td>45</td>\n",
       "      <td>55</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id_062e434219b5a3612a414996.txt</td>\n",
       "      <td>abw</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id_062e434219b5a3612a414996.txt</td>\n",
       "      <td>pbw</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id_07741c21ea0cf05171f394ac.txt</td>\n",
       "      <td>abw</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1291</th>\n",
       "      <td>id_22a36dae9a130ff5d7751e93.txt</td>\n",
       "      <td>pbw</td>\n",
       "      <td>33</td>\n",
       "      <td>35</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1292</th>\n",
       "      <td>id_c6af05f4d28a2f31f2735e7d.txt</td>\n",
       "      <td>abw</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1293</th>\n",
       "      <td>id_c6af05f4d28a2f31f2735e7d.txt</td>\n",
       "      <td>pbw</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1294</th>\n",
       "      <td>id_bddefe14f422e0ab79aec5b2.txt</td>\n",
       "      <td>abw</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1295</th>\n",
       "      <td>id_bddefe14f422e0ab79aec5b2.txt</td>\n",
       "      <td>pbw</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1296 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             image_id worm_type  pred_count  count  diff\n",
       "0     id_095efeae209e071b85c680ce.txt       abw           0      0     0\n",
       "1     id_095efeae209e071b85c680ce.txt       pbw          45     55    10\n",
       "2     id_062e434219b5a3612a414996.txt       abw           0      0     0\n",
       "3     id_062e434219b5a3612a414996.txt       pbw           4      5     1\n",
       "4     id_07741c21ea0cf05171f394ac.txt       abw           0      0     0\n",
       "...                               ...       ...         ...    ...   ...\n",
       "1291  id_22a36dae9a130ff5d7751e93.txt       pbw          33     35     2\n",
       "1292  id_c6af05f4d28a2f31f2735e7d.txt       abw           0      0     0\n",
       "1293  id_c6af05f4d28a2f31f2735e7d.txt       pbw           0      2     2\n",
       "1294  id_bddefe14f422e0ab79aec5b2.txt       abw           4      4     0\n",
       "1295  id_bddefe14f422e0ab79aec5b2.txt       pbw           4      0     4\n",
       "\n",
       "[1296 rows x 5 columns]"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "9e61ffe3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(97, 97)"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "thr=5\n",
    "len(merged_df1[merged_df1[\"diff\"]>thr]),len(merged_df2[merged_df2[\"diff\"]>thr])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15959546",
   "metadata": {},
   "source": [
    "## Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "f55f2657",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "initial_last_256_150\t    test_initial_last_256_150  val_last_640_50\r\n",
      "test_initial_last_1024_100  test_last_640_50\t       val_last_highres++_100\r\n"
     ]
    }
   ],
   "source": [
    "!ls detection/yolov5/runs/detect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "5e44dc4c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Creating DF:   0%|                                                                                                                                         | 0/1993 [00:00<?, ?it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:   2%|███                                                                                                                            | 48/1993 [00:00<00:04, 470.59it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:   5%|██████                                                                                                                         | 96/1993 [00:00<00:04, 462.50it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:   7%|█████████                                                                                                                     | 143/1993 [00:00<00:04, 457.98it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:   9%|███████████▉                                                                                                                  | 189/1993 [00:00<00:03, 455.00it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  12%|██████████████▊                                                                                                               | 235/1993 [00:00<00:03, 446.37it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  14%|█████████████████▋                                                                                                            | 280/1993 [00:00<00:03, 446.85it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  16%|████████████████████▌                                                                                                         | 326/1993 [00:00<00:03, 448.70it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  19%|███████████████████████▌                                                                                                      | 372/1993 [00:00<00:03, 449.35it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  21%|██████████████████████████▍                                                                                                   | 418/1993 [00:00<00:03, 449.65it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  23%|█████████████████████████████▎                                                                                                | 463/1993 [00:01<00:03, 449.53it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  25%|████████████████████████████████                                                                                              | 508/1993 [00:01<00:03, 448.16it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  28%|██████████████████████████████████▉                                                                                           | 553/1993 [00:01<00:03, 447.61it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  30%|█████████████████████████████████████▊                                                                                        | 598/1993 [00:01<00:03, 447.00it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  32%|████████████████████████████████████████▋                                                                                     | 643/1993 [00:01<00:03, 446.35it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  35%|███████████████████████████████████████████▍                                                                                  | 688/1993 [00:01<00:02, 445.51it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  37%|██████████████████████████████████████████████▎                                                                               | 733/1993 [00:01<00:02, 444.29it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  39%|█████████████████████████████████████████████████▏                                                                            | 778/1993 [00:01<00:02, 442.04it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  41%|████████████████████████████████████████████████████                                                                          | 823/1993 [00:01<00:02, 440.85it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  44%|██████████████████████████████████████████████████████▉                                                                       | 868/1993 [00:01<00:02, 440.22it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  46%|█████████████████████████████████████████████████████████▋                                                                    | 913/1993 [00:02<00:02, 439.65it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  48%|████████████████████████████████████████████████████████████▌                                                                 | 957/1993 [00:02<00:02, 438.63it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  50%|██████████████████████████████████████████████████████████████▊                                                              | 1001/1993 [00:02<00:02, 437.81it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  52%|█████████████████████████████████████████████████████████████████▌                                                           | 1045/1993 [00:02<00:02, 436.90it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  55%|████████████████████████████████████████████████████████████████████▎                                                        | 1089/1993 [00:02<00:02, 435.91it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  57%|███████████████████████████████████████████████████████████████████████                                                      | 1133/1993 [00:02<00:01, 434.85it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  59%|█████████████████████████████████████████████████████████████████████████▊                                                   | 1177/1993 [00:02<00:01, 433.81it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  61%|████████████████████████████████████████████████████████████████████████████▌                                                | 1221/1993 [00:02<00:01, 432.85it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  63%|███████████████████████████████████████████████████████████████████████████████▎                                             | 1265/1993 [00:02<00:01, 431.85it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  66%|██████████████████████████████████████████████████████████████████████████████████                                           | 1309/1993 [00:02<00:01, 431.21it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  68%|████████████████████████████████████████████████████████████████████████████████████▊                                        | 1353/1993 [00:03<00:01, 430.42it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  70%|███████████████████████████████████████████████████████████████████████████████████████▌                                     | 1397/1993 [00:03<00:01, 429.25it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  72%|██████████████████████████████████████████████████████████████████████████████████████████▎                                  | 1440/1993 [00:03<00:01, 428.25it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  74%|█████████████████████████████████████████████████████████████████████████████████████████████                                | 1483/1993 [00:03<00:01, 427.62it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  77%|███████████████████████████████████████████████████████████████████████████████████████████████▋                             | 1526/1993 [00:03<00:01, 426.51it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  79%|██████████████████████████████████████████████████████████████████████████████████████████████████▍                          | 1569/1993 [00:03<00:00, 424.80it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  81%|█████████████████████████████████████████████████████████████████████████████████████████████████████                        | 1612/1993 [00:03<00:00, 423.44it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  83%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊                     | 1655/1993 [00:03<00:00, 422.64it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍                  | 1698/1993 [00:03<00:00, 421.66it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▏               | 1741/1993 [00:03<00:00, 420.62it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▉             | 1784/1993 [00:04<00:00, 419.31it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌          | 1826/1993 [00:04<00:00, 418.42it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏       | 1868/1993 [00:04<00:00, 417.63it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF:  96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 1910/1993 [00:04<00:00, 416.48it/s]\u001b[A\u001b[A\n",
      "\n",
      "Creating DF: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1993/1993 [00:04<00:00, 434.34it/s]\u001b[A\u001b[A\n"
     ]
    }
   ],
   "source": [
    "iou_thr = 0.4\n",
    "skip_box_thr = 0.4\n",
    "experiments = [\"test_initial_last_256_150\",\"test_initial_last_1024_100\"]\n",
    "test_data = pd.read_csv(\"data/preds/test_droupout0.3_filter_preds.csv\")\n",
    "img_ids =test_data[test_data[\"preds\"]==1].image_id_worm.values\n",
    "raw_data = \"data/preproc/\"\n",
    "wbf_count_df  = run_wbf(img_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "1b43bde8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>worm_type</th>\n",
       "      <th>pred_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id_00332970f80fa9a47a39516d.txt</td>\n",
       "      <td>abw</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id_00332970f80fa9a47a39516d.txt</td>\n",
       "      <td>pbw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id_0035981bc3ae42eb5b57a317.txt</td>\n",
       "      <td>abw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id_0035981bc3ae42eb5b57a317.txt</td>\n",
       "      <td>pbw</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id_005102f664b820f778291dee.txt</td>\n",
       "      <td>abw</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3981</th>\n",
       "      <td>id_ffad8f3773a4222f8fe5ba1a.txt</td>\n",
       "      <td>pbw</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3982</th>\n",
       "      <td>id_ffb65e6de900c49d8f2ef95a.txt</td>\n",
       "      <td>abw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3983</th>\n",
       "      <td>id_ffb65e6de900c49d8f2ef95a.txt</td>\n",
       "      <td>pbw</td>\n",
       "      <td>124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3984</th>\n",
       "      <td>id_fff8c253115aacded09ad7ed.txt</td>\n",
       "      <td>abw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3985</th>\n",
       "      <td>id_fff8c253115aacded09ad7ed.txt</td>\n",
       "      <td>pbw</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3986 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             image_id worm_type  pred_count\n",
       "0     id_00332970f80fa9a47a39516d.txt       abw           4\n",
       "1     id_00332970f80fa9a47a39516d.txt       pbw           0\n",
       "2     id_0035981bc3ae42eb5b57a317.txt       abw           0\n",
       "3     id_0035981bc3ae42eb5b57a317.txt       pbw          23\n",
       "4     id_005102f664b820f778291dee.txt       abw           9\n",
       "...                               ...       ...         ...\n",
       "3981  id_ffad8f3773a4222f8fe5ba1a.txt       pbw           8\n",
       "3982  id_ffb65e6de900c49d8f2ef95a.txt       abw           0\n",
       "3983  id_ffb65e6de900c49d8f2ef95a.txt       pbw         124\n",
       "3984  id_fff8c253115aacded09ad7ed.txt       abw           0\n",
       "3985  id_fff8c253115aacded09ad7ed.txt       pbw           8\n",
       "\n",
       "[3986 rows x 3 columns]"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wbf_count_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "8400006d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id_worm</th>\n",
       "      <th>number_of_worms</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id_00332970f80fa9a47a39516d_abw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id_00332970f80fa9a47a39516d_pbw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id_0035981bc3ae42eb5b57a317_abw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id_0035981bc3ae42eb5b57a317_pbw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id_005102f664b820f778291dee_abw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5601</th>\n",
       "      <td>id_ffbcb27fa549278f47505515_pbw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5602</th>\n",
       "      <td>id_ffc0e41e10b0c964d4a02811_abw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5603</th>\n",
       "      <td>id_ffc0e41e10b0c964d4a02811_pbw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5604</th>\n",
       "      <td>id_fff8c253115aacded09ad7ed_abw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5605</th>\n",
       "      <td>id_fff8c253115aacded09ad7ed_pbw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5606 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                        image_id_worm  number_of_worms\n",
       "0     id_00332970f80fa9a47a39516d_abw                0\n",
       "1     id_00332970f80fa9a47a39516d_pbw                0\n",
       "2     id_0035981bc3ae42eb5b57a317_abw                0\n",
       "3     id_0035981bc3ae42eb5b57a317_pbw                0\n",
       "4     id_005102f664b820f778291dee_abw                0\n",
       "...                               ...              ...\n",
       "5601  id_ffbcb27fa549278f47505515_pbw                0\n",
       "5602  id_ffc0e41e10b0c964d4a02811_abw                0\n",
       "5603  id_ffc0e41e10b0c964d4a02811_pbw                0\n",
       "5604  id_fff8c253115aacded09ad7ed_abw                0\n",
       "5605  id_fff8c253115aacded09ad7ed_pbw                0\n",
       "\n",
       "[5606 rows x 2 columns]"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_df =pd.read_csv(\"data/raw/SampleSubmission.csv\")\n",
    "sub_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "71b2ffa6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id_worm</th>\n",
       "      <th>number_of_worms</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id_00332970f80fa9a47a39516d_abw</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id_00332970f80fa9a47a39516d_pbw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id_0035981bc3ae42eb5b57a317_abw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id_0035981bc3ae42eb5b57a317_pbw</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id_005102f664b820f778291dee_abw</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3981</th>\n",
       "      <td>id_ffad8f3773a4222f8fe5ba1a_pbw</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3982</th>\n",
       "      <td>id_ffb65e6de900c49d8f2ef95a_abw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3983</th>\n",
       "      <td>id_ffb65e6de900c49d8f2ef95a_pbw</td>\n",
       "      <td>124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3984</th>\n",
       "      <td>id_fff8c253115aacded09ad7ed_abw</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3985</th>\n",
       "      <td>id_fff8c253115aacded09ad7ed_pbw</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3986 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                        image_id_worm  number_of_worms\n",
       "0     id_00332970f80fa9a47a39516d_abw                4\n",
       "1     id_00332970f80fa9a47a39516d_pbw                0\n",
       "2     id_0035981bc3ae42eb5b57a317_abw                0\n",
       "3     id_0035981bc3ae42eb5b57a317_pbw               23\n",
       "4     id_005102f664b820f778291dee_abw                9\n",
       "...                               ...              ...\n",
       "3981  id_ffad8f3773a4222f8fe5ba1a_pbw                8\n",
       "3982  id_ffb65e6de900c49d8f2ef95a_abw                0\n",
       "3983  id_ffb65e6de900c49d8f2ef95a_pbw              124\n",
       "3984  id_fff8c253115aacded09ad7ed_abw                0\n",
       "3985  id_fff8c253115aacded09ad7ed_pbw                8\n",
       "\n",
       "[3986 rows x 2 columns]"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wbf_count_df[\"image_id_worm\"] = wbf_count_df.apply(lambda x: f\"{x['image_id'][:-4]}_{x['worm_type']}\",axis=1)\n",
    "wbf_count_df = wbf_count_df[[\"image_id_worm\",\"pred_count\"]]\n",
    "wbf_count_df.columns  = [\"image_id_worm\",\"number_of_worms\"]\n",
    "\n",
    "wbf_count_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "9d6d8645",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id_worm</th>\n",
       "      <th>number_of_worms</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id_00332970f80fa9a47a39516d_abw</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id_00332970f80fa9a47a39516d_pbw</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id_0035981bc3ae42eb5b57a317_abw</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id_0035981bc3ae42eb5b57a317_pbw</td>\n",
       "      <td>23.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id_005102f664b820f778291dee_abw</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5601</th>\n",
       "      <td>id_ffbcb27fa549278f47505515_pbw</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5602</th>\n",
       "      <td>id_ffc0e41e10b0c964d4a02811_abw</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5603</th>\n",
       "      <td>id_ffc0e41e10b0c964d4a02811_pbw</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5604</th>\n",
       "      <td>id_fff8c253115aacded09ad7ed_abw</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5605</th>\n",
       "      <td>id_fff8c253115aacded09ad7ed_pbw</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5606 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                        image_id_worm  number_of_worms\n",
       "0     id_00332970f80fa9a47a39516d_abw              4.0\n",
       "1     id_00332970f80fa9a47a39516d_pbw              0.0\n",
       "2     id_0035981bc3ae42eb5b57a317_abw              0.0\n",
       "3     id_0035981bc3ae42eb5b57a317_pbw             23.0\n",
       "4     id_005102f664b820f778291dee_abw              9.0\n",
       "...                               ...              ...\n",
       "5601  id_ffbcb27fa549278f47505515_pbw              0.0\n",
       "5602  id_ffc0e41e10b0c964d4a02811_abw              0.0\n",
       "5603  id_ffc0e41e10b0c964d4a02811_pbw              0.0\n",
       "5604  id_fff8c253115aacded09ad7ed_abw              0.0\n",
       "5605  id_fff8c253115aacded09ad7ed_pbw              8.0\n",
       "\n",
       "[5606 rows x 2 columns]"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_df = sub_df[[\"image_id_worm\"]].merge(wbf_count_df, on=[\"image_id_worm\"],how='left').fillna(0)\n",
    "sub_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "395e2b7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_df.to_csv(\"data/submission/ensemble_256_1024_150.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9491903",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}