Week3_code.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# WEEK 3 CODE\n",
    "\n",
    "### Content:\n",
    "* TASK 1\n",
    "* TASK 2\n",
    "* TASK 3\n",
    "* TASK 4\n",
    "* TASK 5\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#IMPORTS\n",
    "import sys\n",
    "import cv2\n",
    "import os\n",
    "import os.path\n",
    "import numpy as np\n",
    "import math\n",
    "np.set_printoptions(threshold=np.inf)\n",
    "import matplotlib.patches as patches\n",
    "import ml_metrics as metrics\n",
    "import pickle\n",
    "from matplotlib import pyplot as plt\n",
    "from os.path import isfile, join\n",
    "from os import listdir\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import recall_score,precision_score,f1_score\n",
    "from scipy.spatial import distance\n",
    "from scipy.signal import convolve2d\n",
    "from scipy.fftpack import dct\n",
    "from scipy import ndimage\n",
    "from skimage.feature import local_binary_pattern\n",
    "from skimage.feature import hog\n",
    "from skimage.filters import roberts, sobel, sobel_h, sobel_v\n",
    "from skimage.morphology import skeletonize, dilation, square, opening, closing, erosion\n",
    "from pathlib import Path\n",
    "from cv2 import boundingRect\n",
    "from PIL import Image, ImageDraw"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TASK 1: Filter noise with linear or non-linear filters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DEFINE FUNCTIONS THAT WILL BE USED IN TASK 1\n",
    "\n",
    "def detectnoise(img):\n",
    "    # Calcula la sigma del soroll. Llavors les imatges que tenen soroll tenen una sigma més gran \n",
    "    #que la resta de les imatges\n",
    "    thr = 8.0\n",
    "    H, W = gray.shape\n",
    "\n",
    "    M = [[1, -2, 1],\n",
    "        [-2, 4, -2],\n",
    "        [1, -2, 1]]\n",
    "    \n",
    "    sigma = np.sum(np.sum(np.absolute(convolve2d(gray, M))))\n",
    "    sigma = sigma * math.sqrt(0.5 * math.pi) / (6 * (W-2) * (H-2))\n",
    "    if sigma>thr:\n",
    "        return True\n",
    "    else:\n",
    "        return False\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "query2folder = './qst2_w3/'\n",
    "#query2folder = './qsd2_w3/'\n",
    "# CREEM UN DIRECTORI ON GUARDEM LES IMATGES SENSE SOROLL\n",
    "path = 'qst2_w3_denoise'\n",
    "if not os.path.exists(path):\n",
    "    os.mkdir(path)\n",
    "\n",
    "\n",
    "for filename in sorted(listdir(query2folder)):\n",
    "    if(filename != '.DS_Store' and ('jpg' in filename)):\n",
    "        #print (\"\\r Processing image...   {}\".format(query2folder+filename), end=\"\")\n",
    "        query_img = cv2.imread(query2folder + filename)\n",
    "        gray = cv2.cvtColor(query_img, cv2.COLOR_BGR2GRAY)\n",
    "        noise = detectnoise(gray)\n",
    "        if noise:\n",
    "            denoise_img = cv2.bilateralFilter(query_img, 9, 120, 120)\n",
    "            denoise_img_2 = cv2.fastNlMeansDenoisingColored(denoise_img,None,15,15,10,21) \n",
    "            #plt.figure()\n",
    "            #plt.imshow(denoise_img)\n",
    "            #plt.figure()\n",
    "            #plt.imshow(denoise_img_2)\n",
    "        else:\n",
    "            denoise_img_2 = query_img\n",
    "        \n",
    "        cv2.imwrite(os.path.join(path,filename), denoise_img_2)\n",
    "\n",
    "            \n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TASK 2: Implement text descriptors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_part_box(crop,ratio,box,a,counter,center):\n",
    "    pos_bboxes = []\n",
    "    pos_bbox = []\n",
    "    area = []\n",
    "\n",
    "    for c, color in enumerate(center):\n",
    "        pos = np.where(crop == color)\n",
    "        aux = np.zeros(crop.shape[:2], dtype='uint8')\n",
    "        aux[pos[0], pos[1]] = 1\n",
    "        aux = cv2.morphologyEx(aux, cv2.MORPH_OPEN, np.ones((5, 5), np.uint8))\n",
    "        ret, labels = cv2.connectedComponents(aux)\n",
    "\n",
    "        for l in range(ret):\n",
    "            lab_pos = np.where(labels == l)\n",
    "\n",
    "            if (any(lab_pos[1] == 200) and any(lab_pos[1] == 250) and any(lab_pos[1] == 150)) and (\n",
    "                    any(lab_pos[0] == 20) or any(lab_pos[0] == 40)\n",
    "                    or any(lab_pos[0] == 60) or any(lab_pos[0] == 75)):\n",
    "                continue\n",
    "            else:\n",
    "                aux[lab_pos[0], lab_pos[1]] = 0\n",
    "\n",
    "        area.append(np.sum(aux))\n",
    "        pos_bboxes.append(np.where(aux > 0))\n",
    "\n",
    "    idx = sorted(range(len(area)), key=lambda k: area[k])\n",
    "\n",
    "    if len(idx) > 1:\n",
    "\n",
    "        aux1 = np.zeros(crop.shape[:2], dtype='uint8')\n",
    "        aux1[pos_bboxes[idx[-1]][0], pos_bboxes[idx[-1]][1]] = 255\n",
    "        (x, y, w, h) = cv2.boundingRect(aux1)\n",
    "        aux_col1 = np.stack((aux1, aux1, aux1), axis=2)\n",
    "        if (w * h) is 0:\n",
    "            ratio[counter] = 0\n",
    "            box[:, :, counter] = aux1\n",
    "            a[counter] = area[idx[-1]]\n",
    "        else:\n",
    "            cv2.rectangle(aux_col1, (x, y), (x + w, y + h), (0, 0, 255), 2)\n",
    "            ratio1 = (np.sum(aux1[y:y + h, x:x + w]) / 255) / (w * h)\n",
    "\n",
    "            if not area[idx[-2]] == 0:\n",
    "\n",
    "                aux2 = np.zeros(crop.shape[:2], dtype='uint8')\n",
    "                aux2[pos_bboxes[idx[-2]][0], pos_bboxes[idx[-2]][1]] = 255\n",
    "                (x, y, w, h) = cv2.boundingRect(aux2)\n",
    "                aux_col2 = np.stack((aux2, aux2, aux2), axis=2)\n",
    "                cv2.rectangle(aux_col2, (x, y), (x + w, y + h), (0, 0, 255), 2)\n",
    "\n",
    "                ratio2 = (np.sum(aux2[y:y + h, x:x + w]) / 255) / (w * h)\n",
    "\n",
    "                if ratio1 > ratio2:\n",
    "                    ratio[counter] = ratio1\n",
    "                    box[:, :, counter] = aux1\n",
    "                    a[counter] = area[idx[-1]]\n",
    "                else:\n",
    "                    ratio[counter] = ratio2\n",
    "                    box[:, :, counter] = aux2\n",
    "                    a[counter] = area[idx[-2]]\n",
    "\n",
    "            else:\n",
    "                ratio[counter] = ratio1\n",
    "                box[:, :, counter] = aux1\n",
    "                a[counter] = area[idx[-1]]\n",
    "                        \n",
    "    return ratio,box,a\n",
    "\n",
    "def detect_corners(mask):\n",
    "    \"\"\"\n",
    "    Finds four points corresponding to rectangle corners\n",
    "\n",
    "    :param mask: (ndarray) binary image\n",
    "    :return: (int) points from corners\n",
    "    \"\"\"\n",
    "\n",
    "    width = mask.shape[1]\n",
    "    height = mask.shape[0]\n",
    "    coords = np.argwhere(np.ones([height, width]))\n",
    "    coords_x = coords[:, 1]\n",
    "    coords_y = coords[:, 0]\n",
    "\n",
    "    coords_x_filtered = np.extract(mask, coords_x)\n",
    "    coords_y_filtered = np.extract(mask, coords_y)\n",
    "    max_br = np.argmax(coords_x_filtered + coords_y_filtered)\n",
    "    max_tr = np.argmax(coords_x_filtered - coords_y_filtered)\n",
    "    max_tl = np.argmax(-coords_x_filtered - coords_y_filtered)\n",
    "    max_bl = np.argmax(-coords_x_filtered + coords_y_filtered)\n",
    "\n",
    "    tl_x, tl_y = int(coords_x_filtered[max_tl]), int(coords_y_filtered[max_tl])\n",
    "    tr_x, tr_y = int(coords_x_filtered[max_tr]), int(coords_y_filtered[max_tr])\n",
    "    bl_x, bl_y = int(coords_x_filtered[max_bl]), int(coords_y_filtered[max_bl])\n",
    "    br_x, br_y = int(coords_x_filtered[max_br]), int(coords_y_filtered[max_br])\n",
    "\n",
    "    return tl_x, tl_y, bl_x, bl_y, br_x, br_y, tr_x, tr_y\n",
    "\n",
    "def find_box(query_img):\n",
    "    \n",
    "    query_img = cv2.cvtColor(query_img, cv2.COLOR_BGR2RGB)\n",
    "\n",
    "    resize_size = (500, 500)\n",
    "    image_shape = query_img.shape[:2]\n",
    "    img = cv2.resize(query_img, resize_size)\n",
    "\n",
    "    image_dark = cv2.morphologyEx(img, cv2.MORPH_OPEN, np.ones((6, 10), np.uint8))\n",
    "    image_white = cv2.morphologyEx(img, cv2.MORPH_CLOSE, np.ones((6, 10), np.uint8))\n",
    "\n",
    "    images = np.stack((image_dark, image_white))\n",
    "\n",
    "    ratio_top = np.zeros(2)\n",
    "    box_top = np.zeros((100, 400, 2), np.uint8)\n",
    "    area_top = np.zeros(2)\n",
    "\n",
    "    ratio_bottom = np.zeros(2)\n",
    "    box_bottom = np.zeros((100, 400, 2), np.uint8)\n",
    "    area_bottom = np.zeros(2)\n",
    "\n",
    "    count = 0\n",
    "    for image in images:\n",
    "        # plt.figure()\n",
    "        # plt.imshow(image)\n",
    "\n",
    "        Z = image.reshape((-1, 3))\n",
    "        # convert to np.float32\n",
    "        Z = np.float32(Z)\n",
    "\n",
    "        criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 10, 1.0)\n",
    "        ret, label, center = cv2.kmeans(Z, 12, None, criteria, 10, cv2.KMEANS_RANDOM_CENTERS)\n",
    "\n",
    "        center = np.uint8(center)\n",
    "        res = center[label.flatten()]\n",
    "        res2 = res.reshape((image.shape))\n",
    "\n",
    "        crop_top = res2[:100, 50:-50]\n",
    "        crop_bottom = res2[-100:, 50:-50]\n",
    "\n",
    "        '''plt.figure()\n",
    "        plt.subplot(121)\n",
    "        plt.imshow(crop_top)\n",
    "        plt.subplot(122)\n",
    "        plt.imshow(crop_bottom)'''\n",
    "\n",
    "        mask = np.zeros(resize_size)\n",
    "\n",
    "        ratio_top,box_top,area_top = find_part_box(crop_top,ratio_top,box_top,area_top,count,center)\n",
    "        ratio_bottom,box_bottom,area_bottom = find_part_box(crop_bottom,ratio_bottom,box_bottom,area_bottom,count,center)\n",
    "\n",
    "        count += 1\n",
    "\n",
    "    if ratio_top[0] > ratio_top[1]:\n",
    "        final_ratio_top = ratio_top[0]\n",
    "        final_box_top = box_top[:,:,0]\n",
    "    else:\n",
    "        final_ratio_top = ratio_top[1]\n",
    "        final_box_top = box_top[:,:,1]\n",
    "\n",
    "    if ratio_bottom[0] > ratio_bottom[1]:\n",
    "        final_ratio_bottom = ratio_bottom[0]\n",
    "        final_box_bottom = box_bottom[:,:,0]\n",
    "    else:\n",
    "        final_ratio_bottom = ratio_bottom[1]\n",
    "        final_box_bottom = box_bottom[:,:,1]\n",
    "\n",
    "    top = np.where(final_box_top>0)\n",
    "    bottom = np.where(final_box_bottom>0)\n",
    "\n",
    "    mask = np.zeros(resize_size,np.uint8)\n",
    "    mask[top[0],top[1]+50] = 1\n",
    "    mask[bottom[0]+400,bottom[1]+50] = 1\n",
    "\n",
    "    mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, np.ones((5, 5), np.uint8))        \n",
    "    mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, np.ones((5, 5), np.uint8))\n",
    "    mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, np.ones((5, 5), np.uint8))\n",
    "\n",
    "    mask = cv2.resize(mask,(image_shape[1],image_shape[0]))\n",
    "\n",
    "    ret, labels = cv2.connectedComponents(mask)\n",
    "\n",
    "    max_ratio=0\n",
    "    \n",
    "    bbox=[]\n",
    "    \n",
    "    for label in range(ret):\n",
    "\n",
    "        if np.sum(mask[labels==label])==0:\n",
    "            continue\n",
    "\n",
    "        aux = np.zeros(image_shape,np.uint8)\n",
    "        aux[labels==label]=1\n",
    "        (x, y, w, h) = cv2.boundingRect(aux)\n",
    "\n",
    "        ratio = (np.sum(aux[y:y + h, x:x + w])) / (w * h)\n",
    "\n",
    "        if ratio >= max_ratio:\n",
    "            max_ratio = ratio\n",
    "            final_mask = aux\n",
    "\n",
    "            corners = detect_corners(aux)\n",
    "\n",
    "            bbox = [corners[0],corners[1],corners[4],corners[5]]\n",
    "\n",
    "    return bbox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "query2folder = './qsd1_w3_denoise/'\n",
    "\n",
    "bboxes=[]\n",
    "\n",
    "for filename in sorted(listdir(query2folder)):\n",
    "    if (filename != '.DS_Store' and ('jpg' in filename)):\n",
    "        print (\"\\r Processing image...   {}\".format(query2folder+filename), end=\"\")\n",
    "        query_img = cv2.imread(query2folder + filename)\n",
    "        bbox = find_box(query_img)\n",
    "        bboxes.append([bbox])\n",
    "        \n",
    "pickle_out = open(\"qst1_w3_text.pkl\", \"wb\")\n",
    "pickle.dump(bboxes, pickle_out)\n",
    "pickle_out.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read txt from bbdd_text \n",
    "import codecs\n",
    "textfolder = './bbdd_text/'\n",
    "bbdd_text = []\n",
    "for filename in sorted(listdir(textfolder)):\n",
    "    if(filename != '.DS_Store' and ('txt' in filename)):\n",
    "        with codecs.open(textfolder + filename,'r',encoding='latin1') as f:\n",
    "            if  os.stat(textfolder + filename).st_size == 0:\n",
    "                line = ''\n",
    "                bbdd_text.append(line)\n",
    "                #print(filename + line)\n",
    "            else:\n",
    "                for line in f.readlines():\n",
    "                    lim1 = line.find(\"'\",1)\n",
    "                    lim2 = line.find(\"'\",2)\n",
    "                    bbdd_text.append(line[lim1+1:lim2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read txt from GT \n",
    "textfolder = './qst1_w3/'\n",
    "GTtext = []\n",
    "for filename in sorted(listdir(textfolder)):\n",
    "    if(filename != '.DS_Store' and ('txt' in filename)):\n",
    "        with codecs.open(textfolder + filename,'r',encoding='latin1') as f:\n",
    "            if  os.stat(textfolder + filename).st_size == 0:\n",
    "                line = ''\n",
    "                GTtext.append(line)\n",
    "                #print(filename + line)\n",
    "            else:\n",
    "                for line in f.readlines():\n",
    "                    lim1 = line.find(\"'\",1)\n",
    "                    lim2 = line.find(\"'\",2) \n",
    "                    GTtext.append(line[lim1+1:lim2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DETECT TEXTS INTO BBOXES\n",
    "\n",
    "#brew install tesseract\n",
    "import pytesseract\n",
    "#import pytesseract\n",
    "with open('qst1_w3_text.pkl', 'rb') as fd:\n",
    "    ll = pickle.load(fd)\n",
    "    #print(ll)\n",
    "            \n",
    "query2folder = './qst1_w3_denoise/'\n",
    "count = 0\n",
    "query_text = []\n",
    "for filename in sorted(listdir(query2folder)):\n",
    "    if(filename != '.DS_Store' and ('jpg' in filename)):\n",
    "        query_img = cv2.imread(query2folder + filename)\n",
    "        img = query_img[ll[count][0][1]:ll[count][0][3],ll[count][0][0]:ll[count][0][2]]\n",
    "        extractedInformation = pytesseract.image_to_string(img)\n",
    "        query_text.append(extractedInformation)\n",
    "        count = count + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def levenshtein(seq1, seq2):\n",
    "    size_x = len(seq1) + 1\n",
    "    size_y = len(seq2) + 1\n",
    "    matrix = np.zeros ((size_x, size_y))\n",
    "    for x in range(size_x):\n",
    "        matrix [x, 0] = x\n",
    "    for y in range(size_y):\n",
    "        matrix [0, y] = y\n",
    "\n",
    "    for x in range(1, size_x):\n",
    "        for y in range(1, size_y):\n",
    "            if seq1[x-1] == seq2[y-1]:\n",
    "                matrix [x,y] = min(\n",
    "                    matrix[x-1, y] + 1,\n",
    "                    matrix[x-1, y-1],\n",
    "                    matrix[x, y-1] + 1\n",
    "                )\n",
    "            else:\n",
    "                matrix [x,y] = min(\n",
    "                    matrix[x-1,y] + 1,\n",
    "                    matrix[x-1,y-1] + 1,\n",
    "                    matrix[x,y-1] + 1\n",
    "                )\n",
    "    #print (matrix)\n",
    "    return (matrix[size_x - 1, size_y - 1])\n",
    "\n",
    "def computeMatrixRetrievalText(bbdd_text,query_text,K=10):\n",
    "    dst = np.zeros((30, 280))\n",
    "    matrix_retrieval_text = np.zeros((30,K))\n",
    "    \n",
    "    for query_text_index in range(0,len(query_text)):\n",
    "        for bbdd_text_index in range(0,len(bbdd_text)):\n",
    "            dst[query_text_index,bbdd_text_index] = levenshtein(query_text[query_text_index], bbdd_text[bbdd_text_index])\n",
    "            #print(bbdd_text_index)\n",
    "        matrix_retrieval_text[query_text_index,:] = np.argsort(dst[query_text_index,:])[:K]\n",
    "        \n",
    "     #Convertir idx a list of lists\n",
    "    matrix_retrieval_text = matrix_retrieval_text.astype(int)\n",
    "    matrix_retrieval_text_lst = matrix_retrieval_text.tolist()\n",
    "    return matrix_retrieval_text_lst  \n",
    "\n",
    "\n",
    "def evaluationTask1(matrix_retrieval, K, query_folder):  \n",
    "    #LLEGIR GT\n",
    "    with open(query_folder + 'gt_corresps.pkl', 'rb') as fd:\n",
    "            ll = pickle.load(fd)\n",
    "            #print(ll)\n",
    "    gt = np.empty((0,0))\n",
    "\n",
    "    ll_prp = np.zeros((len(ll),1))\n",
    "\n",
    "    for i in range(0,len(ll)):\n",
    "        ll_prp[i] = ll[i][0]\n",
    "\n",
    "    ll_prp_lst = ll_prp.tolist()\n",
    "    #print(ll_prp_lst)\n",
    "\n",
    "    mapak = metrics.average_precision.mapk(ll_prp_lst, matrix_retrieval, k=K)    \n",
    "    score = mapak*100\n",
    "    #print(\"SCORE: \",score, \"%\")\n",
    "\n",
    "    #GUARDAR PICKLE RESULT\n",
    "    pickle_out = open(\"result_qsd2.pkl\", \"wb\")\n",
    "    pickle.dump(matrix_retrieval, pickle_out)\n",
    "    pickle_out.close()\n",
    "    \n",
    "    return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "K = 10\n",
    "query_folder = './qsd1_w3/'\n",
    "textMatrixRetrieval = computeMatrixRetrievalText(bbdd_text,query_text,K=10)\n",
    "score = evaluationTask1(textMatrixRetrieval,K,query_folder)\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TASK 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Implement texture descriptors \n",
    "Descriptors:\n",
    "* LBP\n",
    "* DCT\n",
    "* HoG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def zigzag(input):\n",
    "\n",
    "    # initializing the variables\n",
    "    # ----------------------------------\n",
    "    h = 0\n",
    "    v = 0\n",
    "    vmin = 0\n",
    "    hmin = 0\n",
    "    vmax = input.shape[0]-1\n",
    "    hmax = input.shape[1]-1\n",
    "\n",
    "    i = 0\n",
    "    output = np.zeros((vmax+1) * (hmax+1))\n",
    "    # ----------------------------------\n",
    "\n",
    "    while v <= vmax and h <= hmax:\n",
    "        if np.mod(h + v, 2) == 0:\n",
    "            if v == vmin:\n",
    "                output[i] = input[v, h]     # if we got to the first line\n",
    "                if h == hmax:\n",
    "                    v += 1\n",
    "                else:\n",
    "                    h += 1\n",
    "                i += 1\n",
    "                continue\n",
    "            if h == hmax and v < vmax:      # if we got to the last column\n",
    "                output[i] = input[v, h]\n",
    "                v += 1\n",
    "                i += 1\n",
    "            if v > vmin and h < hmax:       # all other cases\n",
    "                output[i] = input[v, h]\n",
    "                v -= 1\n",
    "                h += 1\n",
    "                i += 1\n",
    "        else:\n",
    "            if v == vmax and h <= hmax:     # if we got to the last line\n",
    "                output[i] = input[v, h]\n",
    "                h += 1\n",
    "                i += 1\n",
    "            if v != vmax and h == hmin:     # if we got to the first column\n",
    "                output[i] = input[v, h]\n",
    "                if v == vmax:\n",
    "                    h += 1\n",
    "                else:\n",
    "                    v += 1\n",
    "                i += 1\n",
    "            if v < vmax and h > hmin:       # all other cases\n",
    "                output[i] = input[v, h]\n",
    "                v += 1\n",
    "                h -= 1\n",
    "                i += 1\n",
    "        if v == vmax and h == hmax:         # bottom right element\n",
    "            output[i] = input[v, h]\n",
    "            break\n",
    "\n",
    "    return output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def colorDescriptor(img,level,numberBins):\n",
    "    img = cv2.cvtColor(img,cv2.COLOR_BGR2Lab)\n",
    "    \n",
    "    #Calculem tamany del block\n",
    "    height, width = img.shape[:2]\n",
    "    block_height = int(height/level)\n",
    "    block_width = int(width/level)\n",
    "    hist_img = np.empty([0,0])\n",
    "    colorx = ('x','y','z')\n",
    "    canal = 0\n",
    "    for i,col in enumerate(colorx): #Bucle per recorrer els canals de cada imatge  \n",
    "        for r in range(level): #Bucle per recorrer els blocs d'una mateixa row\n",
    "            for c in range(level): #Bucle per recorrer els blocs d'una mateixa column\n",
    "                \n",
    "                block = img[r*block_height:r*block_height+block_height, c*block_width:c*block_width+block_width] \n",
    "                hist = cv2.calcHist([block],[i],None,[numberBins],[0,256]) #Calculem histogrames per blocs\n",
    "                hist_t = hist.transpose()      \n",
    "                #print(hist_t.shape)\n",
    "                #print(hist_img.shape)\n",
    "                if (i==0) and (r==0) and (c==0):\n",
    "                    hist_img = hist_t\n",
    "                else:\n",
    "                    hist_img = np.concatenate((hist_img, hist_t), axis = 1)\n",
    "\n",
    "    cv2.normalize(hist_img, hist_img, alpha=0, beta=1, norm_type=cv2.NORM_MINMAX)\n",
    "    return hist_img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def LbpDescriptor(img, level, numberBins, radius, n_poinys):\n",
    "    #Calculem tamany del block\n",
    "    height, width = img.shape[:2]\n",
    "    block_height = int(height/level)\n",
    "    block_width = int(width/level)\n",
    "    hist_img = np.empty([0,0])\n",
    "    colorx = ('x','y','z')\n",
    "    canal = 0\n",
    "    \n",
    "    for r in range(level): #Bucle per recorrer els blocs d'una mateixa row\n",
    "        for c in range(level): #Bucle per recorrer els blocs d'una mateixa column\n",
    "            gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
    "            block = gray[r*block_height:r*block_height+block_height, c*block_width:c*block_width+block_width]\n",
    "            lbp = local_binary_pattern(block, n_points, radius, 'uniform')\n",
    "            lbp = lbp.astype(np.uint8)\n",
    "            hist = cv2.calcHist([lbp],[0],None,[numberBins],[0,256]) #Calculem histogrames per blocs\n",
    "            hist_t = hist.transpose() \n",
    "            if (r==0) and (c==0):\n",
    "                hist_img = hist_t\n",
    "            else:\n",
    "                hist_img = np.concatenate((hist_img, hist_t), axis = 1)\n",
    "    cv2.normalize(hist_img, hist_img, alpha=0, beta=1, norm_type=cv2.NORM_MINMAX)\n",
    "    return hist_img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def HogDescriptor(img):\n",
    "    fd, hog_image = hog(img, orientations=8, pixels_per_cell=(16, 16),\n",
    "                        cells_per_block=(1, 1), visualize=True, multichannel=True)\n",
    "    #print(\"1 - hog_image.shape\", hog_image.shape)\n",
    "    #hog_image = hog_image.astype(np.uint8)\n",
    "    #print(\"2 - hog_image.shape\", hog_image.shape)\n",
    "\n",
    "    return hog_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def DctDescriptor(img):\n",
    "    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
    "    I_dct = dct(dct(block.T).T)\n",
    "    I_zz = zigzag(I_dct)\n",
    "    I_dct = I_zz[0:nCoef]\n",
    "    return I_dct"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test query system using QSD1-W2 using only texture descriptors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. Define Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DEFINING GLOBAL VARIABLES\n",
    "Level = 1\n",
    "descriptor = 'hog'\n",
    "radius = 5\n",
    "n_points = 8 * radius\n",
    "\n",
    "if descriptor == 'hog':\n",
    "    numberBins = 500*500\n",
    "    nCoef = numberBins*Level**2\n",
    "    \n",
    "if descriptor == 'LBP':\n",
    "    numberBins = n_points + 1\n",
    "    nCoef = numberBins*Level**2\n",
    "\n",
    "if descriptor == 'DCT2D':\n",
    "    numberBins = 32\n",
    "    nCoef = numberBins*Level**2\n",
    "    \n",
    "if descriptor == 'color':\n",
    "    numberBins = 256\n",
    "    nCoef = numberBins*3*Level**2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. Compute BBDD descriptors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "folder = './bbdd/'\n",
    "print(nCoef)\n",
    "histogram_bbdd_matrix = np.empty([0, nCoef]) #Creem una matriu buida\n",
    "print(histogram_bbdd_matrix.shape)\n",
    "\n",
    "for filename in sorted(listdir(folder)):\n",
    "    if(filename != '.DS_Store' and ('jpg' in filename)):\n",
    "        print (\"\\r Processing image...   {}\".format(folder+filename), end=\"\")\n",
    "        img = cv2.imread(folder + filename)\n",
    "        if descriptor == 'LBP':\n",
    "            hist_img = LbpDescriptor(img, Level, numberBins, radius, n_points)\n",
    "        if descriptor == 'hog':\n",
    "            resize_size = (500,500)\n",
    "            image_shape = img.shape[:2]\n",
    "            image = cv2.resize(img,resize_size)\n",
    "            hist_img = HogDescriptor(image)\n",
    "            hist_img = np.array(hist_img).flatten()\n",
    "        if descriptor == 'DCT2D':\n",
    "            hist_img = DctDescriptor(img)\n",
    "        if descriptor == 'color':\n",
    "            hist_img = colorDescriptor(img,Level,numberBins)\n",
    "        print(hist_img.shape)\n",
    "        histogram_bbdd_matrix = np.vstack((histogram_bbdd_matrix,hist_img))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. Compute QSD1_W3 denoised"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder = './qsd1_w2/'\n",
    "histogram_query_matrix = np.empty([0, nCoef]) #Creem una matriu buida\n",
    "\n",
    "for filename in sorted(listdir(folder)):\n",
    "    if(filename != '.DS_Store' and ('jpg' in filename)):\n",
    "        print (\"\\r Processing image...   {}\".format(folder+filename), end=\"\")\n",
    "        img = cv2.imread(folder + filename)\n",
    "        if descriptor == 'LBP':\n",
    "            hist_img = LbpDescriptor(img, Level, numberBins, radius, n_points)\n",
    "        if descriptor == 'hog':\n",
    "            resize_size = (500,500)\n",
    "            image_shape = img.shape[:2]\n",
    "            image = cv2.resize(img,resize_size)\n",
    "            hist_img = HogDescriptor(image)\n",
    "            hist_img = np.array(hist_img).flatten()\n",
    "        if descriptor == 'DCT2D':\n",
    "            hist_img = DctDescriptor(img)\n",
    "        if descriptor == 'color':\n",
    "            hist_img = colorDescriptor(img,Level,numberBins)\n",
    "        print(hist_img.shape)\n",
    "        histogram_query_matrix = np.vstack((histogram_query_matrix,hist_img))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4. Evaluation using only texture descriptors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DEFINE EVALUATION FUNCTIONS\n",
    "\n",
    "def computeMatrixRetrieval(histogram_bbdd, histogram_query, K=10):\n",
    "    dst = np.zeros((30, 279))\n",
    "    matrix_retrieval = np.zeros((30,K))\n",
    "\n",
    "    for query_image_index in range(0,len(histogram_query)): \n",
    "        for bbdd_image_index in range(0, len(histogram_bbdd)):\n",
    "\n",
    "            dst[query_image_index,bbdd_image_index] = distance.braycurtis(histogram_query[query_image_index,:], histogram_bbdd[bbdd_image_index,:])\n",
    "\n",
    "        matrix_retrieval[query_image_index,:] = np.argsort(dst[query_image_index,:])[:K]\n",
    "\n",
    "    #Convertir idx a list of lists\n",
    "    matrix_retrieval = matrix_retrieval.astype(int)\n",
    "    matrix_retrieval_lst = matrix_retrieval.tolist()\n",
    "    return matrix_retrieval_lst\n",
    "\n",
    "\n",
    "def evaluationTask1(matrix_retrieval, K, query_folder):  \n",
    "    #LLEGIR GT\n",
    "    with open(query_folder + 'gt_corresps.pkl', 'rb') as fd:\n",
    "            ll = pickle.load(fd)\n",
    "            #print(ll)\n",
    "    gt = np.empty((0,0))\n",
    "\n",
    "    ll_prp = np.zeros((len(ll),1))\n",
    "\n",
    "    for i in range(0,len(ll)):\n",
    "        ll_prp[i] = ll[i][0]\n",
    "\n",
    "    ll_prp_lst = ll_prp.tolist()\n",
    "    #print(ll_prp_lst)\n",
    "\n",
    "    mapak = metrics.average_precision.mapk(ll_prp_lst, matrix_retrieval, k=K)    \n",
    "    score = mapak*100\n",
    "    #print(\"SCORE: \",score, \"%\")\n",
    "\n",
    "    #GUARDAR PICKLE RESULT\n",
    "    pickle_out = open(\"result_qsd2.pkl\", \"wb\")\n",
    "    pickle.dump(matrix_retrieval, pickle_out)\n",
    "    pickle_out.close()\n",
    "    \n",
    "    return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "K=3\n",
    "folder = './qsd1_w2/'\n",
    "matrix_retrieval = computeMatrixRetrieval(histogram_bbdd_matrix, histogram_query_matrix, K)\n",
    "print(matrix_retrieval)\n",
    "evaluationTask1(matrix_retrieval, K, folder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TASK4 - Combine descriptors\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## text + color"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DEFINING GLOBAL VARIABLES\n",
    "Level = 1\n",
    "numberBins= 256\n",
    "nCoef_color = numberBins*3*Level**2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. BBDD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import codecs\n",
    "folder = './bbdd/'\n",
    "histogram_bbdd_matrix_color = np.empty([0, nCoef_color]) #Creem una matriu buida\n",
    "print(histogram_bbdd_matrix_color.shape)\n",
    "textfolder = './bbdd_text/'\n",
    "bbdd_text = []\n",
    "\n",
    "for filename in sorted(listdir(folder)):\n",
    "    if(filename != '.DS_Store' and ('jpg' in filename)):\n",
    "        print (\"\\r Processing image...   {}\".format(folder+filename), end=\"\")\n",
    "        img = cv2.imread(folder + filename)\n",
    "        \n",
    "        #COLOR\n",
    "        hist_img_color = colorDescriptor(img,Level,numberBins)\n",
    "        histogram_bbdd_matrix_color = np.vstack((histogram_bbdd_matrix_color,hist_img_color))\n",
    "\n",
    "\n",
    "\n",
    "for filename in sorted(listdir(textfolder)):\n",
    "    if(filename != '.DS_Store' and ('txt' in filename)):\n",
    "        with codecs.open(textfolder + filename,'r',encoding='latin1') as f:\n",
    "            if  os.stat(textfolder + filename).st_size == 0:\n",
    "                line = ''\n",
    "                bbdd_text.append(line)\n",
    "                #print(filename + line)\n",
    "            else:\n",
    "                for line in f.readlines():\n",
    "                    lim1 = line.find(\"'\",1)\n",
    "                    lim2 = line.find(\"'\",2)\n",
    "                    bbdd_text.append(line[lim1+1:lim2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. QSD1_W3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### TEXT\n",
    "folder = './denoising_qsd1_w3/'\n",
    "histogram_query_matrix_color = np.empty([0, nCoef_color]) #Creem una matriu buida\n",
    "print(histogram_query_matrix_color.shape)\n",
    "query_text = []\n",
    "import pytesseract\n",
    "with open('qsd1_w3_text.pkl', 'rb') as fd:\n",
    "    ll = pickle.load(fd)\n",
    "    #print(ll)\n",
    "\n",
    "count = 0\n",
    "for filename in sorted(listdir(folder)):\n",
    "    if(filename != '.DS_Store' and ('jpg' in filename)):\n",
    "        print (\"\\r Processing image...   {}\".format(folder+filename), end=\"\")\n",
    "        img = cv2.imread(folder + filename)\n",
    "        \n",
    "        #COLOR\n",
    "        hist_img_color = colorDescriptor(img,Level,numberBins)\n",
    "        histogram_query_matrix_color = np.vstack((histogram_query_matrix_color,hist_img_color))\n",
    "\n",
    "        #TEXT\n",
    "        img = img[ll[count][1]:ll[count][3],ll[count][0]:ll[count][2]]\n",
    "        extractedInformation = pytesseract.image_to_string(img)\n",
    "        query_text.append(extractedInformation)\n",
    "        count = count + 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. Evaluation text + color"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def computeMatrixRetrieval_combText(histogram_bbdd, histogram_query, bbdd_text, query_text, K=10):\n",
    "    dst_color = np.zeros((30, 279))\n",
    "    dst_tex = np.zeros((30, 279))\n",
    "    matrix_retrieval = np.zeros((30,K))\n",
    "\n",
    "    for query_image_index_c in range(0,len(histogram_query)): \n",
    "        for bbdd_image_index_c in range(0, len(histogram_bbdd)):\n",
    "\n",
    "            dst_color[query_image_index_c,bbdd_image_index_c] = distance.braycurtis(histogram_query[query_image_index_c,:], histogram_bbdd[bbdd_image_index_c,:])\n",
    "\n",
    "    for query_text_index in range(0,len(query_text)):\n",
    "        for bbdd_text_index in range(0,len(bbdd_text)-1):\n",
    "            dst_tex[query_text_index,bbdd_text_index] = levenshtein(query_text[query_text_index], bbdd_text[bbdd_text_index])\n",
    "\n",
    "            \n",
    "    dst = ((5/6)*dst_tex) + ((1/6)*dst_color)\n",
    "    for query_image_index in range(0,len(dst)):\n",
    "        matrix_retrieval[query_image_index,:] = np.argsort(dst[query_image_index,:])[:K]\n",
    "\n",
    "    #Convertir idx a list of lists\n",
    "    matrix_retrieval = matrix_retrieval.astype(int)\n",
    "    matrix_retrieval_lst = matrix_retrieval.tolist()\n",
    "    return matrix_retrieval_lst\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "K=10\n",
    "folder = './qsd1_w3/'\n",
    "matrix_retrieval = computeMatrixRetrieval_combText(histogram_bbdd_matrix_color, histogram_query_matrix_color, bbdd_text,query_text, K)\n",
    "evaluationTask1(matrix_retrieval, K, folder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## text + texture"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. BBDD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder = './bbdd/'\n",
    "nCoef_texture = 500*500\n",
    "histogram_bbdd_matrix_texture = np.empty([0, nCoef_texture]) #Creem una matriu buida\n",
    "print(histogram_bbdd_matrix_texture.shape)\n",
    "\n",
    "\n",
    "for filename in sorted(listdir(folder)):\n",
    "    if(filename != '.DS_Store' and ('jpg' in filename)):\n",
    "        print (\"\\r Processing image...   {}\".format(folder+filename), end=\"\")\n",
    "        img = cv2.imread(folder + filename)\n",
    "\n",
    "        #TEXTURE\n",
    "        resize_size = (500,500)\n",
    "        image_shape = img.shape[:2]\n",
    "        image = cv2.resize(img,resize_size)\n",
    "        hist_img_texture = HogDescriptor(image)\n",
    "        hist_img_texture = np.array(hist_img_texture).flatten()\n",
    "        histogram_bbdd_matrix_texture = np.vstack((histogram_bbdd_matrix_texture,hist_img_texture))\n",
    "\n",
    "\n",
    "# Read txt from bbdd_text\n",
    "textfolder = './bbdd_text/'\n",
    "bbdd_text = []\n",
    "for filename in sorted(listdir(textfolder)):\n",
    "    if(filename != '.DS_Store' and ('txt' in filename)):\n",
    "        with codecs.open(textfolder + filename,'r',encoding='latin1') as f:\n",
    "            if  os.stat(textfolder + filename).st_size == 0:\n",
    "                line = ''\n",
    "                bbdd_text.append(line)\n",
    "                #print(filename + line)\n",
    "            else:\n",
    "                for line in f.readlines():\n",
    "                    lim1 = line.find(\"'\",1)\n",
    "                    lim2 = line.find(\"'\",2)\n",
    "                    bbdd_text.append(line[lim1+1:lim2])\n",
    "        \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. QSD1_W3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder = './denoising_qsd1_w3/'\n",
    "histogram_query_matrix_texture = np.empty([0, nCoef_texture]) #Creem una matriu buida\n",
    "print(histogram_query_matrix_texture.shape)\n",
    "\n",
    "query_text = []\n",
    "import pytesseract\n",
    "with open('qsd1_w3_text.pkl', 'rb') as fd:\n",
    "    ll = pickle.load(fd)\n",
    "    #print(ll)\n",
    "\n",
    "count = 0\n",
    "for filename in sorted(listdir(folder)):\n",
    "    if(filename != '.DS_Store' and ('jpg' in filename)):\n",
    "        print (\"\\r Processing image...   {}\".format(folder+filename), end=\"\")\n",
    "        img = cv2.imread(folder + filename)\n",
    "\n",
    "        #TEXTURE\n",
    "        resize_size = (500,500)\n",
    "        image_shape = img.shape[:2]\n",
    "        image = cv2.resize(img,resize_size)\n",
    "        hist_img_texture = HogDescriptor(image)\n",
    "        hist_img_texture = np.array(hist_img_texture).flatten()\n",
    "        histogram_query_matrix_texture = np.vstack((histogram_query_matrix_texture,hist_img_texture))\n",
    "        \n",
    "        \n",
    "        #TEXT\n",
    "        img = img[ll[count][1]:ll[count][3],ll[count][0]:ll[count][2]]\n",
    "        extractedInformation = pytesseract.image_to_string(img)\n",
    "        query_text.append(extractedInformation)\n",
    "        count = count + 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. Evaluation text + texture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def computeMatrixRetrieval_combText(histogram_bbdd, histogram_query, bbdd_text, query_text, K=10):\n",
    "    dst_color = np.zeros((30, 279))\n",
    "    dst_tex = np.zeros((30, 279))\n",
    "    matrix_retrieval = np.zeros((30,K))\n",
    "\n",
    "    for query_image_index_c in range(0,len(histogram_query)): \n",
    "        for bbdd_image_index_c in range(0, len(histogram_bbdd)):\n",
    "\n",
    "            dst_color[query_image_index_c,bbdd_image_index_c] = distance.braycurtis(histogram_query[query_image_index_c,:], histogram_bbdd[bbdd_image_index_c,:])\n",
    "\n",
    "    for query_text_index in range(0,len(query_text)):\n",
    "        for bbdd_text_index in range(0,len(bbdd_text)-1):\n",
    "            dst_tex[query_text_index,bbdd_text_index] = levenshtein(query_text[query_text_index], bbdd_text[bbdd_text_index])\n",
    "\n",
    "            \n",
    "    dst = ((3/4)*dst_tex) + ((1/3)*dst_color)\n",
    "    for query_image_index in range(0,len(dst)):\n",
    "        matrix_retrieval[query_image_index,:] = np.argsort(dst[query_image_index,:])[:K]\n",
    "\n",
    "    #Convertir idx a list of lists\n",
    "    matrix_retrieval = matrix_retrieval.astype(int)\n",
    "    matrix_retrieval_lst = matrix_retrieval.tolist()\n",
    "    return matrix_retrieval_lst\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "K=3\n",
    "folder = './qsd1_w3/'\n",
    "matrix_retrieval = computeMatrixRetrieval_combText(histogram_bbdd_matrix_texture, histogram_query_matrix_texture, bbdd_text,query_text, K)\n",
    "evaluationTask1(matrix_retrieval, K, folder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## texture + color"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DEFINING GLOBAL VARIABLES\n",
    "\n",
    "nCoef_texture = 500*500\n",
    "numberBins_color = 256\n",
    "nCoef_color = numberBins_color*3*Level**2\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. BBDD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder = './bbdd/'\n",
    "histogram_bbdd_matrix_color = np.empty([0, nCoef_color]) #Creem una matriu buida\n",
    "histogram_bbdd_matrix_texture = np.empty([0, nCoef_texture]) #Creem una matriu buida\n",
    "print(histogram_bbdd_matrix_color.shape)\n",
    "print(histogram_bbdd_matrix_texture.shape)\n",
    "\n",
    "\n",
    "for filename in sorted(listdir(folder)):\n",
    "    if(filename != '.DS_Store' and ('jpg' in filename)):\n",
    "        print (\"\\r Processing image...   {}\".format(folder+filename), end=\"\")\n",
    "        img = cv2.imread(folder + filename)\n",
    "        \n",
    "        #COLOR\n",
    "        hist_img_color = colorDescriptor(img,Level,numberBins_color)\n",
    "        histogram_bbdd_matrix_color = np.vstack((histogram_bbdd_matrix_color,hist_img_color))\n",
    "\n",
    "        #TEXTURE\n",
    "        resize_size = (500,500)\n",
    "        image_shape = img.shape[:2]\n",
    "        image = cv2.resize(img,resize_size)\n",
    "        hist_img_texture = HogDescriptor(image)\n",
    "        hist_img_texture = np.array(hist_img_texture).flatten()\n",
    "        histogram_bbdd_matrix_texture = np.vstack((histogram_bbdd_matrix_texture,hist_img_texture))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. QSD1_W3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder = './denoising_qsd1_w3/'\n",
    "histogram_query_matrix_color = np.empty([0, nCoef_color]) #Creem una matriu buida\n",
    "histogram_query_matrix_texture = np.empty([0, nCoef_texture]) #Creem una matriu buida\n",
    "print(histogram_query_matrix_color.shape)\n",
    "print(histogram_query_matrix_texture.shape)\n",
    "\n",
    "\n",
    "for filename in sorted(listdir(folder)):\n",
    "    if(filename != '.DS_Store' and ('jpg' in filename)):\n",
    "        print (\"\\r Processing image...   {}\".format(folder+filename), end=\"\")\n",
    "        img = cv2.imread(folder + filename)\n",
    "        \n",
    "        #COLOR\n",
    "        hist_img_color = colorDescriptor(img,Level,numberBins_color)\n",
    "        histogram_query_matrix_color = np.vstack((histogram_query_matrix_color,hist_img_color))\n",
    "\n",
    "        #TEXTURE\n",
    "        resize_size = (500,500)\n",
    "        image_shape = img.shape[:2]\n",
    "        image = cv2.resize(img,resize_size)\n",
    "        hist_img_texture = HogDescriptor(image)\n",
    "        hist_img_texture = np.array(hist_img_texture).flatten()\n",
    "        histogram_query_matrix_texture = np.vstack((histogram_query_matrix_texture,hist_img_texture))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. Evaluation texture + color"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def computeMatrixRetrieval_colorTexture(histogram_bbdd_color, histogram_query_color, histogram_bbdd_tex, histogram_query_tex , K=10):\n",
    "    dst_color = np.zeros((30, 279))\n",
    "    dst_tex = np.zeros((30, 279))\n",
    "    matrix_retrieval = np.zeros((30,K))\n",
    "\n",
    "    for query_image_index_c in range(0,len(histogram_query_color)): \n",
    "        for bbdd_image_index_c in range(0, len(histogram_bbdd_color)):\n",
    "\n",
    "            dst_color[query_image_index_c,bbdd_image_index_c] = distance.braycurtis(histogram_query_color[query_image_index_c,:], histogram_bbdd_color[bbdd_image_index_c,:])\n",
    "\n",
    "    for query_image_index_t in range(0,len(histogram_query_tex)): \n",
    "        for bbdd_image_index_t in range(0, len(histogram_bbdd_tex)):\n",
    "            \n",
    "            dst_tex[query_image_index_t,bbdd_image_index_t] = distance.braycurtis(histogram_query_tex[query_image_index_t,:], histogram_bbdd_tex[bbdd_image_index_t,:])\n",
    "\n",
    "            \n",
    "    dst = ((5/6)*dst_tex) + ((1/6)*dst_color)\n",
    "    for query_image_index in range(0,len(dst)):\n",
    "        matrix_retrieval[query_image_index,:] = np.argsort(dst[query_image_index,:])[:K]\n",
    "\n",
    "    #Convertir idx a list of lists\n",
    "    matrix_retrieval = matrix_retrieval.astype(int)\n",
    "    matrix_retrieval_lst = matrix_retrieval.tolist()\n",
    "    return matrix_retrieval_lst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "K=3\n",
    "folder = './qsd1_w3/'\n",
    "matrix_retrieval = computeMatrixRetrieval_colorTexture(histogram_bbdd_matrix_color, histogram_query_matrix_color, histogram_bbdd_matrix_texture,histogram_query_matrix_texture, K)\n",
    "evaluationTask1(matrix_retrieval, K, folder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## text + color + texture"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. BBDD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DEFINING GLOBAL VARIABLES\n",
    "Level = 1\n",
    "nCoef_texture = 500*500\n",
    "numberBins_color = 256\n",
    "nCoef_color = numberBins_color*3*Level**2\n",
    "\n",
    "folder = './bbdd/'\n",
    "histogram_bbdd_matrix_color = np.empty([0, nCoef_color]) #Creem una matriu buida\n",
    "histogram_bbdd_matrix_texture = np.empty([0, nCoef_texture]) #Creem una matriu buida\n",
    "print(histogram_bbdd_matrix_color.shape)\n",
    "print(histogram_bbdd_matrix_texture.shape)\n",
    "\n",
    "\n",
    "for filename in sorted(listdir(folder)):\n",
    "    if(filename != '.DS_Store' and ('jpg' in filename)):\n",
    "        print (\"\\r Processing image...   {}\".format(folder+filename), end=\"\")\n",
    "        img = cv2.imread(folder + filename)\n",
    "        \n",
    "        #COLOR\n",
    "        hist_img_color = colorDescriptor(img,Level,numberBins_color)\n",
    "        histogram_bbdd_matrix_color = np.vstack((histogram_bbdd_matrix_color,hist_img_color))\n",
    "\n",
    "        #TEXTURE\n",
    "        resize_size = (500,500)\n",
    "        image_shape = img.shape[:2]\n",
    "        image = cv2.resize(img,resize_size)\n",
    "        hist_img_texture = HogDescriptor(image)\n",
    "        hist_img_texture = np.array(hist_img_texture).flatten()\n",
    "        histogram_bbdd_matrix_texture = np.vstack((histogram_bbdd_matrix_texture,hist_img_texture))\n",
    "        \n",
    "\n",
    "\n",
    "# Read txt from bbdd_text\n",
    "textfolder = './bbdd_text/'\n",
    "bbdd_text = []\n",
    "for filename in sorted(listdir(textfolder)):\n",
    "    if(filename != '.DS_Store' and ('txt' in filename)):\n",
    "        with codecs.open(textfolder + filename,'r',encoding='latin1') as f:\n",
    "            if  os.stat(textfolder + filename).st_size == 0:\n",
    "                line = ''\n",
    "                bbdd_text.append(line)\n",
    "                #print(filename + line)\n",
    "            else:\n",
    "                for line in f.readlines():\n",
    "                    lim1 = line.find(\"'\",1)\n",
    "                    lim2 = line.find(\"'\",2)\n",
    "                    bbdd_text.append(line[lim1+1:lim2])\n",
    "        \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. QSD1_W3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder = './qst1_w3_denoise/'\n",
    "histogram_query_matrix_color = np.empty([0, nCoef_color]) #Creem una matriu buida\n",
    "histogram_query_matrix_texture = np.empty([0, nCoef_texture]) #Creem una matriu buida\n",
    "print(histogram_query_matrix_color.shape)\n",
    "print(histogram_query_matrix_texture.shape)\n",
    "query_text = []\n",
    "import pytesseract\n",
    "with open('qst1_w3_text.pkl', 'rb') as fd:\n",
    "    ll = pickle.load(fd)\n",
    "    #print(ll)\n",
    "\n",
    "count = 0\n",
    "\n",
    "for filename in sorted(listdir(folder)):\n",
    "    if(filename != '.DS_Store' and ('jpg' in filename)):\n",
    "        print (\"\\r Processing image...   {}\".format(folder+filename), end=\"\")\n",
    "        img = cv2.imread(folder + filename)\n",
    "        \n",
    "        #COLOR\n",
    "        hist_img_color = colorDescriptor(img,Level,numberBins_color)\n",
    "        histogram_query_matrix_color = np.vstack((histogram_query_matrix_color,hist_img_color))\n",
    "\n",
    "        #TEXTURE\n",
    "        resize_size = (500,500)\n",
    "        image_shape = img.shape[:2]\n",
    "        image = cv2.resize(img,resize_size)\n",
    "        hist_img_texture = HogDescriptor(image)\n",
    "        hist_img_texture = np.array(hist_img_texture).flatten()\n",
    "        histogram_query_matrix_texture = np.vstack((histogram_query_matrix_texture,hist_img_texture))\n",
    "        \n",
    "        #TEXT\n",
    "        img = img[ll[count][0][1]:ll[count][0][3],ll[count][0][0]:ll[count][0][2]]\n",
    "        extractedInformation = pytesseract.image_to_string(img)\n",
    "        query_text.append(extractedInformation)\n",
    "        count = count + 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. Evaluation text + color"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def computeMatrixRetrieval_colorTexture(histogram_bbdd_color, histogram_query_color, histogram_bbdd_tex, histogram_query_tex, bbdd_text, query_text, K=10):\n",
    "    dst_color = np.zeros((50, 279))\n",
    "    dst_tex = np.zeros((50, 279))\n",
    "    dst_text = np.zeros((50, 279))\n",
    "    matrix_retrieval = np.zeros((50,K))\n",
    "\n",
    "    for query_image_index_c in range(0,len(histogram_query_color)): \n",
    "        for bbdd_image_index_c in range(0, len(histogram_bbdd_color)):\n",
    "\n",
    "            dst_color[query_image_index_c,bbdd_image_index_c] = distance.braycurtis(histogram_query_color[query_image_index_c,:], histogram_bbdd_color[bbdd_image_index_c,:])\n",
    "\n",
    "    for query_image_index_t in range(0,len(histogram_query_tex)): \n",
    "        for bbdd_image_index_t in range(0, len(histogram_bbdd_tex)):\n",
    "            \n",
    "            dst_tex[query_image_index_t,bbdd_image_index_t] = distance.braycurtis(histogram_query_tex[query_image_index_t,:], histogram_bbdd_tex[bbdd_image_index_t,:])\n",
    "    \n",
    "    for query_text_index in range(0,len(query_text)):\n",
    "        for bbdd_text_index in range(0,len(bbdd_text)-1):\n",
    "            dst_text[query_text_index,bbdd_text_index] = levenshtein(query_text[query_text_index], bbdd_text[bbdd_text_index])       \n",
    "            \n",
    "    dst = ((1/3)*dst_tex) + ((1/3)*dst_color) + ((1/3)*dst_text)\n",
    "    for query_image_index in range(0,len(dst)):\n",
    "        matrix_retrieval[query_image_index,:] = np.argsort(dst[query_image_index,:])[:K]\n",
    "\n",
    "    #Convertir idx a list of lists\n",
    "    matrix_retrieval = matrix_retrieval.astype(int)\n",
    "    matrix_retrieval_lst = matrix_retrieval.tolist()\n",
    "    return matrix_retrieval_lst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "matrix_retrieval = computeMatrixRetrieval_colorTexture(histogram_bbdd_matrix_color, histogram_query_matrix_color, histogram_bbdd_matrix_texture,histogram_query_matrix_texture, bbdd_text, query_text, K)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create pikel\n",
    "matrix_retrieval_aux = []\n",
    "for x in matrix_retrieval:\n",
    "    matrix_retrieval_aux.append([x]) \n",
    "\n",
    "pickle_out = open('result_QST1_w3.pkl','wb')\n",
    "pickle.dump(matrix_retrieval_aux,pickle_out)\n",
    "pickle_out.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "matrix_retrieval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "K=10\n",
    "folder = './qsd1_w3/'\n",
    "matrix_retrieval = computeMatrixRetrieval_colorTexture(histogram_bbdd_matrix_color, histogram_query_matrix_color, histogram_bbdd_matrix_texture,histogram_query_matrix_texture, bbdd_text, query_text, K)\n",
    "evaluationTask1(matrix_retrieval, K, folder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TASK 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "query2folder = './qst2_w3/'\n",
    "# CREEM UN DIRECTORI ON GUARDEM LES IMATGES SENSE SOROLL\n",
    "path = 'qst2_w3_denoise'\n",
    "if not os.path.exists(path):\n",
    "    os.mkdir(path)\n",
    "\n",
    "\n",
    "for filename in sorted(listdir(query2folder)):\n",
    "    if(filename != '.DS_Store' and ('jpg' in filename)):\n",
    "        #print (\"\\r Processing image...   {}\".format(query2folder+filename), end=\"\")\n",
    "        query_img = cv2.imread(query2folder + filename)\n",
    "        gray = cv2.cvtColor(query_img, cv2.COLOR_BGR2GRAY)\n",
    "        noise = detectnoise(gray)\n",
    "        if noise:\n",
    "            denoise_img = cv2.bilateralFilter(query_img, 9, 120, 120)\n",
    "            denoise_img_2 = cv2.fastNlMeansDenoisingColored(denoise_img,None,15,15,10,21) \n",
    "            #plt.figure()\n",
    "            #plt.imshow(denoise_img)\n",
    "            #plt.figure()\n",
    "            #plt.imshow(denoise_img_2)\n",
    "        else:\n",
    "            denoise_img_2 = query_img\n",
    "        \n",
    "        cv2.imwrite(os.path.join(path,filename), denoise_img_2)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "query2folder = './qst2_w3/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def waterShed(query_img, color_space, sat_chan):\n",
    "    \"\"\"\n",
    "    Applies watershed to detect foreground and baskground\n",
    "\n",
    "    :param query_img: (ndarray) query image\n",
    "    :param sat: (ndarray) query image saturation\n",
    "    :param part: (int) one paint(0), two paints left part(1),  two paints right part(2)\n",
    "    :return: mask: predicted mask\n",
    "    \"\"\"\n",
    "    \n",
    "    if sat_chan:\n",
    "        ret, thresh = cv2.threshold(color_space, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)\n",
    "        thresh = 255 - thresh\n",
    "    else:\n",
    "        thresh = k_means(color_space)\n",
    "\n",
    "    # noise removal\n",
    "    kernel = np.ones((3, 3), np.uint8)\n",
    "    morph = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel, iterations=1)\n",
    "    # fillin holes\n",
    "    kernel = np.ones((5, 5), np.uint8)\n",
    "    opening = cv2.morphologyEx(morph, cv2.MORPH_OPEN, kernel, iterations=2)\n",
    "\n",
    "    # sure background area\n",
    "    sure_bg = cv2.dilate(opening, kernel, iterations=3)  # Fals. El sure\n",
    "\n",
    "    # Finding sure foreground area\n",
    "    dist_transform = cv2.distanceTransform(opening, cv2.DIST_L2, 3)\n",
    "    ret, sure_fg = cv2.threshold(dist_transform, 0.13 * dist_transform.max(), 255, 0)  # 0.15 26.57%\n",
    "\n",
    "    # Finding unknown region\n",
    "    sure_fg = np.uint8(sure_fg)\n",
    "    unknown = cv2.subtract(sure_bg, sure_fg)\n",
    "\n",
    "    # Marker labelling\n",
    "    ret, markers = cv2.connectedComponents(sure_fg)\n",
    "\n",
    "    markers[markers != 0] = 200\n",
    "\n",
    "    border_top = int(sat.shape[0] * .03)\n",
    "    borde_lat = int(sat.shape[1] * .02)\n",
    "\n",
    "    # define known sure background\n",
    "    markers[0:border_top, :] = 100\n",
    "    sure_fg[0:border_top, :] = 100\n",
    "\n",
    "    markers[len(markers) - border_top:len(markers) - 1, :] = 100\n",
    "    sure_fg[len(markers) - border_top:len(markers) - 1, :] = 100\n",
    "\n",
    "    markers[:, 0:borde_lat] = 100\n",
    "    sure_fg[:, 0:borde_lat] = 100\n",
    "    markers[:, len(markers[0, :]) - borde_lat:len(markers[0, :]) - 1] = 100\n",
    "    sure_fg[:, len(markers[0, :]) - borde_lat:len(markers[0, :]) - 1] = 100\n",
    "\n",
    "    # Add one to all labels so that sure background is not 0, but 1\n",
    "    markers = markers + 1\n",
    "\n",
    "    # Now, mark the region of unknown with zero\n",
    "    markers[sure_fg == 0] = 0\n",
    "\n",
    "    markersres = cv2.watershed(query_img, markers)\n",
    "\n",
    "    markersres[markersres == 201] = 255\n",
    "    markersres[markersres == 101] = 0\n",
    "    markersres[markersres == -1] = 255\n",
    "\n",
    "    # Make the mask binary\n",
    "    markersres = markersres / np.max(markersres)\n",
    "            \n",
    "    # Remove imperfections with morphological filers (noise)\n",
    "    markersres = cv2.morphologyEx(markersres, cv2.MORPH_OPEN, np.ones((3, 3), np.uint8), iterations=1)\n",
    "\n",
    "    # Set predicted mask\n",
    "    mask_pred = markersres.astype('uint8')\n",
    "\n",
    "    # Get connected components\n",
    "    ret, labels = cv2.connectedComponents(mask_pred)\n",
    "\n",
    "    # Initialize mask for paintings\n",
    "    mask = np.zeros((mask_pred.shape[0], mask_pred.shape[1], 2), dtype=\"uint8\")\n",
    "\n",
    "    # If exist more than two connected components (background and one painting)\n",
    "    if ret > 2:\n",
    "\n",
    "        # Compute area of each connected component\n",
    "        area = []\n",
    "        for i, lab in enumerate(np.unique(labels)):\n",
    "            area.append(mask_pred[labels == lab].size)\n",
    "\n",
    "        # Sort indexes from length of areas\n",
    "        idx = sorted(range(len(area)), key=lambda k: area[k])\n",
    "\n",
    "        # Check if more than three connected components (background and two paintings)\n",
    "        if ret > 3:\n",
    "            for i, j in enumerate(idx):\n",
    "                # Remove smallest areas\n",
    "                if (len(area)-i) > 3:\n",
    "                    mask_pred[labels == j] = 0\n",
    "\n",
    "        # Remove indexes labels where its pixels are now zero\n",
    "        idx = [n for _, n in enumerate(idx) if np.sum(mask_pred[labels == n]) != 0]\n",
    "\n",
    "        # Positions of non-zero elements of each painting\n",
    "        obj1 = np.where(labels == idx[0])\n",
    "        obj2 = np.where(labels == idx[1])\n",
    "\n",
    "        # Determine if there exist two paintings\n",
    "        if (area[idx[0]] > .28*area[idx[1]]):\n",
    "\n",
    "            # Check that one connected component is not inside the other\n",
    "            if (((np.min(obj1[1][:]) - np.max(obj2[1][:])) < 0) and ((np.max(obj1[1][:]) - np.min(obj2[1][:])) > 0)) \\\n",
    "                    or (((np.min(obj2[1][:]) - np.max(obj1[1][:])) < 0) and\n",
    "                        ((np.max(obj2[1][:]) - np.min(obj1[1][:])) > 0)):\n",
    "                mask = mask_pred\n",
    "            else:\n",
    "                # Set one mask to each painting\n",
    "                mask[labels == idx[0], 0] = 1\n",
    "                mask[labels == idx[1], 1] = 1\n",
    "\n",
    "        else:\n",
    "            # Set mask for the painting\n",
    "            mask = mask_pred\n",
    "\n",
    "    else:\n",
    "        # Set mask for the painting\n",
    "        mask = mask_pred\n",
    "\n",
    "    ## POLYGON - Generate a polygon from mask corners ##\n",
    "\n",
    "    mask_f = np.zeros((mask_pred.shape[0], mask_pred.shape[1], 2), dtype=\"uint8\")\n",
    "\n",
    "    # For two paintings\n",
    "    if len(mask.shape) > 2:\n",
    "        # Get Corners\n",
    "        corners0 = detect_corners(mask[:, :, 0])\n",
    "\n",
    "        # Draw polygon\n",
    "        poly0 = Image.new('RGB', (mask_pred.shape[1], mask_pred.shape[0]), (0))\n",
    "        pdraw0 = ImageDraw.Draw(poly0)\n",
    "        pdraw0.polygon(xy=corners0, fill=(1), outline=None)\n",
    "        mask_pred_f = np.array(poly0)[:, :, 0]\n",
    "        mask_f[:, :, 0] = mask_pred_f\n",
    "\n",
    "        # Get Corners\n",
    "        corners1 = detect_corners(mask[:, :, 1])\n",
    "\n",
    "        # Draw polygon\n",
    "        poly1 = Image.new('RGB', (mask_pred.shape[1], mask_pred.shape[0]), (0))\n",
    "        pdraw1 = ImageDraw.Draw(poly1)\n",
    "        pdraw1.polygon(xy=corners1, fill=(1), outline=None)\n",
    "        mask_f[:, :, 1] = np.array(poly1)[:, :, 0]\n",
    "        mask_pred_f = mask_pred_f + mask_f[:, :, 1]\n",
    "\n",
    "    # Just one painting\n",
    "    else:\n",
    "        # Get Corners\n",
    "        corners = detect_corners(mask)\n",
    "\n",
    "        # Draw polygon\n",
    "        poly = Image.new('RGB', (mask_pred.shape[1], mask_pred.shape[0]), (0))\n",
    "        pdraw = ImageDraw.Draw(poly)\n",
    "        pdraw.polygon(xy=corners, fill=(1), outline=None)\n",
    "        mask_pred_f = np.array(poly)[:, :, 0]\n",
    "        mask_f = mask_pred_f\n",
    "\n",
    "    return mask_pred_f, mask_f\n",
    "\n",
    "\n",
    "def k_means(color_space):\n",
    "    \n",
    "    Z = color_space.reshape((-1))\n",
    "    # convert to np.float32\n",
    "    Z = np.float32(Z)\n",
    "\n",
    "    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 10, 1.0)\n",
    "    ret, label, center = cv2.kmeans(Z, 5, None, criteria, 10, cv2.KMEANS_RANDOM_CENTERS)\n",
    "\n",
    "    center = np.uint8(center)\n",
    "    res = center[label.flatten()]\n",
    "    res2 = res.reshape((gray.shape))\n",
    "\n",
    "    mask = np.zeros(color_space.shape[:2],np.uint8)\n",
    "\n",
    "    for l in range(len(center)):\n",
    "        lab_pos = np.where(res2==center[l])\n",
    "        if any(lab_pos[0]==0) or any(lab_pos[1]==0) or any(lab_pos[0]==color_space.shape[0]-1) or any(lab_pos[1]==color_space.shape[1]-1):\n",
    "            continue\n",
    "        else:\n",
    "            mask[lab_pos[0],lab_pos[1]] = 255\n",
    "    \n",
    "    if np.sum(mask) == 0:\n",
    "        pos = np.where(res2==res2[int(color_space.shape[0]/2),int(color_space.shape[1]/2)])\n",
    "        mask[pos[0],pos[1]] = 255\n",
    "        \n",
    "    return mask\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. Remove background"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "query2folder = './qsd2_w3_denoise/'\n",
    "\n",
    "for filename in sorted(listdir(query2folder)):\n",
    "    if (filename != '.DS_Store' and ('jpg' in filename)):\n",
    "        print (\"\\r Processing image...   {}\".format(query2folder+filename), end=\"\")\n",
    "        query_img = cv2.imread(query2folder + filename)\n",
    "        hsv = cv2.cvtColor(query_img, cv2.COLOR_BGR2HSV)\n",
    "        gray = cv2.cvtColor(query_img, cv2.COLOR_BGR2GRAY)\n",
    "        \n",
    "        # Get saturation and grayscale from query image\n",
    "        sat = cv2.cvtColor(query_img, cv2.COLOR_BGR2HSV)[:,:,1]\n",
    "        gray = cv2.cvtColor(query_img, cv2.COLOR_BGR2GRAY)\n",
    "        \n",
    "        if ndimage.variance(sat)<ndimage.variance(gray):\n",
    "            color_space = gray\n",
    "            sat_channel=False\n",
    "            \n",
    "        else:\n",
    "            color_space = sat\n",
    "            sat_channel=True\n",
    "            \n",
    "        # Predict mask\n",
    "        mask_pred,mask = waterShed(query_img, color_space, sat_channel)\n",
    "        \n",
    "        # Save masks\n",
    "        path = './masks_qd'\n",
    "        if not os.path.exists(path):\n",
    "            os.mkdir(path)\n",
    "\n",
    "        cv2.imwrite('masks_qd/'+filename[:-4]+'.png',mask_pred*255)\n",
    "        if len(mask.shape) > 2:\n",
    "            cv2.imwrite('masks_qd/mask0_'+filename[:-4]+'.png',mask[:,:,0]*255)\n",
    "            cv2.imwrite('masks_qd/mask1_'+filename[:-4]+'.png',mask[:,:,1]*255)\n",
    "        else:\n",
    "            cv2.imwrite('masks_qd/mask0_'+filename[:-4]+'.png',mask[:,:]*255)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def crop(query_im, mask_crop):\n",
    "    # Coordinates of non-black pixels.\n",
    "    coords = np.argwhere(mask_crop)\n",
    "\n",
    "    # Bounding box of non-black pixels.\n",
    "    x0, y0 = coords.min(axis=0)\n",
    "    x1, y1 = coords.max(axis=0) + 1   # slices are exclusive at the top\n",
    "\n",
    "    # Get the contents of the bounding box.\n",
    "    cropped = query_im[x0:x1, y0:y1]\n",
    "    mask_cropped = mask_crop[x0:x1, y0:y1]\n",
    "    \n",
    "    return cropped, mask_cropped, x0, y0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. Detect text bounding boxes for each painting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "query2masks = './masks_qd/'\n",
    "\n",
    "bboxes = []\n",
    "\n",
    "for filename in sorted(listdir(query2masks)):\n",
    "    if (filename != '.DS_Store' and ('png' in filename) and ('mask' in filename)):\n",
    "        print (\"\\r Processing image...   {}\".format(query2masks+filename), end=\"\")\n",
    "        \n",
    "        mask = cv2.imread(query2masks + filename)\n",
    "        mask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)\n",
    "\n",
    "        img_num = filename[6:11]\n",
    "\n",
    "        query_img = cv2.imread(query2folder + img_num + '.jpg')\n",
    "\n",
    "        cropped, mask_cropped, X0, Y0 = crop(query_img,mask)\n",
    "\n",
    "        bbox = find_box(cropped)\n",
    "\n",
    "        if len(bboxes)>int(img_num):\n",
    "            bboxes[int(img_num)].append(bbox)\n",
    "        else:\n",
    "            bboxes.append([bbox])\n",
    "\n",
    "pickle_out = open(\"qsd2_w3_text.pkl\", \"wb\")\n",
    "pickle.dump(bboxes, pickle_out)\n",
    "pickle_out.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. Predict correspondences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "histogram_query_matrix_color = np.empty([0, nCoef_color]) #Creem una matriu buida\n",
    "histogram_query_matrix_texture = np.empty([0, nCoef_texture]) #Creem una matriu buida\n",
    "query_text = []\n",
    "\n",
    "with open('qsd2_w3_text.pkl', 'rb') as fd:\n",
    "    ll = pickle.load(fd)\n",
    "\n",
    "for filename in sorted(listdir(query2masks)):\n",
    "    if (filename != '.DS_Store' and ('png' in filename) and ('mask' in filename)):\n",
    "        print (\"\\r Processing image...   {}\".format(query2masks+filename), end=\"\")\n",
    "        \n",
    "        mask = cv2.imread(query2masks + filename)\n",
    "        mask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)\n",
    "        \n",
    "        img_num = filename[6:11]\n",
    "        num_paint = int(filename[4])\n",
    "\n",
    "        query_img = cv2.imread(query2folder + img_num + '.jpg')\n",
    "\n",
    "        cropped, mask_cropped, X0, Y0 = crop(query_img,mask)\n",
    "        \n",
    "        #COLOR\n",
    "        hist_img_color = colorDescriptor(cropped,Level,numberBins_color)\n",
    "        histogram_query_matrix_color = np.vstack((histogram_query_matrix_color,hist_img_color))\n",
    "\n",
    "        #TEXTURE\n",
    "        resize_size = (500,500)\n",
    "        image_shape = cropped.shape[:2]\n",
    "        image = cv2.resize(cropped,resize_size)\n",
    "        hist_img_texture = HogDescriptor(image)\n",
    "        hist_img_texture = np.array(hist_img_texture).flatten()\n",
    "        histogram_query_matrix_texture = np.vstack((histogram_query_matrix_texture,hist_img_texture))\n",
    "        \n",
    "        #TEXT\n",
    "        img = cropped[ll[int(img_num)][num_paint][1]:ll[int(img_num)][num_paint][3],ll[int(img_num)][num_paint][0]:ll[int(img_num)][num_paint][2]]\n",
    "        if img.shape[0]<4:\n",
    "            query_text.append('')\n",
    "        else:\n",
    "            extractedInformation = pytesseract.image_to_string(img)\n",
    "            query_text.append(extractedInformation)\n",
    "        \n",
    "        with open(img_num+'.txt', 'a') as txtf:\n",
    "            if num_paint==0:\n",
    "                txtf.write(extractedInformation)\n",
    "            else:\n",
    "                txtf.write('\\n'+extractedInformation)\n",
    "            txtf.close()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4. Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "K=10\n",
    "matrix_retrieval = computeMatrixRetrieval_colorTexture(histogram_bbdd_matrix_color, histogram_query_matrix_color, histogram_bbdd_matrix_texture,histogram_query_matrix_texture, bbdd_text, query_text, K)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "final_result = []\n",
    "\n",
    "count = 0\n",
    "for l in ll:\n",
    "    for i in range(len(l)):\n",
    "        if i == 0:\n",
    "            final_result.append([matrix_retrieval[count]])\n",
    "        else:\n",
    "            final_result[len(final_result)-1].append(matrix_retrieval[count])\n",
    "        count += 1\n",
    "\n",
    "pickle_out = open('result_QSD2_w3.pkl','wb')\n",
    "pickle.dump(final_result,pickle_out)\n",
    "pickle_out.close()\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}