From 7778a786261a369eb39defdb84b3cbb34e440bf5 Mon Sep 17 00:00:00 2001 From: Mohammed Salman <31631107+salman-moh@users.noreply.github.com> Date: Wed, 19 Oct 2022 12:43:29 +0530 Subject: [PATCH 01/16] streamlit app using Table Transformer and OCR addition of OCR to download tables directly as csv files. --- Table Transformer/app.py | 504 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 504 insertions(+) create mode 100644 Table Transformer/app.py diff --git a/Table Transformer/app.py b/Table Transformer/app.py new file mode 100644 index 00000000..f8fc842b --- /dev/null +++ b/Table Transformer/app.py @@ -0,0 +1,504 @@ +import streamlit as st +from PIL import Image, ImageEnhance +import statistics +import os +import string +from collections import Counter +from itertools import tee, count +# import TDTSR +import pytesseract +from pytesseract import Output +import json +import pandas as pd +import matplotlib.pyplot as plt +import cv2 +import numpy as np +# from transformers import TrOCRProcessor, VisionEncoderDecoderModel +# from cv2 import dnn_superres +from transformers import DetrFeatureExtractor +from transformers import DetrForObjectDetection +import torch +import asyncio +# pytesseract.pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract.exe' + + +st.set_option('deprecation.showPyplotGlobalUse', False) +st.set_page_config(layout='wide') +st.title("Table Detection and Table Structure Recognition") +st.write("Implemented by MSFT team: https://github.com/microsoft/table-transformer") + + +def PIL_to_cv(pil_img): + return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) + +def cv_to_PIL(cv_img): + return Image.fromarray(cv2.cvtColor(cv_img, cv2.COLOR_BGR2RGB)) + + +async def pytess(cell_pil_img): + return ' '.join(pytesseract.image_to_data(cell_pil_img, output_type=Output.DICT, config='-c tessedit_char_blacklist=œ˜â€œï¬â™Ã©œ¢!|”?«“¥ --psm 6 preserve_interword_spaces')['text']).strip() + + +# def super_res(pil_img): + # ''' + # Useful for low-res docs + # ''' + # requires opencv-contrib-python installed without the opencv-python + # sr = dnn_superres.DnnSuperResImpl_create() + # image = PIL_to_cv(pil_img) + # model_path = "/data/Salman/TRD/code/table-transformer/transformers/LapSRN_x2.pb" + # model_name = 'lapsrn' + # model_scale = 2 + # sr.readModel(model_path) + # sr.setModel(model_name, model_scale) + # final_img = sr.upsample(image) + # final_img = cv_to_PIL(final_img) + + # return final_img + + +def sharpen_image(pil_img): + + img = PIL_to_cv(pil_img) + sharpen_kernel = np.array([[-1, -1, -1], + [-1, 9, -1], + [-1, -1, -1]]) + + sharpen = cv2.filter2D(img, -1, sharpen_kernel) + pil_img = cv_to_PIL(sharpen) + return pil_img + + +def uniquify(seq, suffs = count(1)): + """Make all the items unique by adding a suffix (1, 2, etc). + Credit: https://stackoverflow.com/questions/30650474/python-rename-duplicates-in-list-with-progressive-numbers-without-sorting-list + `seq` is mutable sequence of strings. + `suffs` is an optional alternative suffix iterable. + """ + not_unique = [k for k,v in Counter(seq).items() if v>1] + + suff_gens = dict(zip(not_unique, tee(suffs, len(not_unique)))) + for idx,s in enumerate(seq): + try: + suffix = str(next(suff_gens[s])) + except KeyError: + continue + else: + seq[idx] += suffix + + return seq + +def binarizeBlur_image(pil_img): + image = PIL_to_cv(pil_img) + thresh = cv2.threshold(image, 150, 255, cv2.THRESH_BINARY_INV)[1] + + result = cv2.GaussianBlur(thresh, (5,5), 0) + result = 255 - result + return cv_to_PIL(result) + + + +def td_postprocess(pil_img): + ''' + Removes gray background from tables + ''' + img = PIL_to_cv(pil_img) + + hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV) + mask = cv2.inRange(hsv, (0, 0, 100), (255, 5, 255)) # (0, 0, 100), (255, 5, 255) + nzmask = cv2.inRange(hsv, (0, 0, 5), (255, 255, 255)) # (0, 0, 5), (255, 255, 255)) + nzmask = cv2.erode(nzmask, np.ones((3,3))) # (3,3) + mask = mask & nzmask + + new_img = img.copy() + new_img[np.where(mask)] = 255 + + + return cv_to_PIL(new_img) + +# def super_res(pil_img): +# # requires opencv-contrib-python installed without the opencv-python +# sr = dnn_superres.DnnSuperResImpl_create() +# image = PIL_to_cv(pil_img) +# model_path = "./LapSRN_x8.pb" +# model_name = model_path.split('/')[1].split('_')[0].lower() +# model_scale = int(model_path.split('/')[1].split('_')[1].split('.')[0][1]) + +# sr.readModel(model_path) +# sr.setModel(model_name, model_scale) +# final_img = sr.upsample(image) +# final_img = cv_to_PIL(final_img) + +# return final_img + +def table_detector(image, THRESHOLD_PROBA): + ''' + Table detection using DEtect-object TRansformer pre-trained on 1 million tables + + ''' + + feature_extractor = DetrFeatureExtractor(do_resize=True, size=800, max_size=800) + encoding = feature_extractor(image, return_tensors="pt") + + model = DetrForObjectDetection.from_pretrained("SalML/DETR-table-detection") + + with torch.no_grad(): + outputs = model(**encoding) + + probas = outputs.logits.softmax(-1)[0, :, :-1] + keep = probas.max(-1).values > THRESHOLD_PROBA + + target_sizes = torch.tensor(image.size[::-1]).unsqueeze(0) + postprocessed_outputs = feature_extractor.post_process(outputs, target_sizes) + bboxes_scaled = postprocessed_outputs[0]['boxes'][keep] + + return (model, probas[keep], bboxes_scaled) + + +def table_struct_recog(image, THRESHOLD_PROBA): + ''' + Table structure recognition using DEtect-object TRansformer pre-trained on 1 million tables + ''' + + feature_extractor = DetrFeatureExtractor(do_resize=True, size=1000, max_size=1000) + encoding = feature_extractor(image, return_tensors="pt") + + model = DetrForObjectDetection.from_pretrained("SalML/DETR-table-structure-recognition") + with torch.no_grad(): + outputs = model(**encoding) + + probas = outputs.logits.softmax(-1)[0, :, :-1] + keep = probas.max(-1).values > THRESHOLD_PROBA + + target_sizes = torch.tensor(image.size[::-1]).unsqueeze(0) + postprocessed_outputs = feature_extractor.post_process(outputs, target_sizes) + bboxes_scaled = postprocessed_outputs[0]['boxes'][keep] + + return (model, probas[keep], bboxes_scaled) + + + + + +class TableExtractionPipeline(): + + colors = ["red", "blue", "green", "yellow", "orange", "violet"] + + # colors = ["red", "blue", "green", "red", "red", "red"] + + def add_padding(self, pil_img, top, right, bottom, left, color=(255,255,255)): + ''' + Image padding as part of TSR pre-processing to prevent missing table edges + ''' + width, height = pil_img.size + new_width = width + right + left + new_height = height + top + bottom + result = Image.new(pil_img.mode, (new_width, new_height), color) + result.paste(pil_img, (left, top)) + return result + + def plot_results_detection(self, c1, model, pil_img, prob, boxes, delta_xmin, delta_ymin, delta_xmax, delta_ymax): + ''' + crop_tables and plot_results_detection must have same co-ord shifts because 1 only plots the other one updates co-ordinates + ''' + # st.write('img_obj') + # st.write(pil_img) + plt.imshow(pil_img) + ax = plt.gca() + + for p, (xmin, ymin, xmax, ymax) in zip(prob, boxes.tolist()): + cl = p.argmax() + xmin, ymin, xmax, ymax = xmin-delta_xmin, ymin-delta_ymin, xmax+delta_xmax, ymax+delta_ymax + ax.add_patch(plt.Rectangle((xmin, ymin), xmax - xmin, ymax - ymin,fill=False, color='red', linewidth=3)) + text = f'{model.config.id2label[cl.item()]}: {p[cl]:0.2f}' + ax.text(xmin-20, ymin-50, text, fontsize=10,bbox=dict(facecolor='yellow', alpha=0.5)) + plt.axis('off') + c1.pyplot() + + + def crop_tables(self, pil_img, prob, boxes, delta_xmin, delta_ymin, delta_xmax, delta_ymax): + ''' + crop_tables and plot_results_detection must have same co-ord shifts because 1 only plots the other one updates co-ordinates + ''' + cropped_img_list = [] + + for p, (xmin, ymin, xmax, ymax) in zip(prob, boxes.tolist()): + + xmin, ymin, xmax, ymax = xmin-delta_xmin, ymin-delta_ymin, xmax+delta_xmax, ymax+delta_ymax + cropped_img = pil_img.crop((xmin, ymin, xmax, ymax)) + cropped_img_list.append(cropped_img) + + + return cropped_img_list + + def generate_structure(self, c2, model, pil_img, prob, boxes, expand_rowcol_bbox_top, expand_rowcol_bbox_bottom): + ''' + Co-ordinates are adjusted here by 3 'pixels' + To plot table pillow image and the TSR bounding boxes on the table + ''' + # st.write('img_obj') + # st.write(pil_img) + plt.figure(figsize=(32,20)) + plt.imshow(pil_img) + ax = plt.gca() + rows = {} + cols = {} + idx = 0 + + + for p, (xmin, ymin, xmax, ymax) in zip(prob, boxes.tolist()): + + xmin, ymin, xmax, ymax = xmin, ymin, xmax, ymax + cl = p.argmax() + class_text = model.config.id2label[cl.item()] + text = f'{class_text}: {p[cl]:0.2f}' + # or (class_text == 'table column') + if (class_text == 'table row') or (class_text =='table projected row header') or (class_text == 'table column'): + ax.add_patch(plt.Rectangle((xmin, ymin), xmax - xmin, ymax - ymin,fill=False, color=self.colors[cl.item()], linewidth=2)) + ax.text(xmin-10, ymin-10, text, fontsize=5, bbox=dict(facecolor='yellow', alpha=0.5)) + + if class_text == 'table row': + rows['table row.'+str(idx)] = (xmin, ymin-expand_rowcol_bbox_top, xmax, ymax+expand_rowcol_bbox_bottom) + if class_text == 'table column': + cols['table column.'+str(idx)] = (xmin, ymin-expand_rowcol_bbox_top, xmax, ymax+expand_rowcol_bbox_bottom) + + idx += 1 + + + plt.axis('on') + c2.pyplot() + return rows, cols + + def sort_table_featuresv2(self, rows:dict, cols:dict): + # Sometimes the header and first row overlap, and we need the header bbox not to have first row's bbox inside the headers bbox + rows_ = {table_feature : (xmin, ymin, xmax, ymax) for table_feature, (xmin, ymin, xmax, ymax) in sorted(rows.items(), key=lambda tup: tup[1][1])} + cols_ = {table_feature : (xmin, ymin, xmax, ymax) for table_feature, (xmin, ymin, xmax, ymax) in sorted(cols.items(), key=lambda tup: tup[1][0])} + + return rows_, cols_ + + def individual_table_featuresv2(self, pil_img, rows:dict, cols:dict): + + for k, v in rows.items(): + xmin, ymin, xmax, ymax = v + cropped_img = pil_img.crop((xmin, ymin, xmax, ymax)) + rows[k] = xmin, ymin, xmax, ymax, cropped_img + + for k, v in cols.items(): + xmin, ymin, xmax, ymax = v + cropped_img = pil_img.crop((xmin, ymin, xmax, ymax)) + cols[k] = xmin, ymin, xmax, ymax, cropped_img + + return rows, cols + + + def object_to_cellsv2(self, master_row:dict, cols:dict, expand_rowcol_bbox_top, expand_rowcol_bbox_bottom, padd_left): + '''Removes redundant bbox for rows&columns and divides each row into cells from columns + Args: + + Returns: + + + ''' + cells_img = {} + header_idx = 0 + row_idx = 0 + previous_xmax_col = 0 + new_cols = {} + new_master_row = {} + previous_ymin_row = 0 + new_cols = cols + new_master_row = master_row + ## Below 2 for loops remove redundant bounding boxes ### + # for k_col, v_col in cols.items(): + # xmin_col, _, xmax_col, _, col_img = v_col + # if (np.isclose(previous_xmax_col, xmax_col, atol=5)) or (xmin_col >= xmax_col): + # print('Found a column with double bbox') + # continue + # previous_xmax_col = xmax_col + # new_cols[k_col] = v_col + + # for k_row, v_row in master_row.items(): + # _, ymin_row, _, ymax_row, row_img = v_row + # if (np.isclose(previous_ymin_row, ymin_row, atol=5)) or (ymin_row >= ymax_row): + # print('Found a row with double bbox') + # continue + # previous_ymin_row = ymin_row + # new_master_row[k_row] = v_row + ###################################################### + for k_row, v_row in new_master_row.items(): + + _, _, _, _, row_img = v_row + xmax, ymax = row_img.size + xa, ya, xb, yb = 0, 0, 0, ymax + row_img_list = [] + # plt.imshow(row_img) + # st.pyplot() + for idx, kv in enumerate(new_cols.items()): + k_col, v_col = kv + xmin_col, _, xmax_col, _, col_img = v_col + xmin_col, xmax_col = xmin_col - padd_left - 10, xmax_col - padd_left + # plt.imshow(col_img) + # st.pyplot() + # xa + 3 : to remove borders on the left side of the cropped cell + # yb = 3: to remove row information from the above row of the cropped cell + # xb - 3: to remove borders on the right side of the cropped cell + xa = xmin_col + xb = xmax_col + if idx == 0: + xa = 0 + if idx == len(new_cols)-1: + xb = xmax + xa, ya, xb, yb = xa, ya, xb, yb + + row_img_cropped = row_img.crop((xa, ya, xb, yb)) + row_img_list.append(row_img_cropped) + + cells_img[k_row+'.'+str(row_idx)] = row_img_list + row_idx += 1 + + return cells_img, len(new_cols), len(new_master_row)-1 + + def clean_dataframe(self, df): + ''' + Remove irrelevant symbols that appear with tesseractOCR + ''' + # df.columns = [col.replace('|', '') for col in df.columns] + + for col in df.columns: + + df[col]=df[col].str.replace("'", '', regex=True) + df[col]=df[col].str.replace('"', '', regex=True) + df[col]=df[col].str.replace(']', '', regex=True) + df[col]=df[col].str.replace('[', '', regex=True) + df[col]=df[col].str.replace('{', '', regex=True) + df[col]=df[col].str.replace('}', '', regex=True) + return df + + @st.cache + def convert_df(self, df): + return df.to_csv().encode('utf-8') + + + def create_dataframe(self, c3, cells_pytess_result:list, max_cols:int, max_rows:int): + '''Create dataframe using list of cell values of the table, also checks for valid header of dataframe + Args: + cells_pytess_result: list of strings, each element representing a cell in a table + max_cols, max_rows: number of columns and rows + Returns: + dataframe : final dataframe after all pre-processing + ''' + + headers = cells_pytess_result[:max_cols] + new_headers = uniquify(headers, (f' {x!s}' for x in string.ascii_lowercase)) + counter = 0 + + cells_list = cells_pytess_result[max_cols:] + df = pd.DataFrame("", index=range(0, max_rows), columns=new_headers) + + cell_idx = 0 + for nrows in range(max_rows): + for ncols in range(max_cols): + df.iat[nrows, ncols] = str(cells_list[cell_idx]) + cell_idx += 1 + + ## To check if there are duplicate headers if result of uniquify+col == col + ## This check removes headers when all headers are empty or if median of header word count is less than 6 + for x, col in zip(string.ascii_lowercase, new_headers): + if f' {x!s}' == col: + counter += 1 + header_char_count = [len(col) for col in new_headers] + + # if (counter == len(new_headers)) or (statistics.median(header_char_count) < 6): + # st.write('woooot') + # df.columns = uniquify(df.iloc[0], (f' {x!s}' for x in string.ascii_lowercase)) + # df = df.iloc[1:,:] + + df = self.clean_dataframe(df) + + c3.dataframe(df) + csv = self.convert_df(df) + c3.download_button("Download table", csv, "file.csv", "text/csv", key='download-csv') + + return df + + + + + + + async def start_process(self, image_path:str, TD_THRESHOLD, TSR_THRESHOLD, padd_top, padd_left, padd_bottom, padd_right, delta_xmin, delta_ymin, delta_xmax, delta_ymax, expand_rowcol_bbox_top, expand_rowcol_bbox_bottom): + ''' + Initiates process of generating pandas dataframes from raw pdf-page images + + ''' + image = Image.open(image_path).convert("RGB") + model, probas, bboxes_scaled = table_detector(image, THRESHOLD_PROBA=TD_THRESHOLD) + + if bboxes_scaled.nelement() == 0: + print('No table found in the pdf-page image'+image_path.split('/')[-1]) + return '' + + # try: + # st.write('Document: '+image_path.split('/')[-1]) + c1, c2, c3 = st.columns((1,1,1)) + + self.plot_results_detection(c1, model, image, probas, bboxes_scaled, delta_xmin, delta_ymin, delta_xmax, delta_ymax) + cropped_img_list = self.crop_tables(image, probas, bboxes_scaled, delta_xmin, delta_ymin, delta_xmax, delta_ymax) + + for unpadded_table in cropped_img_list: + + table = self.add_padding(unpadded_table, padd_top, padd_right, padd_bottom, padd_left) + # table = super_res(table) + # table = binarizeBlur_image(table) + # table = sharpen_image(table) # Test sharpen image next + # table = td_postprocess(table) + + model, probas, bboxes_scaled = table_struct_recog(table, THRESHOLD_PROBA=TSR_THRESHOLD) + rows, cols = self.generate_structure(c2, model, table, probas, bboxes_scaled, expand_rowcol_bbox_top, expand_rowcol_bbox_bottom) + # st.write(len(rows), len(cols)) + rows, cols = self.sort_table_featuresv2(rows, cols) + master_row, cols = self.individual_table_featuresv2(table, rows, cols) + + cells_img, max_cols, max_rows = self.object_to_cellsv2(master_row, cols, expand_rowcol_bbox_top, expand_rowcol_bbox_bottom, padd_left) + + sequential_cell_img_list = [] + for k, img_list in cells_img.items(): + for img in img_list: + # img = super_res(img) + # img = sharpen_image(img) # Test sharpen image next + # img = binarizeBlur_image(img) + # img = self.add_padding(img, 10,10,10,10) + # plt.imshow(img) + # c3.pyplot() + sequential_cell_img_list.append(pytess(img)) + + cells_pytess_result = await asyncio.gather(*sequential_cell_img_list) + + + self.create_dataframe(c3, cells_pytess_result, max_cols, max_rows) + st.write('Errors in OCR is due to either quality of the image or performance of the OCR') + # except: + # st.write('Either incorrectly identified table or no table, to debug remove try/except') + # break + # break + + + + +if __name__ == "__main__": + + img_name = st.file_uploader("Upload an image with table(s)") + + padd_top = st.slider('Padding top', 0, 200, 20) + padd_left = st.slider('Padding left', 0, 200, 20) + padd_right = st.slider('Padding right', 0, 200, 20) + padd_bottom = st.slider('Padding bottom', 0, 200, 20) + + + te = TableExtractionPipeline() + # for img in image_list: + if img_name is not None: + asyncio.run(te.start_process(img_name, TD_THRESHOLD=0.6, TSR_THRESHOLD=0.8, padd_top=padd_top, padd_left=padd_left, padd_bottom=padd_bottom, padd_right=padd_right, delta_xmin=0, delta_ymin=0, delta_xmax=0, delta_ymax=0, expand_rowcol_bbox_top=0, expand_rowcol_bbox_bottom=0)) + + + From c3b4228d6fc576ff2b3221792c8cfe5e99a53916 Mon Sep 17 00:00:00 2001 From: Mohammed Salman <31631107+salman-moh@users.noreply.github.com> Date: Wed, 19 Oct 2022 12:56:25 +0530 Subject: [PATCH 02/16] updated streamlit app using Table Transformer + OCR --- Table Transformer/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Table Transformer/app.py b/Table Transformer/app.py index f8fc842b..910824bd 100644 --- a/Table Transformer/app.py +++ b/Table Transformer/app.py @@ -140,7 +140,7 @@ def table_detector(image, THRESHOLD_PROBA): feature_extractor = DetrFeatureExtractor(do_resize=True, size=800, max_size=800) encoding = feature_extractor(image, return_tensors="pt") - model = DetrForObjectDetection.from_pretrained("SalML/DETR-table-detection") + model = DetrForObjectDetection.from_pretrained("microsoft/table-transformer-detection") with torch.no_grad(): outputs = model(**encoding) @@ -163,7 +163,7 @@ def table_struct_recog(image, THRESHOLD_PROBA): feature_extractor = DetrFeatureExtractor(do_resize=True, size=1000, max_size=1000) encoding = feature_extractor(image, return_tensors="pt") - model = DetrForObjectDetection.from_pretrained("SalML/DETR-table-structure-recognition") + model = DetrForObjectDetection.from_pretrained("microsoft/table-transformer-structure-recognition") with torch.no_grad(): outputs = model(**encoding) From c210ab4a7b107934e48b8050a5f2e22b08f85a6d Mon Sep 17 00:00:00 2001 From: Mohammed Salman <31631107+salman-moh@users.noreply.github.com> Date: Wed, 19 Oct 2022 12:58:43 +0530 Subject: [PATCH 03/16] Updated README to include HF space link --- Table Transformer/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Table Transformer/README.md b/Table Transformer/README.md index d1c26b48..f5d5342b 100644 --- a/Table Transformer/README.md +++ b/Table Transformer/README.md @@ -7,3 +7,6 @@ can be done as shown in the notebooks found in [this folder](https://github.com/ The only difference is that the Table Transformer applies a "normalize before" operation, which means that layernorms are applied before, rather than after MLPs/attention. + +To download Table as a CSV file, theres a HuggingFace space based on the Table Transformer+OCR can be found [here](https://huggingface.co/spaces/SalML/TableTransformer2CSV) + From a6b0b545c5cd61c045e06f1fdefdf7532eaa23d4 Mon Sep 17 00:00:00 2001 From: Mohammed Salman <31631107+salman-moh@users.noreply.github.com> Date: Wed, 19 Oct 2022 13:04:04 +0530 Subject: [PATCH 04/16] Update Table Transformer/app.py Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> --- Table Transformer/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Table Transformer/app.py b/Table Transformer/app.py index 910824bd..059beb4a 100644 --- a/Table Transformer/app.py +++ b/Table Transformer/app.py @@ -140,7 +140,7 @@ def table_detector(image, THRESHOLD_PROBA): feature_extractor = DetrFeatureExtractor(do_resize=True, size=800, max_size=800) encoding = feature_extractor(image, return_tensors="pt") - model = DetrForObjectDetection.from_pretrained("microsoft/table-transformer-detection") + model = TableTransformerForObjectDetection.from_pretrained("microsoft/table-transformer-detection") with torch.no_grad(): outputs = model(**encoding) From 12c4a81507c69b752446cd5377667e57ac119ae7 Mon Sep 17 00:00:00 2001 From: Mohammed Salman <31631107+salman-moh@users.noreply.github.com> Date: Wed, 19 Oct 2022 13:04:13 +0530 Subject: [PATCH 05/16] Update Table Transformer/app.py Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> --- Table Transformer/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Table Transformer/app.py b/Table Transformer/app.py index 059beb4a..5deb5e87 100644 --- a/Table Transformer/app.py +++ b/Table Transformer/app.py @@ -163,7 +163,7 @@ def table_struct_recog(image, THRESHOLD_PROBA): feature_extractor = DetrFeatureExtractor(do_resize=True, size=1000, max_size=1000) encoding = feature_extractor(image, return_tensors="pt") - model = DetrForObjectDetection.from_pretrained("microsoft/table-transformer-structure-recognition") + model = TableTransformerForObjectDetection.from_pretrained("microsoft/table-transformer-structure-recognition") with torch.no_grad(): outputs = model(**encoding) From 5e7e9f85d12f15dfba36a48f4e2e3a8321eb8a24 Mon Sep 17 00:00:00 2001 From: Mohammed Salman <31631107+salman-moh@users.noreply.github.com> Date: Wed, 19 Oct 2022 13:13:39 +0530 Subject: [PATCH 06/16] Table Transformer details added --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d0a8b955..44c4e539 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,8 @@ Currently, it contains the following demos: * X-CLIP ([paper](https://arxiv.org/abs/2208.02816)): - performing zero-shot video classification with X-CLIP [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/X-CLIP/Video_text_matching_with_X_CLIP.ipynb) - zero-shot classifying a YouTube video with X-CLIP [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/X-CLIP/Zero_shot_classify_a_YouTube_video_with_X_CLIP.ipynb) - +* Table Transformer ([paper](https://arxiv.org/abs/2110.00061)): + - detects table and recognizes table structure on image with table [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/Table%20Transformer/Using_Table_Transformer_for_table_detection_and_table_structure_recognition.ipynb) ... more to come! 🤗 If you have any questions regarding these demos, feel free to open an issue on this repository. From 1474d2fa0d793bd0ffab31369f8f6a943ae7f6d4 Mon Sep 17 00:00:00 2001 From: Mohammed Salman <31631107+salman-moh@users.noreply.github.com> Date: Wed, 19 Oct 2022 13:14:21 +0530 Subject: [PATCH 07/16] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 44c4e539..18d37f80 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,7 @@ Currently, it contains the following demos: - zero-shot classifying a YouTube video with X-CLIP [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/X-CLIP/Zero_shot_classify_a_YouTube_video_with_X_CLIP.ipynb) * Table Transformer ([paper](https://arxiv.org/abs/2110.00061)): - detects table and recognizes table structure on image with table [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/Table%20Transformer/Using_Table_Transformer_for_table_detection_and_table_structure_recognition.ipynb) + ... more to come! 🤗 If you have any questions regarding these demos, feel free to open an issue on this repository. From 2226e5cb2b364075c73ead427708e50f2132bc0c Mon Sep 17 00:00:00 2001 From: Mohammed Salman <31631107+salman-moh@users.noreply.github.com> Date: Wed, 19 Oct 2022 13:15:05 +0530 Subject: [PATCH 08/16] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 18d37f80..e4548afa 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,7 @@ Btw, I was also the main contributor to add the following algorithms to the libr - VideoMAE by Multimedia Computing Group, Nanjing University - X-CLIP by Microsoft Research - MarkupLM by Microsoft Research +- Table Transformer by Microsoft Research All of them were an incredible learning experience. I can recommend anyone to contribute an AI algorithm to the library! From 58571e4621696203559c1ebeacbda7c7b47e70dc Mon Sep 17 00:00:00 2001 From: Mohammed Salman <31631107+salman-moh@users.noreply.github.com> Date: Wed, 19 Oct 2022 14:59:55 +0530 Subject: [PATCH 09/16] updated case for when there is no table --- Table Transformer/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Table Transformer/app.py b/Table Transformer/app.py index 5deb5e87..c052b2ad 100644 --- a/Table Transformer/app.py +++ b/Table Transformer/app.py @@ -435,7 +435,7 @@ async def start_process(self, image_path:str, TD_THRESHOLD, TSR_THRESHOLD, padd_ model, probas, bboxes_scaled = table_detector(image, THRESHOLD_PROBA=TD_THRESHOLD) if bboxes_scaled.nelement() == 0: - print('No table found in the pdf-page image'+image_path.split('/')[-1]) + st.write('No table found in the pdf-page image') return '' # try: From aeef89bc9177d16f9020838cd9828194cf8dc2d2 Mon Sep 17 00:00:00 2001 From: Mohammed Salman <31631107+salman-moh@users.noreply.github.com> Date: Wed, 19 Oct 2022 15:24:43 +0530 Subject: [PATCH 10/16] Update app.py --- Table Transformer/app.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Table Transformer/app.py b/Table Transformer/app.py index c052b2ad..4ba5cb19 100644 --- a/Table Transformer/app.py +++ b/Table Transformer/app.py @@ -488,17 +488,19 @@ async def start_process(self, image_path:str, TD_THRESHOLD, TSR_THRESHOLD, padd_ if __name__ == "__main__": img_name = st.file_uploader("Upload an image with table(s)") + st1, st2 = st.columns((1,1)) + TD_th = st1.slider('Table detection threshold', 0.0, 1.0, 0.6) + TSR_th = st2.slider('Table structure recognition threshold', 0.0, 1.0, 0.8) - padd_top = st.slider('Padding top', 0, 200, 20) - padd_left = st.slider('Padding left', 0, 200, 20) - padd_right = st.slider('Padding right', 0, 200, 20) - padd_bottom = st.slider('Padding bottom', 0, 200, 20) + st1, st2, st3, st4 = st.columns((1,1,1,1)) + padd_top = st1.slider('Padding top', 0, 200, 20) + padd_left = st2.slider('Padding left', 0, 200, 20) + padd_right = st3.slider('Padding right', 0, 200, 20) + padd_bottom = st4.slider('Padding bottom', 0, 200, 20) te = TableExtractionPipeline() # for img in image_list: if img_name is not None: - asyncio.run(te.start_process(img_name, TD_THRESHOLD=0.6, TSR_THRESHOLD=0.8, padd_top=padd_top, padd_left=padd_left, padd_bottom=padd_bottom, padd_right=padd_right, delta_xmin=0, delta_ymin=0, delta_xmax=0, delta_ymax=0, expand_rowcol_bbox_top=0, expand_rowcol_bbox_bottom=0)) - - + asyncio.run(te.start_process(img_name, TD_THRESHOLD=TD_th , TSR_THRESHOLD=TSR_th , padd_top=padd_top, padd_left=padd_left, padd_bottom=padd_bottom, padd_right=padd_right, delta_xmin=0, delta_ymin=0, delta_xmax=0, delta_ymax=0, expand_rowcol_bbox_top=0, expand_rowcol_bbox_bottom=0)) From 2f9623f32d31be083133ba1ce4ffee8c71d558a6 Mon Sep 17 00:00:00 2001 From: Mohammed Salman <31631107+salman-moh@users.noreply.github.com> Date: Wed, 19 Oct 2022 16:25:34 +0530 Subject: [PATCH 11/16] reflects HF Space streamlit app --- Table Transformer/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Table Transformer/app.py b/Table Transformer/app.py index 4ba5cb19..391578f6 100644 --- a/Table Transformer/app.py +++ b/Table Transformer/app.py @@ -16,7 +16,8 @@ # from transformers import TrOCRProcessor, VisionEncoderDecoderModel # from cv2 import dnn_superres from transformers import DetrFeatureExtractor -from transformers import DetrForObjectDetection +#from transformers import DetrForObjectDetection +from transformers import TableTransformerForObjectDetection import torch import asyncio # pytesseract.pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract.exe' @@ -503,4 +504,3 @@ async def start_process(self, image_path:str, TD_THRESHOLD, TSR_THRESHOLD, padd_ # for img in image_list: if img_name is not None: asyncio.run(te.start_process(img_name, TD_THRESHOLD=TD_th , TSR_THRESHOLD=TSR_th , padd_top=padd_top, padd_left=padd_left, padd_bottom=padd_bottom, padd_right=padd_right, delta_xmin=0, delta_ymin=0, delta_xmax=0, delta_ymax=0, expand_rowcol_bbox_top=0, expand_rowcol_bbox_bottom=0)) - From 2b5bce37c093ac0b48a1ddd96330a9336d5201e3 Mon Sep 17 00:00:00 2001 From: Mohammed Salman <31631107+salman-moh@users.noreply.github.com> Date: Sat, 22 Oct 2022 14:35:13 +0530 Subject: [PATCH 12/16] Update README.md --- Table Transformer/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Table Transformer/README.md b/Table Transformer/README.md index f5d5342b..d9834d13 100644 --- a/Table Transformer/README.md +++ b/Table Transformer/README.md @@ -8,5 +8,5 @@ can be done as shown in the notebooks found in [this folder](https://github.com/ The only difference is that the Table Transformer applies a "normalize before" operation, which means that layernorms are applied before, rather than after MLPs/attention. -To download Table as a CSV file, theres a HuggingFace space based on the Table Transformer+OCR can be found [here](https://huggingface.co/spaces/SalML/TableTransformer2CSV) +To download Table as a CSV file, theres a [DEMO](https://huggingface.co/spaces/SalML/TableTransformer2CSV) on HuggingFace space based on the Table Transformer+OCR. From 4922aa40dde4c1ef42d621e827344f3c1b07d8f2 Mon Sep 17 00:00:00 2001 From: Mohammed Salman <31631107+salman-moh@users.noreply.github.com> Date: Sat, 22 Oct 2022 14:55:24 +0530 Subject: [PATCH 13/16] Deleted to have this repo as notebook-only --- Table Transformer/app.py | 506 --------------------------------------- 1 file changed, 506 deletions(-) delete mode 100644 Table Transformer/app.py diff --git a/Table Transformer/app.py b/Table Transformer/app.py deleted file mode 100644 index 391578f6..00000000 --- a/Table Transformer/app.py +++ /dev/null @@ -1,506 +0,0 @@ -import streamlit as st -from PIL import Image, ImageEnhance -import statistics -import os -import string -from collections import Counter -from itertools import tee, count -# import TDTSR -import pytesseract -from pytesseract import Output -import json -import pandas as pd -import matplotlib.pyplot as plt -import cv2 -import numpy as np -# from transformers import TrOCRProcessor, VisionEncoderDecoderModel -# from cv2 import dnn_superres -from transformers import DetrFeatureExtractor -#from transformers import DetrForObjectDetection -from transformers import TableTransformerForObjectDetection -import torch -import asyncio -# pytesseract.pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract.exe' - - -st.set_option('deprecation.showPyplotGlobalUse', False) -st.set_page_config(layout='wide') -st.title("Table Detection and Table Structure Recognition") -st.write("Implemented by MSFT team: https://github.com/microsoft/table-transformer") - - -def PIL_to_cv(pil_img): - return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) - -def cv_to_PIL(cv_img): - return Image.fromarray(cv2.cvtColor(cv_img, cv2.COLOR_BGR2RGB)) - - -async def pytess(cell_pil_img): - return ' '.join(pytesseract.image_to_data(cell_pil_img, output_type=Output.DICT, config='-c tessedit_char_blacklist=œ˜â€œï¬â™Ã©œ¢!|”?«“¥ --psm 6 preserve_interword_spaces')['text']).strip() - - -# def super_res(pil_img): - # ''' - # Useful for low-res docs - # ''' - # requires opencv-contrib-python installed without the opencv-python - # sr = dnn_superres.DnnSuperResImpl_create() - # image = PIL_to_cv(pil_img) - # model_path = "/data/Salman/TRD/code/table-transformer/transformers/LapSRN_x2.pb" - # model_name = 'lapsrn' - # model_scale = 2 - # sr.readModel(model_path) - # sr.setModel(model_name, model_scale) - # final_img = sr.upsample(image) - # final_img = cv_to_PIL(final_img) - - # return final_img - - -def sharpen_image(pil_img): - - img = PIL_to_cv(pil_img) - sharpen_kernel = np.array([[-1, -1, -1], - [-1, 9, -1], - [-1, -1, -1]]) - - sharpen = cv2.filter2D(img, -1, sharpen_kernel) - pil_img = cv_to_PIL(sharpen) - return pil_img - - -def uniquify(seq, suffs = count(1)): - """Make all the items unique by adding a suffix (1, 2, etc). - Credit: https://stackoverflow.com/questions/30650474/python-rename-duplicates-in-list-with-progressive-numbers-without-sorting-list - `seq` is mutable sequence of strings. - `suffs` is an optional alternative suffix iterable. - """ - not_unique = [k for k,v in Counter(seq).items() if v>1] - - suff_gens = dict(zip(not_unique, tee(suffs, len(not_unique)))) - for idx,s in enumerate(seq): - try: - suffix = str(next(suff_gens[s])) - except KeyError: - continue - else: - seq[idx] += suffix - - return seq - -def binarizeBlur_image(pil_img): - image = PIL_to_cv(pil_img) - thresh = cv2.threshold(image, 150, 255, cv2.THRESH_BINARY_INV)[1] - - result = cv2.GaussianBlur(thresh, (5,5), 0) - result = 255 - result - return cv_to_PIL(result) - - - -def td_postprocess(pil_img): - ''' - Removes gray background from tables - ''' - img = PIL_to_cv(pil_img) - - hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV) - mask = cv2.inRange(hsv, (0, 0, 100), (255, 5, 255)) # (0, 0, 100), (255, 5, 255) - nzmask = cv2.inRange(hsv, (0, 0, 5), (255, 255, 255)) # (0, 0, 5), (255, 255, 255)) - nzmask = cv2.erode(nzmask, np.ones((3,3))) # (3,3) - mask = mask & nzmask - - new_img = img.copy() - new_img[np.where(mask)] = 255 - - - return cv_to_PIL(new_img) - -# def super_res(pil_img): -# # requires opencv-contrib-python installed without the opencv-python -# sr = dnn_superres.DnnSuperResImpl_create() -# image = PIL_to_cv(pil_img) -# model_path = "./LapSRN_x8.pb" -# model_name = model_path.split('/')[1].split('_')[0].lower() -# model_scale = int(model_path.split('/')[1].split('_')[1].split('.')[0][1]) - -# sr.readModel(model_path) -# sr.setModel(model_name, model_scale) -# final_img = sr.upsample(image) -# final_img = cv_to_PIL(final_img) - -# return final_img - -def table_detector(image, THRESHOLD_PROBA): - ''' - Table detection using DEtect-object TRansformer pre-trained on 1 million tables - - ''' - - feature_extractor = DetrFeatureExtractor(do_resize=True, size=800, max_size=800) - encoding = feature_extractor(image, return_tensors="pt") - - model = TableTransformerForObjectDetection.from_pretrained("microsoft/table-transformer-detection") - - with torch.no_grad(): - outputs = model(**encoding) - - probas = outputs.logits.softmax(-1)[0, :, :-1] - keep = probas.max(-1).values > THRESHOLD_PROBA - - target_sizes = torch.tensor(image.size[::-1]).unsqueeze(0) - postprocessed_outputs = feature_extractor.post_process(outputs, target_sizes) - bboxes_scaled = postprocessed_outputs[0]['boxes'][keep] - - return (model, probas[keep], bboxes_scaled) - - -def table_struct_recog(image, THRESHOLD_PROBA): - ''' - Table structure recognition using DEtect-object TRansformer pre-trained on 1 million tables - ''' - - feature_extractor = DetrFeatureExtractor(do_resize=True, size=1000, max_size=1000) - encoding = feature_extractor(image, return_tensors="pt") - - model = TableTransformerForObjectDetection.from_pretrained("microsoft/table-transformer-structure-recognition") - with torch.no_grad(): - outputs = model(**encoding) - - probas = outputs.logits.softmax(-1)[0, :, :-1] - keep = probas.max(-1).values > THRESHOLD_PROBA - - target_sizes = torch.tensor(image.size[::-1]).unsqueeze(0) - postprocessed_outputs = feature_extractor.post_process(outputs, target_sizes) - bboxes_scaled = postprocessed_outputs[0]['boxes'][keep] - - return (model, probas[keep], bboxes_scaled) - - - - - -class TableExtractionPipeline(): - - colors = ["red", "blue", "green", "yellow", "orange", "violet"] - - # colors = ["red", "blue", "green", "red", "red", "red"] - - def add_padding(self, pil_img, top, right, bottom, left, color=(255,255,255)): - ''' - Image padding as part of TSR pre-processing to prevent missing table edges - ''' - width, height = pil_img.size - new_width = width + right + left - new_height = height + top + bottom - result = Image.new(pil_img.mode, (new_width, new_height), color) - result.paste(pil_img, (left, top)) - return result - - def plot_results_detection(self, c1, model, pil_img, prob, boxes, delta_xmin, delta_ymin, delta_xmax, delta_ymax): - ''' - crop_tables and plot_results_detection must have same co-ord shifts because 1 only plots the other one updates co-ordinates - ''' - # st.write('img_obj') - # st.write(pil_img) - plt.imshow(pil_img) - ax = plt.gca() - - for p, (xmin, ymin, xmax, ymax) in zip(prob, boxes.tolist()): - cl = p.argmax() - xmin, ymin, xmax, ymax = xmin-delta_xmin, ymin-delta_ymin, xmax+delta_xmax, ymax+delta_ymax - ax.add_patch(plt.Rectangle((xmin, ymin), xmax - xmin, ymax - ymin,fill=False, color='red', linewidth=3)) - text = f'{model.config.id2label[cl.item()]}: {p[cl]:0.2f}' - ax.text(xmin-20, ymin-50, text, fontsize=10,bbox=dict(facecolor='yellow', alpha=0.5)) - plt.axis('off') - c1.pyplot() - - - def crop_tables(self, pil_img, prob, boxes, delta_xmin, delta_ymin, delta_xmax, delta_ymax): - ''' - crop_tables and plot_results_detection must have same co-ord shifts because 1 only plots the other one updates co-ordinates - ''' - cropped_img_list = [] - - for p, (xmin, ymin, xmax, ymax) in zip(prob, boxes.tolist()): - - xmin, ymin, xmax, ymax = xmin-delta_xmin, ymin-delta_ymin, xmax+delta_xmax, ymax+delta_ymax - cropped_img = pil_img.crop((xmin, ymin, xmax, ymax)) - cropped_img_list.append(cropped_img) - - - return cropped_img_list - - def generate_structure(self, c2, model, pil_img, prob, boxes, expand_rowcol_bbox_top, expand_rowcol_bbox_bottom): - ''' - Co-ordinates are adjusted here by 3 'pixels' - To plot table pillow image and the TSR bounding boxes on the table - ''' - # st.write('img_obj') - # st.write(pil_img) - plt.figure(figsize=(32,20)) - plt.imshow(pil_img) - ax = plt.gca() - rows = {} - cols = {} - idx = 0 - - - for p, (xmin, ymin, xmax, ymax) in zip(prob, boxes.tolist()): - - xmin, ymin, xmax, ymax = xmin, ymin, xmax, ymax - cl = p.argmax() - class_text = model.config.id2label[cl.item()] - text = f'{class_text}: {p[cl]:0.2f}' - # or (class_text == 'table column') - if (class_text == 'table row') or (class_text =='table projected row header') or (class_text == 'table column'): - ax.add_patch(plt.Rectangle((xmin, ymin), xmax - xmin, ymax - ymin,fill=False, color=self.colors[cl.item()], linewidth=2)) - ax.text(xmin-10, ymin-10, text, fontsize=5, bbox=dict(facecolor='yellow', alpha=0.5)) - - if class_text == 'table row': - rows['table row.'+str(idx)] = (xmin, ymin-expand_rowcol_bbox_top, xmax, ymax+expand_rowcol_bbox_bottom) - if class_text == 'table column': - cols['table column.'+str(idx)] = (xmin, ymin-expand_rowcol_bbox_top, xmax, ymax+expand_rowcol_bbox_bottom) - - idx += 1 - - - plt.axis('on') - c2.pyplot() - return rows, cols - - def sort_table_featuresv2(self, rows:dict, cols:dict): - # Sometimes the header and first row overlap, and we need the header bbox not to have first row's bbox inside the headers bbox - rows_ = {table_feature : (xmin, ymin, xmax, ymax) for table_feature, (xmin, ymin, xmax, ymax) in sorted(rows.items(), key=lambda tup: tup[1][1])} - cols_ = {table_feature : (xmin, ymin, xmax, ymax) for table_feature, (xmin, ymin, xmax, ymax) in sorted(cols.items(), key=lambda tup: tup[1][0])} - - return rows_, cols_ - - def individual_table_featuresv2(self, pil_img, rows:dict, cols:dict): - - for k, v in rows.items(): - xmin, ymin, xmax, ymax = v - cropped_img = pil_img.crop((xmin, ymin, xmax, ymax)) - rows[k] = xmin, ymin, xmax, ymax, cropped_img - - for k, v in cols.items(): - xmin, ymin, xmax, ymax = v - cropped_img = pil_img.crop((xmin, ymin, xmax, ymax)) - cols[k] = xmin, ymin, xmax, ymax, cropped_img - - return rows, cols - - - def object_to_cellsv2(self, master_row:dict, cols:dict, expand_rowcol_bbox_top, expand_rowcol_bbox_bottom, padd_left): - '''Removes redundant bbox for rows&columns and divides each row into cells from columns - Args: - - Returns: - - - ''' - cells_img = {} - header_idx = 0 - row_idx = 0 - previous_xmax_col = 0 - new_cols = {} - new_master_row = {} - previous_ymin_row = 0 - new_cols = cols - new_master_row = master_row - ## Below 2 for loops remove redundant bounding boxes ### - # for k_col, v_col in cols.items(): - # xmin_col, _, xmax_col, _, col_img = v_col - # if (np.isclose(previous_xmax_col, xmax_col, atol=5)) or (xmin_col >= xmax_col): - # print('Found a column with double bbox') - # continue - # previous_xmax_col = xmax_col - # new_cols[k_col] = v_col - - # for k_row, v_row in master_row.items(): - # _, ymin_row, _, ymax_row, row_img = v_row - # if (np.isclose(previous_ymin_row, ymin_row, atol=5)) or (ymin_row >= ymax_row): - # print('Found a row with double bbox') - # continue - # previous_ymin_row = ymin_row - # new_master_row[k_row] = v_row - ###################################################### - for k_row, v_row in new_master_row.items(): - - _, _, _, _, row_img = v_row - xmax, ymax = row_img.size - xa, ya, xb, yb = 0, 0, 0, ymax - row_img_list = [] - # plt.imshow(row_img) - # st.pyplot() - for idx, kv in enumerate(new_cols.items()): - k_col, v_col = kv - xmin_col, _, xmax_col, _, col_img = v_col - xmin_col, xmax_col = xmin_col - padd_left - 10, xmax_col - padd_left - # plt.imshow(col_img) - # st.pyplot() - # xa + 3 : to remove borders on the left side of the cropped cell - # yb = 3: to remove row information from the above row of the cropped cell - # xb - 3: to remove borders on the right side of the cropped cell - xa = xmin_col - xb = xmax_col - if idx == 0: - xa = 0 - if idx == len(new_cols)-1: - xb = xmax - xa, ya, xb, yb = xa, ya, xb, yb - - row_img_cropped = row_img.crop((xa, ya, xb, yb)) - row_img_list.append(row_img_cropped) - - cells_img[k_row+'.'+str(row_idx)] = row_img_list - row_idx += 1 - - return cells_img, len(new_cols), len(new_master_row)-1 - - def clean_dataframe(self, df): - ''' - Remove irrelevant symbols that appear with tesseractOCR - ''' - # df.columns = [col.replace('|', '') for col in df.columns] - - for col in df.columns: - - df[col]=df[col].str.replace("'", '', regex=True) - df[col]=df[col].str.replace('"', '', regex=True) - df[col]=df[col].str.replace(']', '', regex=True) - df[col]=df[col].str.replace('[', '', regex=True) - df[col]=df[col].str.replace('{', '', regex=True) - df[col]=df[col].str.replace('}', '', regex=True) - return df - - @st.cache - def convert_df(self, df): - return df.to_csv().encode('utf-8') - - - def create_dataframe(self, c3, cells_pytess_result:list, max_cols:int, max_rows:int): - '''Create dataframe using list of cell values of the table, also checks for valid header of dataframe - Args: - cells_pytess_result: list of strings, each element representing a cell in a table - max_cols, max_rows: number of columns and rows - Returns: - dataframe : final dataframe after all pre-processing - ''' - - headers = cells_pytess_result[:max_cols] - new_headers = uniquify(headers, (f' {x!s}' for x in string.ascii_lowercase)) - counter = 0 - - cells_list = cells_pytess_result[max_cols:] - df = pd.DataFrame("", index=range(0, max_rows), columns=new_headers) - - cell_idx = 0 - for nrows in range(max_rows): - for ncols in range(max_cols): - df.iat[nrows, ncols] = str(cells_list[cell_idx]) - cell_idx += 1 - - ## To check if there are duplicate headers if result of uniquify+col == col - ## This check removes headers when all headers are empty or if median of header word count is less than 6 - for x, col in zip(string.ascii_lowercase, new_headers): - if f' {x!s}' == col: - counter += 1 - header_char_count = [len(col) for col in new_headers] - - # if (counter == len(new_headers)) or (statistics.median(header_char_count) < 6): - # st.write('woooot') - # df.columns = uniquify(df.iloc[0], (f' {x!s}' for x in string.ascii_lowercase)) - # df = df.iloc[1:,:] - - df = self.clean_dataframe(df) - - c3.dataframe(df) - csv = self.convert_df(df) - c3.download_button("Download table", csv, "file.csv", "text/csv", key='download-csv') - - return df - - - - - - - async def start_process(self, image_path:str, TD_THRESHOLD, TSR_THRESHOLD, padd_top, padd_left, padd_bottom, padd_right, delta_xmin, delta_ymin, delta_xmax, delta_ymax, expand_rowcol_bbox_top, expand_rowcol_bbox_bottom): - ''' - Initiates process of generating pandas dataframes from raw pdf-page images - - ''' - image = Image.open(image_path).convert("RGB") - model, probas, bboxes_scaled = table_detector(image, THRESHOLD_PROBA=TD_THRESHOLD) - - if bboxes_scaled.nelement() == 0: - st.write('No table found in the pdf-page image') - return '' - - # try: - # st.write('Document: '+image_path.split('/')[-1]) - c1, c2, c3 = st.columns((1,1,1)) - - self.plot_results_detection(c1, model, image, probas, bboxes_scaled, delta_xmin, delta_ymin, delta_xmax, delta_ymax) - cropped_img_list = self.crop_tables(image, probas, bboxes_scaled, delta_xmin, delta_ymin, delta_xmax, delta_ymax) - - for unpadded_table in cropped_img_list: - - table = self.add_padding(unpadded_table, padd_top, padd_right, padd_bottom, padd_left) - # table = super_res(table) - # table = binarizeBlur_image(table) - # table = sharpen_image(table) # Test sharpen image next - # table = td_postprocess(table) - - model, probas, bboxes_scaled = table_struct_recog(table, THRESHOLD_PROBA=TSR_THRESHOLD) - rows, cols = self.generate_structure(c2, model, table, probas, bboxes_scaled, expand_rowcol_bbox_top, expand_rowcol_bbox_bottom) - # st.write(len(rows), len(cols)) - rows, cols = self.sort_table_featuresv2(rows, cols) - master_row, cols = self.individual_table_featuresv2(table, rows, cols) - - cells_img, max_cols, max_rows = self.object_to_cellsv2(master_row, cols, expand_rowcol_bbox_top, expand_rowcol_bbox_bottom, padd_left) - - sequential_cell_img_list = [] - for k, img_list in cells_img.items(): - for img in img_list: - # img = super_res(img) - # img = sharpen_image(img) # Test sharpen image next - # img = binarizeBlur_image(img) - # img = self.add_padding(img, 10,10,10,10) - # plt.imshow(img) - # c3.pyplot() - sequential_cell_img_list.append(pytess(img)) - - cells_pytess_result = await asyncio.gather(*sequential_cell_img_list) - - - self.create_dataframe(c3, cells_pytess_result, max_cols, max_rows) - st.write('Errors in OCR is due to either quality of the image or performance of the OCR') - # except: - # st.write('Either incorrectly identified table or no table, to debug remove try/except') - # break - # break - - - - -if __name__ == "__main__": - - img_name = st.file_uploader("Upload an image with table(s)") - st1, st2 = st.columns((1,1)) - TD_th = st1.slider('Table detection threshold', 0.0, 1.0, 0.6) - TSR_th = st2.slider('Table structure recognition threshold', 0.0, 1.0, 0.8) - - st1, st2, st3, st4 = st.columns((1,1,1,1)) - - padd_top = st1.slider('Padding top', 0, 200, 20) - padd_left = st2.slider('Padding left', 0, 200, 20) - padd_right = st3.slider('Padding right', 0, 200, 20) - padd_bottom = st4.slider('Padding bottom', 0, 200, 20) - - te = TableExtractionPipeline() - # for img in image_list: - if img_name is not None: - asyncio.run(te.start_process(img_name, TD_THRESHOLD=TD_th , TSR_THRESHOLD=TSR_th , padd_top=padd_top, padd_left=padd_left, padd_bottom=padd_bottom, padd_right=padd_right, delta_xmin=0, delta_ymin=0, delta_xmax=0, delta_ymax=0, expand_rowcol_bbox_top=0, expand_rowcol_bbox_bottom=0)) From 3c915df57e4ce5127f0ca17e80465fb747c2aac3 Mon Sep 17 00:00:00 2001 From: Mohammed Salman <31631107+salman-moh@users.noreply.github.com> Date: Sat, 22 Oct 2022 14:58:45 +0530 Subject: [PATCH 14/16] uploaded demo screenshot --- Table Transformer/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Table Transformer/README.md b/Table Transformer/README.md index d9834d13..31ff0a0b 100644 --- a/Table Transformer/README.md +++ b/Table Transformer/README.md @@ -10,3 +10,5 @@ rather than after MLPs/attention. To download Table as a CSV file, theres a [DEMO](https://huggingface.co/spaces/SalML/TableTransformer2CSV) on HuggingFace space based on the Table Transformer+OCR. + +![432d09f05f9178c0929729ae27b2928e](https://user-images.githubusercontent.com/31631107/197332016-de9314bc-2159-44bb-9428-ef07c6a96850.png) From cffc6d6ba68e6bc54a76f1e3bb93009c424e342d Mon Sep 17 00:00:00 2001 From: Mohammed Salman <31631107+salman-moh@users.noreply.github.com> Date: Sat, 22 Oct 2022 15:04:09 +0530 Subject: [PATCH 15/16] Update Table Transformer/README.md Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> --- Table Transformer/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Table Transformer/README.md b/Table Transformer/README.md index 31ff0a0b..9277c153 100644 --- a/Table Transformer/README.md +++ b/Table Transformer/README.md @@ -8,7 +8,7 @@ can be done as shown in the notebooks found in [this folder](https://github.com/ The only difference is that the Table Transformer applies a "normalize before" operation, which means that layernorms are applied before, rather than after MLPs/attention. -To download Table as a CSV file, theres a [DEMO](https://huggingface.co/spaces/SalML/TableTransformer2CSV) on HuggingFace space based on the Table Transformer+OCR. +To automatically parse a table and turn it into a CSV file, check out [this demo](https://huggingface.co/spaces/SalML/TableTransformer2CSV) on HuggingFace Spaces based on the Table Transformer + OCR. ![432d09f05f9178c0929729ae27b2928e](https://user-images.githubusercontent.com/31631107/197332016-de9314bc-2159-44bb-9428-ef07c6a96850.png) From c382285d4cd7bfffce1a5af57d725138e78c9741 Mon Sep 17 00:00:00 2001 From: Mohammed Salman <31631107+salman-moh@users.noreply.github.com> Date: Sun, 23 Oct 2022 14:55:08 +0530 Subject: [PATCH 16/16] Update README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e4548afa..5ff7cc9b 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,8 @@ Currently, it contains the following demos: * T5 ([paper](https://arxiv.org/abs/1910.10683)): - fine-tuning `T5ForConditionalGeneration` on a Dutch summarization dataset on TPU using HuggingFace Accelerate [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/tree/master/T5) - fine-tuning `T5ForConditionalGeneration` (CodeT5) for Ruby code summarization using PyTorch Lightning [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/T5/Fine_tune_CodeT5_for_generating_docstrings_from_Ruby_code.ipynb) +* Table Transformer ([paper](https://arxiv.org/abs/2110.00061)): + - detects table and recognizes table structure on image with table [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/Table%20Transformer/Using_Table_Transformer_for_table_detection_and_table_structure_recognition.ipynb) * TAPAS ([paper](https://arxiv.org/abs/2004.02349)): - fine-tuning `TapasForQuestionAnswering` on the Microsoft [Sequential Question Answering (SQA)](https://www.microsoft.com/en-us/download/details.aspx?id=54253) dataset [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/TAPAS/Fine_tuning_TapasForQuestionAnswering_on_SQA.ipynb) - evaluating `TapasForSequenceClassification` on the [Table Fact Checking (TabFact)](https://tabfact.github.io/) dataset [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/TAPAS/Evaluating_TAPAS_on_the_Tabfact_test_set.ipynb) @@ -111,8 +113,7 @@ Currently, it contains the following demos: * X-CLIP ([paper](https://arxiv.org/abs/2208.02816)): - performing zero-shot video classification with X-CLIP [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/X-CLIP/Video_text_matching_with_X_CLIP.ipynb) - zero-shot classifying a YouTube video with X-CLIP [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/X-CLIP/Zero_shot_classify_a_YouTube_video_with_X_CLIP.ipynb) -* Table Transformer ([paper](https://arxiv.org/abs/2110.00061)): - - detects table and recognizes table structure on image with table [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/Table%20Transformer/Using_Table_Transformer_for_table_detection_and_table_structure_recognition.ipynb) + ... more to come! 🤗