Explainer and its interface.py

# -*- coding: utf-8 -*-
"""Explainer and its Interface.ipynb

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/1ASR8Nt7ql6et3hZ4RGhY6t3tzVlYw4ii

## **Explain the predictions of the Satellite Image Classification Model**

**Author : Purushothaman Natarajan**

Source Dataset : Kaggle

Dataset Name : AID: A scene classification dataset

Link : https://www.kaggle.com/datasets/jiayuanchengala/aid-scene-classification-datasets

Approach: The developed model is explained using the explainers namely LIME and GradCAMM. Both these explainers are post-hoc in nature, so they explain the predictions from model using the input and the output, hence it does not demand the training dataset.

LIME: It perturbs the input and fits it in a linear model to verify whether the selected feature is contributing for the prediction or not.

GradCAMM: It uses class activation maps, where in the gradients with high score is visualized; this method suitable for model based on CNN networks.

Atlast, we also integrate the model and the explainer on a interface (Using Gradio) through which the user can upload a image and get its class label along with its explantion.
"""

!pip install gradio
!pip install tensorflow
!pip install keras
!pip install lime
!pip install grad-cam

from google.colab import drive
drive.mount('/content/drive')

# Load the Model

import os
import tensorflow as tf

model_path = "/content/drive/MyDrive/RS LLM/Classification Model/Model and Data Backups/Model/InceptionResNetV2_full_model.h5"

# Load the model without custom objects
loaded_model = tf.keras.models.load_model(model_path)

# Print the loaded model names
print("Loaded model")

# Load the Encoder-Decoder

import numpy as np
from sklearn.preprocessing import LabelEncoder

master_folder = "/content/drive/MyDrive/RS LLM/Classification Model/Data Backups"

# Load label encoder classes
label_encoder_classes = np.load(os.path.join(master_folder, 'onehot_encoded_labels.npy'))

# Set the classes_ attribute of the LabelEncoder
label_encoder = LabelEncoder()
label_encoder.classes_ = label_encoder_classes

class_labels = ['Airport', 'BareLand', 'BaseballField', 'Beach', 'Bridge', 'Center', 'Church', 'Commercial', 'DenseResidential', 'Desert', 'Farmland'
		'Forest', 'Industrial', 'Meadow', 'MediumResidential', 'Mountain', 'Park', 'Parking', 'Playground', 'Pond', 'Port' , 'RailwayStation',
		'Resort', 'River', 'School', 'SparseResidential', 'Square', 'Stadium', 'StorageTanks', 'Viaduct']

label_encoder.fit(class_labels)

print('Encoder-Decoder Loaded')

"""# **Grad-CAM**"""

loaded_model.summary()

# last convolution layer for grad-camm explanation

last_conv_layer_name = "conv_7b_ac"

## GRAD-CAM

def get_img_array(img_path, size):
    # `img` is a PIL image of size 299x299
    img = keras.utils.load_img(img_path, target_size=size)
    # `array` is a float32 Numpy array of shape (299, 299, 3)
    array = keras.utils.img_to_array(img)
    # We add a dimension to transform our array into a "batch"
    # of size (1, 299, 299, 3)
    array = np.expand_dims(array, axis=0)
    return array

def make_gradcam_heatmap(img_array, model, last_conv_layer_name, pred_index=None):
    # First, we create a model that maps the input image to the activations
    # of the last conv layer as well as the output predictions
    grad_model = keras.models.Model(
        inputs = model.inputs, outputs = [model.get_layer(last_conv_layer_name).output, model.output]
    )

    # Then, we compute the gradient of the top predicted class for our input image
    # with respect to the activations of the last conv layer
    with tf.GradientTape() as tape:
        inputs = tf.cast(img_array, tf.float32)
        last_conv_layer_output, preds = grad_model(img_array)
        class_channel = preds[:, pred_index]
        # if pred_index is None:
        #     pred_index = tf.argmax(preds[0])
        # class_channel = preds[:, pred_index]

    # This is the gradient of the output neuron (top predicted or chosen)
    # with regard to the output feature map of the last conv layer
    grads = tape.gradient(class_channel, last_conv_layer_output)

    # This is a vector where each entry is the mean intensity of the gradient
    # over a specific feature map channel
    pooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))

    # We multiply each channel in the feature map array
    # by "how important this channel is" with regard to the top predicted class
    # then sum all the channels to obtain the heatmap class activation
    last_conv_layer_output = last_conv_layer_output[0]
    heatmap = last_conv_layer_output @ pooled_grads[..., tf.newaxis]
    heatmap = tf.squeeze(heatmap)

    # For visualization purpose, we will also normalize the heatmap between 0 & 1
    heatmap = tf.maximum(heatmap, 0) / tf.math.reduce_max(heatmap)
    return heatmap.numpy()

"""# **SP-LIME**"""

# Initialize the LIME Explainer

import keras
from tensorflow.keras.preprocessing.image import img_to_array, load_img
from lime.lime_image import LimeImageExplainer, SegmentationAlgorithm
import numpy as np
import gradio as gr
import matplotlib as mpl

# Define the target size for resizing
target_size = (299, 299)

# Function to generate SP-LIME mask and overlay on the image for top N predictions
def generate_splime_mask_top_n(img_array, model, explainer, top_n=1, num_features=100, num_samples=300):
    # Use superpixel segmentation for SP-LIME
    segmentation_fn = SegmentationAlgorithm('quickshift', kernel_size=4, max_dist=200, ratio=0.2)

    explanation_instance = explainer.explain_instance(
        img_array, model.predict, top_labels=top_n, hide_color=0,
        num_samples=num_samples, num_features=num_features, segmentation_fn=segmentation_fn
    )
    explanation_mask = explanation_instance.get_image_and_mask(
        explanation_instance.top_labels[0], positive_only=True,
        num_features=num_features, hide_rest=True
    )

    return explanation_mask

# Define different parameters for LimeImageExplainer
segmentation_algorithms = ['quickshift', 'slic']
kernel_sizes = [2]
max_dists = [100]
ratios = [0.1]
num_features_values = [100]
num_samples_values = [300]

# Define label mapping
label_to_class = {
    0: 'Airport',
    1: 'BareLand',
    2: 'BaseballField',
    3: 'Beach',
    4: 'Bridge',
    5: 'Center',
    6: 'Church',
    7: 'Commercial',
    8: 'DenseResidential',
    9: 'Desert',
    10: 'Farmland',
    11: 'Forest',
    12: 'Industrial',
    13: 'Meadow',
    14: 'MediumResidential',
    15: 'Mountain',
    16: 'Park',
    17: 'Parking',
    18: 'Playground',
    19: 'Pond',
    20: 'Port',
    21: 'RailwayStation',
    22: 'Resort',
    23: 'River',
    24: 'School',
    25: 'SparseResidential',
    26: 'Square',
    27: 'Stadium',
    28: 'StorageTanks',
    29: 'Vaiduct'
}

import numpy as np
import gradio as gr
import matplotlib.pyplot as plt
from PIL import Image
from tensorflow.keras.preprocessing.image import img_to_array, load_img
from lime.lime_image import LimeImageExplainer, SegmentationAlgorithm
import os

# path to save the files in the folder
output_folder = "/content/drive/MyDrive/RS LLM/Classification Model/Explainer outputs"
os.makedirs(output_folder, exist_ok=True)  # Ensure the output folder exists

# Counter for images
image_counter = 0


def save_and_display_gradcam(array, heatmap, cam_path="cam.jpg", alpha=0.8):
    img = array
    heatmap = np.uint8(255 * heatmap)
    jet = plt.cm.jet
    jet_colors = jet(np.arange(256))[:, :3]
    jet_heatmap = jet_colors[heatmap]
    jet_heatmap = keras.utils.array_to_img(jet_heatmap)
    jet_heatmap = jet_heatmap.resize((img.shape[1], img.shape[0]))
    jet_heatmap = keras.utils.img_to_array(jet_heatmap)
    superimposed_img = jet_heatmap * alpha + img
    superimposed_img = keras.utils.array_to_img(superimposed_img)
    superimposed_img.save(cam_path)
    return superimposed_img

def classify_image_and_explain(image, num_samples, num_features):
    global image_counter
    image_counter += 1

    target_size = (299, 299)
    image = image.resize(target_size)
    if image.mode != 'RGB':
        image = image.convert('RGB')
    array = img_to_array(image)
    img_array = array / 255.0
    img_array = np.expand_dims(img_array, axis=0)
    # preds = loaded_model.predict(img_array)
    # top_prediction = np.argmax(preds[0])
    # top_label = label_encoder.inverse_transform([top_prediction])[0]
    # top_prob = preds[0][top_prediction]


    # Predict using the loaded model
    preds = loaded_model.predict(img_array)
    top_prediction = np.argmax(preds[0])
    top_label = label_to_class[top_prediction]
    top_prob = preds[0][top_prediction]

    def top_3_predictions(predictions):
        top_3_indices = np.argsort(predictions)[-3:][::-1]
        top_3_predictions = predictions[top_3_indices]
        return top_3_predictions

    top_3_predictions = top_3_predictions(preds[0])
    #top_3_labels = label_encoder.inverse_transform(np.argsort(preds[0])[-3:][::-1])

    explainer = LimeImageExplainer()
    segmentation_fn = SegmentationAlgorithm('quickshift', kernel_size=2, max_dist=100, ratio=0.1)
    splime_mask, explanation_instance = generate_splime_mask_top_n(
        img_array[0], loaded_model, explainer, top_n=3, num_features=num_features, num_samples=num_samples
    )

    loaded_model.layers[-1].activation = None
    output_tensor = loaded_model.output[0]
    heatmap = make_gradcam_heatmap(img_array, loaded_model, last_conv_layer_name)
    grad_cam_explanation = save_and_display_gradcam(array, heatmap)
    # Convert the SPLIME mask to PIL image
    splime_mask_pil = keras.utils.array_to_img(splime_mask)

    # Save input image
    input_img_path = os.path.join(output_folder, f"{top_label}_input_{image_counter}.jpg")
    image.save(input_img_path)
    # Save LIME explanation image
    lime_img_path = os.path.join(output_folder, f"{top_label}_lime_explanation_{image_counter}.jpg")
    splime_mask_pil.save(lime_img_path)
    # Save Grad-CAM explanation image
    gradcam_img_path = os.path.join(output_folder, f"{top_label}_gradcam_explanation_{image_counter}.jpg")
    grad_cam_explanation.save(gradcam_img_path)

    return splime_mask_pil, grad_cam_explanation, top_label, top_prob

explainer_interface = gr.Interface(
    fn=classify_image_and_explain,
    inputs=[
        gr.Image(type="pil"),
        gr.Slider(10, 1000, value=10, step=1, label="Number of Samples", info="Choose between 10 and 1000"),
        gr.Slider(10, 100, value=10, step=10, label="Number of Features", info="Choose between 10 and 100")
    ],
    outputs=[gr.Image(label="LIME Explanation"), gr.Image(label="GradCAM Explanation"), gr.Text(label="Predicted Label"), gr.Text(label="Probability Score")],
    title="Image Classification with LIME & GRAD-CAM explanation",
    description="Upload an image to classify and explain it using SP-LIME and Grad-CAM."
)

if __name__ == "__main__":
    explainer_interface.launch(debug=True, share=True)