Spaces:
Running
Running
File size: 6,240 Bytes
15cafec |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 |
# reference https://stackoverflow.com/questions/69134379/how-to-make-prediction-based-on-model-tensorflow-lite
import numpy as np
import tensorflow as tf
import streamlit as st
from PIL import Image
import io
import matplotlib.pyplot as plt
import keras.backend as K # F1 score metric custom object
import cv2 # Activation heatmap
def predict(image): # to predict raw image input
interpreter = tf.lite.Interpreter('ENet_model.tflite')
interpreter.allocate_tensors()
#get input and output tensors
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
# Read the image and decode to a tensor
img = Image.open(io.BytesIO(image.read()))
img = img.convert('RGB')
# Resize the image to the desired size
img = img.resize((160,160))
img = tf.keras.preprocessing.image.img_to_array(img)
#Preprocess the image to required size and cast
#input_shape = input_details[0]['shape']
input_tensor= np.array(np.expand_dims(img,0), dtype=np.float32)
input_tensor= tf.keras.applications.efficientnet_v2.preprocess_input(input_tensor)
#set the tensor to point to the input data to be inferred
# Invoke the model on the input data
interpreter.set_tensor(input_details[0]['index'], input_tensor)
#Run the inference
interpreter.invoke()
output_details = interpreter.get_tensor(output_details[0]['index'])
return output_details
def orig_img(image):
img = Image.open(io.BytesIO(image.read()))
img = img.convert('RGB')
# Resize the image to the desired size
img = img.resize((160,160))
img = tf.keras.preprocessing.image.img_to_array(img)
#Preprocess the image to required size and cast
#input_shape = input_details[0]['shape']
input_array= np.array(np.expand_dims(img,0), dtype=np.float32)
input_array= tf.keras.applications.efficientnet_v2.preprocess_input(input_array)
input_tensor = tf.convert_to_tensor(input_array) # convert array to tensor
return input_tensor # output tensor format of image
def normalize_image(img): #normalise image
grads_norm = img[:,:,0]+ img[:,:,1]+ img[:,:,2]
grads_norm = (grads_norm - tf.reduce_min(grads_norm))/ (tf.reduce_max(grads_norm)- tf.reduce_min(grads_norm))
return grads_norm
# see this for cmap options: https://matplotlib.org/stable/tutorials/colors/colormaps.html
def plot_maps(img1, img2,vmin=0.3,vmax=0.7, mix_val=2): # saliency map
fig, ax = plt.subplots(figsize=(3.3,3.3))
ax.imshow(img1*mix_val+img2/mix_val, cmap = "terrain" )
plt.axis("off")
fig.savefig("temp_fig.png", transparent=True, frameon=False, bbox_inches='tight', pad_inches = 0)
image = Image.open('temp_fig.png')
st.image(image)
#st.pyplot(fig)
def f1_score(y_true, y_pred): #taken from old keras source code
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
precision = true_positives / (predicted_positives + K.epsilon())
recall = true_positives / (possible_positives + K.epsilon())
f1_val = 2*(precision*recall)/(precision+recall+K.epsilon())
return f1_val
# load full Saved model for Saliency and activation maps, unable to use tf lite model for these unless previously specified upon model construct
model = tf.keras.models.load_model("ENet_ep20_val0.311",
custom_objects={'f1_score': f1_score})
def plot_gradient_maps(input_im): # plot_maps() and predict() function embedded
with tf.GradientTape() as tape:
tape.watch(input_im)
result_img = model(input_im)
max_idx = tf.argmax(result_img,axis = 1)
max_score = tf.math.reduce_max(result_img[0,max_idx[0]]) # tensor max probability
grads = tape.gradient(max_score, input_im)
plot_maps(normalize_image(grads[0]), normalize_image(input_im[0]))
# Activation heatmap
def gradCAM(orig, intensity=0.5, res=270): # function
img = Image.open(io.BytesIO(orig.getvalue()))
img = img.convert('RGB')
# Resize the image to the desired size
img = img.resize((160,160))
x = tf.keras.preprocessing.image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = tf.keras.applications.efficientnet_v2.preprocess_input(x) # shape (1,160,160,3)
with tf.GradientTape() as tape: # Grad-CAM process
last_conv_layer = model.get_layer('top_conv')
iterate = tf.keras.models.Model([model.inputs], [model.output, last_conv_layer.output]) # create mini model function to get model output
model_out, last_conv_layer = iterate(x) # model_out shape (1,4)
class_out = model_out[:, np.argmax(model_out[0])]
grads = tape.gradient(class_out, last_conv_layer)
pooled_grads = K.mean(grads, axis=(0, 1, 2))
heatmap = tf.reduce_mean(tf.multiply(pooled_grads, last_conv_layer), axis=-1)
heatmap = np.maximum(heatmap, 0)
heatmap /= np.max(heatmap) # minmax pixel values (0,1)
heatmap = heatmap.reshape((5, 5)) # reshape to 5x5 array
# img = cv2.imread(orig) # numpy array
img = Image.open(io.BytesIO(orig.getvalue()))
img = img.convert('RGB')
# Resize the image to the desired size
img = img.resize((160,160))
img = tf.keras.preprocessing.image.img_to_array(img)
heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))
heatmap = cv2.applyColorMap(np.uint8(255*heatmap), cv2.COLORMAP_JET) # multiply 255 to convert to RGB form
img = heatmap * intensity + img
img1 = cv2.resize(img, (res, res)) # visualise heatmap overlay
cv2.imwrite('temporary.jpg', img1) # store image as a temporary file for st.image to interpret, unable to direct load from st.image(img1)
st.image('temporary.jpg')
|