Question No 01:

Code: CNN Classification

import os
import numpy as np
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense
import matplotlib.pyplot as plt
from sklearn.metrics import roc_auc_score

data_dir = './Assignment4/Dataset'
img_width, img_height = 64, 64
batch_size = 32
train_samples_per_class = 120
val_samples_per_class = 30

X_train = np.zeros((train_samples_per_class*8, img_width, img_height, 3))

Y_train = np.zeros((train_samples_per_class*8, 8))
X_val = np.zeros((val_samples_per_class*8, img_width, img_height, 3))
Y_val = np.zeros((val_samples_per_class*8, 8))

train_datagen = ImageDataGenerator(rescale=1./255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)

val_datagen = ImageDataGenerator(rescale=1./255)

train_generator = train_datagen.flow_from_directory(data_dir,
target_size=(img_width,
img_height),

batch_size=train_samples_per_class*8,

class_mode='categorical')

val_generator = val_datagen.flow_from_directory(data_dir,
target_size=(img_width,
img_height),

batch_size=val_samples_per_class*8,
class_mode='categorical')

for i, class_folder in enumerate(os.listdir(data_dir)):

class_path = os.path.join(data_dir, class_folder)
train_class_images = train_generator.next()[0]
val_class_images = val_generator.next()[0]
X_train[i*train_samples_per_class:(i+1)*train_samples_per_class] =
train_class_images[:train_samples_per_class]
X_val[i*val_samples_per_class:(i+1)*val_samples_per_class] =
val_class_images[:val_samples_per_class]
Y_train[i*train_samples_per_class:(i+1)*train_samples_per_class, i] = 1
Y_val[i*val_samples_per_class:(i+1)*val_samples_per_class, i] = 1

print("Starting training")
# Define the CNN architecture
model = Sequential()
model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(img_width,
img_height, 3)))
model.add(MaxPooling2D((2, 2)))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D((2, 2)))
model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(MaxPooling2D((2, 2)))
model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(MaxPooling2D((2, 2)))
model.add(Flatten())
model.add(Dense(512, activation='relu'))
model.add(Dense(8, activation='softmax'))

# Compile the model

model.compile(loss='categorical_crossentropy', optimizer='rmsprop',
metrics=['accuracy'])

# Train the model

history = model.fit(X_train, Y_train, epochs=30, batch_size=batch_size,
validation_data=(X_val, Y_val))

# Evaluate the model on the validation data

score = model.evaluate(X_val, Y_val, verbose=0)

# Plot the training and validation accuracy over time

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
epochs = range(1, len(acc) + 1)
plt.plot(epochs, acc, 'bo', label='Training accuracy')
plt.plot(epochs, val_acc, 'b', label='Validation accuracy')
plt.title('Training and validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
plt.show()

# Plot the training and validation loss over time

loss = history.history['loss']
val_loss = history.history['val_loss']
plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.show()

# Predict the probabilities for the validation data

y_pred = model.predict(X_val)

# Calculate the AUC for each class

auc_scores = []
for i in range(8):
auc_score = roc_auc_score(Y_val[:, i], y_pred[:, i])
auc_scores.append(auc_score)

# Plot the AUC scores

labels = ['Class 0', 'Class 1', 'Class 2', 'Class 3', 'Class 4', 'Class 5',
'Class 6', 'Class 7']
plt.bar(labels, auc_scores)
plt.title('AUC scores')
plt.xlabel('Class')
plt.ylabel('AUC')
plt.show()
 Table 1: CNN Architecture

Layer (type) Filter Size Input Shape Output Shape Param #

Conv2D (relu) 3x3x32 64X64X3 62X62X32 896
MaxPooling2D 2x2 62X62X32 31X31X32 0
Conv2D (relu) 3x3x64 31X31X32 29X29X64 18496
MaxPooling2D 2x2 29X29X64 14X14X64 0
Conv2D (relu) 3x3x128 14X14X64 12X12X128 73856
MaxPooling2D 2x2 12X12X128 6X6X128 0
Conv2D (relu) 3x3x128 6X6X128 4X4X128 147584
MaxPooling2D 2x2 4X4X128 2X2X128 0
Flatten 2X2X128 512 0
Dense 512 512 262656
Softmax 512 8 4104

Output:
Question No 02:
Code:
import numpy as np
import cv2
import os
import tensorflow as tf
from skimage.feature import local_binary_pattern
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.multiclass import OneVsOneClassifier, OneVsRestClassifier
from sklearn.preprocessing import LabelEncoder
from keras.applications import MobileNet as SqueezeNet
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.metrics import roc_auc_score
from google.colab import drive
drive.mount('/content/drive')

# Set the input image size

input_size = (64,64)

# Load the pre-trained InceptionV3 model

modelGoogle = tf.keras.applications.InceptionV3(input_shape=(75, 75,
3),
                                          include_top=False,
                                          weights='imagenet')
# Load the pre-trained SqueezeNet model
modelSqueeze = SqueezeNet(input_shape=(input_size[0], input_size[1],
3),
                                        include_top=False,
                                        weights='imagenet')

def extract_features(img,modelname):
    if modelname=='googlenet':
      # Resize the image to the minimum required size of 75x75
      img = cv2.resize(img, (75, 75))
      # Preprocess the image
      img = tf.keras.applications.inception_v3.preprocess_input(img)
      # Extract deep features using the pre-trained GoogleNet model
      deep_features = modelGoogle.predict(np.expand_dims(img,
axis=0)).flatten()
    elif modelname == 'squeezenet':
      # Resize the image to the input size
      img = cv2.resize(img, input_size)
      # Preprocess the image
      img = tf.keras.applications.mobilenet.preprocess_input(img)
      # Extract deep features using the pre-trained SqueezeNet model
      deep_features = modelSqueeze.predict(np.expand_dims(img,
axis=0)).flatten()
    elif modelname == 'googleandsqueeze':
      # Resize the image to the minimum required size of 75x75
      imgGoogle = cv2.resize(img, (75, 75))
      # Preprocess the image
      imgGoogle =
tf.keras.applications.inception_v3.preprocess_input(imgGoogle)
      # Extract deep features using the pre-trained GoogleNet model
      deep_features_google =
modelGoogle.predict(np.expand_dims(imgGoogle, axis=0)).flatten()
      # Resize the image to the input size
      imgSqueeze = cv2.resize(img, input_size)
      # Preprocess the image
      imgSqueeze =
tf.keras.applications.mobilenet.preprocess_input(imgSqueeze)
      # Extract deep features using the pre-trained model
      deep_features_squeeze =
modelSqueeze.predict(np.expand_dims(imgSqueeze, axis=0)).flatten()
          # Concatenate the deep features and LBP features
      deep_features = np.concatenate((deep_features_google,
deep_features_squeeze))

    # Compute LBP features

    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    lbp = local_binary_pattern(gray, 8, 1, method='uniform')
    lbp_hist, _ = np.histogram(lbp.ravel(), bins=np.arange(0, 257),
range=(0, 256))

    # Concatenate the deep features and LBP features

    features = np.concatenate((deep_features, lbp_hist))

    return features

# Define a function to extract features from a set of images

def extract_dataset_features(dataset_path,modelname):
    # Get the list of image files in the dataset directory
    files = [os.path.join(dataset_path, folder, file) for folder in
os.listdir(dataset_path) for file in
os.listdir(os.path.join(dataset_path, folder))]

    # Create an empty list to store the features and labels

    features = []
    labels = []

    # Loop over the image files

    for file in files:
        # Load the image
        img = cv2.imread(file)

        # Extract the label from the directory name

        label = os.path.basename(os.path.dirname(file))

        # Extract features from the image

        features.append(extract_features(img,modelname))

        # Add the label to the list of labels

        labels.append(label)

    # Encode the labels as integers

    le = LabelEncoder()
    labels = le.fit_transform(labels)

    return np.array(features), np.array(labels)

# Define the path to the dataset

dataset_path = '/content/drive/MyDrive/CV_Assignment4/Dataset'

# Extract features of googlenet and LBP from the dataset

featuresGoogleandLBP, labels =
extract_dataset_features(dataset_path,"googlenet")

# Extract features of squeezenet and LBP from the dataset

featuresSqueezeandLBP, labels =
extract_dataset_features(dataset_path,"squeezenet")

# Extract features of squeezenet and LBP from the dataset

featuresGoogleandSqueezeandLBP, labels =
extract_dataset_features(dataset_path,"googleandsqueeze")

# Split the dataset into training and testing sets for googlenet and
LBP
train_features_googleandLBP, test_features_googleandLBP,
train_labels_googleandLBP, test_labels_googleandLBP =
train_test_split(featuresGoogleandLBP, labels, test_size=0.3,
random_state=42)

# Split the dataset into training and testing sets for squeezenet and
LBP
train_features_squeezeandLBP, test_features_squeezeandLBP,
train_labels_squeezeandLBP, test_labels_squeezeandLBP =
train_test_split(featuresSqueezeandLBP, labels, test_size=0.3,
random_state=42)

# Split the dataset into training and testing sets for googlenet and
squeezenet and LBP
train_features_googleandsqueezeandLBP,
test_features_googleandsqueezeandLBP,
train_labels_googleandsqueezeandLBP, test_labels_googleandsqueezeandLBP
= train_test_split(featuresGoogleandSqueezeandLBP, labels,
test_size=0.3, random_state=42)

# Train an SVM classifier using one-vs-one approach for Google and LBP
svm_ovo = OneVsOneClassifier(SVC(kernel='linear', random_state=42))
svm_ovo.fit(train_features_googleandLBP, train_labels_googleandLBP)

# Test the classifier on the testing set for Google and LBP
predovogoogleandLBP = svm_ovo.predict(test_features_googleandLBP)
accovogoogleandLBP = accuracy_score(test_labels_googleandLBP,
predovogoogleandLBP)
print('Accuracy (one-vs-one) for Googlenet and LBP:',
accovogoogleandLBP)

# Train an SVM classifier using one-vs-all approach for Google and LBP
svm_ova = OneVsRestClassifier(SVC(kernel='linear', random_state=42))
svm_ova.fit(train_features_googleandLBP, train_labels_googleandLBP)

# Test the classifier on the testing set for Google and LBP
predovagoogleandLBP = svm_ova.predict(test_features_googleandLBP)
accovagoogleandLBP = accuracy_score(test_labels_googleandLBP,
predovagoogleandLBP)
print('Accuracy (one-vs-all) for Googlenet and LBP:',
accovagoogleandLBP)

# Train an SVM classifier using one-vs-one approach for Squeeze and LBP
svm_ovo = OneVsOneClassifier(SVC(kernel='linear', random_state=42))
svm_ovo.fit(train_features_squeezeandLBP, train_labels_squeezeandLBP)

# Test the classifier on the testing set for Squeeze and LBP
predovosqueezeandLBP = svm_ovo.predict(test_features_squeezeandLBP)
accovosqueezeandLBP = accuracy_score(test_labels_squeezeandLBP,
predovosqueezeandLBP)
print('Accuracy (one-vs-one) for SqueezeNet and LBP:',
accovosqueezeandLBP)

# Train an SVM classifier using one-vs-all approach for Squeeze and LBP
svm_ova = OneVsRestClassifier(SVC(kernel='linear', random_state=42))
svm_ova.fit(train_features_squeezeandLBP, train_labels_squeezeandLBP)

# Test the classifier on the testing set for Squeeze and LBP
predovasqueezeandLBP = svm_ova.predict(test_features_squeezeandLBP)
accovasqueezeandLBP = accuracy_score(test_labels_squeezeandLBP,
predovasqueezeandLBP)
print('Accuracy (one-vs-all) for SqueezeNet and LBP:',
accovasqueezeandLBP)

# Train an SVM classifier using one-vs-one approach for Google and

Squeeze and LBP
svm_ovo = OneVsOneClassifier(SVC(kernel='linear', random_state=42))
svm_ovo.fit(train_features_googleandsqueezeandLBP,
train_labels_googleandsqueezeandLBP)

# Test the classifier on the testing set for Google and Squeeze and LBP
predovogoogleandsqueezeandLBP =
svm_ovo.predict(test_features_googleandsqueezeandLBP)
accovogoogleandsqueezeandLBP =
accuracy_score(test_labels_googleandsqueezeandLBP,
predovogoogleandsqueezeandLBP)
print('Accuracy (one-vs-one) for GoogleNet and SqueezeNet and LBP:',
accovogoogleandsqueezeandLBP)

# Train an SVM classifier using one-vs-all approach for Google and

Squeeze and LBP
svm_ova = OneVsRestClassifier(SVC(kernel='linear', random_state=42))
svm_ova.fit(train_features_googleandsqueezeandLBP,
train_labels_googleandsqueezeandLBP)

# Test the classifier on the testing set for Google and Squeeze and LBP
predovagoogleandsqueezeandLBP =
svm_ova.predict(test_features_googleandsqueezeandLBP)
accovagoogleandsqueezeandLBP =
accuracy_score(test_labels_googleandsqueezeandLBP,
predovagoogleandsqueezeandLBP)
print('Accuracy (one-vs-all) for GoogleNet and SqueezeNet and LBP:',
accovagoogleandsqueezeandLBP)
# Define the results as a dictionary
results = {'Model': ['Googlenet+LBP', 'SqueezeNet+LBP',
'Googlenet+SqueezeNet+LBP'],
           'Accuracy (one-vs-one)': [accovogoogleandLBP,
accovosqueezeandLBP, accovogoogleandsqueezeandLBP],
           'Accuracy (one-vs-all)': [accovagoogleandLBP,
accovasqueezeandLBP, accovagoogleandsqueezeandLBP]}

# Convert the dictionary to a Pandas dataframe

df = pd.DataFrame.from_dict(results)

# Create a bar chart to show the accuracy results

ax = df.plot(kind='bar', x='Model', y=['Accuracy (one-vs-one)',
'Accuracy (one-vs-all)'],
             rot=0, title='Accuracy results')
# Set the legend and axis labels
ax.legend(['One-vs-One', 'One-vs-All'])
ax.set_xlabel('Model')
ax.set_ylabel('Accuracy')

Table 2: CNN Architecture for GoogleNet used in code

Layer Layer Type Filter Input Output Activation Number of
Number Size/Stride Shape Shape Function Parameters
1 Convolutional 1 7x7/2 75x75x3 35x35x64 ReLU 9,688
2 Max Pooling 1 3x3/2 35x35x64 17x17x64 - 0
3 Convolutional 2a 1x1/1 17x17x64 17x17x64 ReLU 4,160
4 Convolutional 2b 3x3/1 17x17x64 15x15x192 ReLU 110,784
5 Max Pooling 2 3x3/2 15x15x192 7x7x192 - 0
6 Inception 3a - 7x7x192 7x7x256 ReLU 68,928
7 Inception 3b - 7x7x256 7x7x480 ReLU 210,624
8 Max Pooling 3 3x3/2 7x7x480 3x3x480 - 0
9 Inception 4a - 3x3x480 3x3x512 ReLU 259,584
10 Inception 4b - 3x3x512 3x3x512 ReLU 380,736
11 Average Pooling 4 Global 3x3x512 1x1x512 - 0
12 Dropout - 1x1x512 1x1x512 - 0
13 Fully Connected - 1x1x512 1x1x256 ReLU 131,328
14 Dropout - 1x1x256 1x1x256 - 0
15 Fully Connected - 1x1x256 1x1x128 ReLU 32,896
16 Average Pooling Global 1x1x128 1x1x128 - 0
17 Dropout - 1x1x128 1x1x128 - 0

Table 3: CNN Architecture for the SqueezeNet used in the code

Layer Layer Type Filter Size Input Shape Output Activation # Parameters
Shape Function Trained
1 Convolution 3x3 64x64x3 64x64x32 ReLU 864
2 Depthwise Convolution 3x3 64x64x32 64x64x32 ReLU 288
3 Pointwise Convolution 1x1 64x64x32 64x64x64 ReLU 2,048
4 Depthwise Convolution 3x3 64x64x64 32x32x64 ReLU 576
5 Pointwise Convolution 1x1 32x32x64 32x32x128 ReLU 8,192
6 Depthwise Convolution 3x3 32x32x128 32x32x128 ReLU 1,152
7 Pointwise Convolution 1x1 32x32x128 32x32x128 ReLU 16,384
8 Depthwise Convolution 3x3 32x32x128 16x16x128 ReLU 1,152
9 Pointwise Convolution 1x1 16x16x128 16x16x256 ReLU 32,768
10 Depthwise Convolution 3x3 16x16x256 16x16x256 ReLU 2,304
11 Pointwise Convolution 1x1 16x16x256 16x16x256 ReLU 65,536
12 Depthwise Convolution 3x3 16x16x256 8x8x256 ReLU 2,304
13 Pointwise Convolution 1x1 8x8x256 8x8x512 ReLU 131,072
14 Depthwise Convolution 3x3 8x8x512 8x8x512 ReLU 4,608
15 Pointwise Convolution 1x1 8x8x512 8x8x512 ReLU 262,144
16 Depthwise Convolution 3x3 8x8x512 4x4x512 ReLU 4,608
17 Pointwise Convolution 1x1 4x4x512 4x4x1024 ReLU 524,288
18 Depthwise Convolution 3x3 4x4x1024 4x4x1024 ReLU 9,216
19 Pointwise Convolution 1x1 4x4x1024 4x4x1024 ReLU 1,048,576
20 Average Pooling Global 4x4x1024 1x1x1024 None 0
21 Dropout - 1x1x1024 1x1x1024 - 0
Output: