Importing Libraries¶

In [1]:
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier, StackingClassifier, BaggingClassifier, VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
from sklearn.preprocessing import LabelEncoder
import warnings
warnings.filterwarnings('ignore')

C:\Users\deban\anaconda3\Lib\site-packages\pandas\core\arrays\masked.py:60: UserWarning: Pandas requires version '1.3.6' or newer of 'bottleneck' (version '1.3.5' currently installed).
  from pandas.core import (

Loading the Dataset¶

In [2]:
df = pd.read_csv('predictive_maintenance.csv')

Data Exploration¶

In [3]:
# First Few Rows
df.head()
Out[3]:
UDI Product ID Type Air temperature [K] Process temperature [K] Rotational speed [rpm] Torque [Nm] Tool wear [min] Target Failure Type
0 1 M14860 M 298.1 308.6 1551 42.8 0 0 No Failure
1 2 L47181 L 298.2 308.7 1408 46.3 3 0 No Failure
2 3 L47182 L 298.1 308.5 1498 49.4 5 0 No Failure
3 4 L47183 L 298.2 308.6 1433 39.5 7 0 No Failure
4 5 L47184 L 298.2 308.7 1408 40.0 9 0 No Failure
In [4]:
# Last Few Rows
df.tail()
Out[4]:
UDI Product ID Type Air temperature [K] Process temperature [K] Rotational speed [rpm] Torque [Nm] Tool wear [min] Target Failure Type
9995 9996 M24855 M 298.8 308.4 1604 29.5 14 0 No Failure
9996 9997 H39410 H 298.9 308.4 1632 31.8 17 0 No Failure
9997 9998 M24857 M 299.0 308.6 1645 33.4 22 0 No Failure
9998 9999 H39412 H 299.0 308.7 1408 48.5 25 0 No Failure
9999 10000 M24859 M 299.0 308.7 1500 40.2 30 0 No Failure
In [5]:
# Shape of the data
df.shape
Out[5]:
(10000, 10)
In [6]:
# Columns of the dataset
df.columns
Out[6]:
Index(['UDI', 'Product ID', 'Type', 'Air temperature [K]',
       'Process temperature [K]', 'Rotational speed [rpm]', 'Torque [Nm]',
       'Tool wear [min]', 'Target', 'Failure Type'],
      dtype='object')
In [7]:
# Data Types
df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 10000 entries, 0 to 9999
Data columns (total 10 columns):
 #   Column                   Non-Null Count  Dtype  
---  ------                   --------------  -----  
 0   UDI                      10000 non-null  int64  
 1   Product ID               10000 non-null  object 
 2   Type                     10000 non-null  object 
 3   Air temperature [K]      10000 non-null  float64
 4   Process temperature [K]  10000 non-null  float64
 5   Rotational speed [rpm]   10000 non-null  int64  
 6   Torque [Nm]              10000 non-null  float64
 7   Tool wear [min]          10000 non-null  int64  
 8   Target                   10000 non-null  int64  
 9   Failure Type             10000 non-null  object 
dtypes: float64(3), int64(4), object(3)
memory usage: 781.4+ KB
In [8]:
# Statistical Analysis
df.describe()
Out[8]:
UDI Air temperature [K] Process temperature [K] Rotational speed [rpm] Torque [Nm] Tool wear [min] Target
count 10000.00000 10000.000000 10000.000000 10000.000000 10000.000000 10000.000000 10000.000000
mean 5000.50000 300.004930 310.005560 1538.776100 39.986910 107.951000 0.033900
std 2886.89568 2.000259 1.483734 179.284096 9.968934 63.654147 0.180981
min 1.00000 295.300000 305.700000 1168.000000 3.800000 0.000000 0.000000
25% 2500.75000 298.300000 308.800000 1423.000000 33.200000 53.000000 0.000000
50% 5000.50000 300.100000 310.100000 1503.000000 40.100000 108.000000 0.000000
75% 7500.25000 301.500000 311.100000 1612.000000 46.800000 162.000000 0.000000
max 10000.00000 304.500000 313.800000 2886.000000 76.600000 253.000000 1.000000

Data Cleaning¶

In [9]:
# Drop Irrelevant Columns
df = df.drop(columns=["UDI", "Product ID"])
In [10]:
# Convert temperature from K to °C to be more understandable
df['Air temperature [K]'] = df['Air temperature [K]'] - 272.15
df['Process temperature [K]'] = df['Process temperature [K]'] - 272.15

df = df.rename(columns={
    'Air temperature [K]': 'Air temperature [C]',
    'Process temperature [K]': 'Process temperature [C]'
})

# create temperator diffrences between Air and Process
df['Diff temperatur [C]'] = df['Process temperature [C]'] - df['Air temperature [C]']
In [11]:
# Checking Null Values
df.isnull().sum()
Out[11]:
Type                       0
Air temperature [C]        0
Process temperature [C]    0
Rotational speed [rpm]     0
Torque [Nm]                0
Tool wear [min]            0
Target                     0
Failure Type               0
Diff temperatur [C]        0
dtype: int64
In [12]:
# Checking Duplicate Values
duplicate_rows = df[df.duplicated()]
num_duplicates = duplicate_rows.shape[0]

print(f"Number of duplicate rows: {num_duplicates}")

Number of duplicate rows: 0

Data Visualisations¶

In [13]:
# Distribution of Air and Process Temperature
plt.figure(figsize=(14, 6))

plt.subplot(1, 2, 1)
sns.histplot(df['Air temperature [C]'], kde=True, bins=30, color='blue')
plt.title('Distribution of Air Temperature [C]')

plt.subplot(1, 2, 2)
sns.histplot(df['Process temperature [C]'], kde=True, bins=30, color='orange')
plt.title('Distribution of Process Temperature [C]')

plt.tight_layout()
plt.show()
In [14]:
# Relationship Between Rotational Speed and Torque
plt.figure(figsize=(10, 6))
sns.scatterplot(data=df, x='Rotational speed [rpm]', y='Torque [Nm]', hue='Target')
plt.title('Relationship Between Rotational Speed and Torque')
plt.show()
In [15]:
# Tool Wear Distribution by Failure Type
plt.figure(figsize=(12, 6))
sns.boxplot(data=df, x='Failure Type', y='Tool wear [min]', palette='flare')
plt.title('Tool Wear Distribution by Failure Type')
plt.show()
In [16]:
# Failure Type Distribution
plt.figure(figsize=(10, 6))
sns.countplot(data=df, x='Failure Type', palette='flare')

# Adding counts on top of bars
for i, bar in enumerate(df['Failure Type'].value_counts()):
    plt.text(i, bar + 5, bar, ha='center', va='bottom', fontsize=10)

plt.title('Failure Type Distribution')
plt.xticks(rotation=45)
plt.show()
In [17]:
# Temperature Difference vs. Target
plt.figure(figsize=(10, 6))
sns.boxplot(data=df, x='Target', y='Diff temperatur [C]', palette='flare')
plt.title('Temperature Difference vs. Target')
plt.show()
In [18]:
# Calculate counts of each Failure Type
failure_type_counts = df['Type'].value_counts()

# Plotting the pie chart
plt.figure(figsize=(8, 8))
plt.pie(failure_type_counts, labels=failure_type_counts.index, autopct='%1.1f%%', startangle=140, colors=sns.color_palette('flare'), wedgeprops={'edgecolor': 'black'})
plt.title('Type Distribution')
plt.axis('equal')
plt.show()
In [19]:
from sklearn.preprocessing import LabelEncoder

# Initialize LabelEncoder
label_encoder = LabelEncoder()

# Encode 'Type' and 'Failure Type' columns
df['Type'] = label_encoder.fit_transform(df['Type'])
df['Failure Type'] = label_encoder.fit_transform(df['Failure Type'])

# Calculate the correlation matrix
corr_matrix = df.corr()

# Plotting the heatmap
plt.figure(figsize=(12, 8))
sns.heatmap(corr_matrix, annot=True, cmap='flare', fmt='.2f', annot_kws={'size': 10})
plt.title('Correlation Heatmap')
plt.show()

Data Splitting¶

In [20]:
from sklearn.model_selection import train_test_split

# Separate features (X) and target variable (y)
X = df[['Type', 'Air temperature [C]', 'Process temperature [C]', 'Rotational speed [rpm]', 'Torque [Nm]', 'Tool wear [min]', 'Diff temperatur [C]']]
y = df['Failure Type']  # Target variable
In [21]:
# Split the data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Print the shapes of the split datasets
print("Shape of X_train:", X_train.shape)
print("Shape of X_test:", X_test.shape)
print("Shape of y_train:", y_train.shape)
print("Shape of y_test:", y_test.shape)

Shape of X_train: (8000, 7)
Shape of X_test: (2000, 7)
Shape of y_train: (8000,)
Shape of y_test: (2000,)

1. Stacking Classifier¶

1.1 Stacking Classifier with Random Forest Classifier

In [22]:
# Initialize the Stacking Classifier with Random Forest Classifier
base_estimators = [
    ('logreg', LogisticRegression(random_state=42)),
    ('svm', SVC(kernel='rbf', C=1, gamma='scale', random_state=42, probability=True)),
    ('knn', KNeighborsClassifier(n_neighbors=5))
]
stacking_rf_classifier = StackingClassifier(estimators=base_estimators, final_estimator=RandomForestClassifier(random_state=42))
In [23]:
# Train the model
stacking_rf_classifier.fit(X_train, y_train)
Out[23]:
StackingClassifier(estimators=[('logreg', LogisticRegression(random_state=42)),
                               ('svm',
                                SVC(C=1, probability=True, random_state=42)),
                               ('knn', KNeighborsClassifier())],
                   final_estimator=RandomForestClassifier(random_state=42))
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
StackingClassifier(estimators=[('logreg', LogisticRegression(random_state=42)),
                               ('svm',
                                SVC(C=1, probability=True, random_state=42)),
                               ('knn', KNeighborsClassifier())],
                   final_estimator=RandomForestClassifier(random_state=42))
LogisticRegression(random_state=42)
SVC(C=1, probability=True, random_state=42)
KNeighborsClassifier()
RandomForestClassifier(random_state=42)
In [24]:
# Predictions on the test set
y_pred_stacking_rf = stacking_rf_classifier.predict(X_test)
In [25]:
# Calculate Accuracy
accuracy_stacking_rf = accuracy_score(y_test, y_pred_stacking_rf)
print(f"Accuracy of Stacking Classifier with Random Forest: {accuracy_stacking_rf*100:.2f}%")

Accuracy of Stacking Classifier with Random Forest: 97.05%
In [26]:
# Classification Report
print("Classification Report for Stacking Classifier with Random Forest:")
print(classification_report(y_test, y_pred_stacking_rf))

Classification Report for Stacking Classifier with Random Forest:
              precision    recall  f1-score   support

           0       0.12      0.07      0.09        15
           1       0.98      1.00      0.99      1935
           2       0.73      0.62      0.67        13
           3       0.75      0.30      0.43        20
           4       0.00      0.00      0.00         6
           5       0.00      0.00      0.00        11

    accuracy                           0.97      2000
   macro avg       0.43      0.33      0.36      2000
weighted avg       0.96      0.97      0.96      2000
In [27]:
# Confusion Matrix
conf_matrix_stacking_rf = confusion_matrix(y_test, y_pred_stacking_rf)

# Plotting the confusion matrix
plt.figure(figsize=(8, 6))
sns.heatmap(conf_matrix_stacking_rf, annot=True, cmap='flare', fmt='d', cbar=False)
plt.title('Confusion Matrix - Stacking Classifier with Random Forest')
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.show()

1.2 Stacking classifier with SVM

In [28]:
# Initialize the Stacking Classifier with SVM Classifier
base_estimators = [
    ('rf', RandomForestClassifier(random_state=42)),
    ('logreg', LogisticRegression(random_state=42)),
    ('knn', KNeighborsClassifier(n_neighbors=5))
]

stacking_svm_classifier = StackingClassifier(estimators=base_estimators, final_estimator=SVC(kernel='rbf', C=1, gamma='scale', random_state=42, probability=True))
In [29]:
# Train the model
stacking_svm_classifier.fit(X_train, y_train)
Out[29]:
StackingClassifier(estimators=[('rf', RandomForestClassifier(random_state=42)),
                               ('logreg', LogisticRegression(random_state=42)),
                               ('knn', KNeighborsClassifier())],
                   final_estimator=SVC(C=1, probability=True, random_state=42))
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
StackingClassifier(estimators=[('rf', RandomForestClassifier(random_state=42)),
                               ('logreg', LogisticRegression(random_state=42)),
                               ('knn', KNeighborsClassifier())],
                   final_estimator=SVC(C=1, probability=True, random_state=42))
RandomForestClassifier(random_state=42)
LogisticRegression(random_state=42)
KNeighborsClassifier()
SVC(C=1, probability=True, random_state=42)
In [30]:
# # Predictions on the test set
y_pred_stacking_svm = stacking_svm_classifier.predict(X_test)
In [31]:
# Calculating accuracy
accuracy_stacking_svm = accuracy_score(y_test, y_pred_stacking_svm)
print(f"Accuracy of Stacking Classifier with SVM: {accuracy_stacking_svm*100:.2f}%")

Accuracy of Stacking Classifier with SVM: 98.35%
In [32]:
# Classification report
print("Classification Report for Stacking Classifier with SVM:")
print(classification_report(y_test, y_pred_stacking_svm))

Classification Report for Stacking Classifier with SVM:
              precision    recall  f1-score   support

           0       0.79      1.00      0.88        15
           1       0.99      1.00      0.99      1935
           2       0.75      0.69      0.72        13
           3       0.82      0.70      0.76        20
           4       0.00      0.00      0.00         6
           5       0.00      0.00      0.00        11

    accuracy                           0.98      2000
   macro avg       0.56      0.56      0.56      2000
weighted avg       0.98      0.98      0.98      2000
In [33]:
# Confusion Matrix
conf_matrix_stacking_svm = confusion_matrix(y_test, y_pred_stacking_svm)

plt.figure(figsize=(8, 6))
sns.heatmap(conf_matrix_stacking_svm, annot=True, cmap='flare', fmt='d', cbar=False)
plt.title('Confusion Matrix - Stacking Classifier with SVM')
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.show()

1.3 Stacking classifier with KNN

In [34]:
# Initialize the Stacking Classifier with KNN Classifier
base_estimators = [
    ('rf', RandomForestClassifier(random_state=42)),
    ('logreg', LogisticRegression(random_state=42)),
    ('svm', SVC(kernel='rbf', C=1, gamma='scale', random_state=42, probability=True))
]

stacking_knn_classifier = StackingClassifier(estimators=base_estimators, final_estimator=KNeighborsClassifier(n_neighbors=5))
In [35]:
# Training the model
stacking_knn_classifier.fit(X_train, y_train)
Out[35]:
StackingClassifier(estimators=[('rf', RandomForestClassifier(random_state=42)),
                               ('logreg', LogisticRegression(random_state=42)),
                               ('svm',
                                SVC(C=1, probability=True, random_state=42))],
                   final_estimator=KNeighborsClassifier())
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
StackingClassifier(estimators=[('rf', RandomForestClassifier(random_state=42)),
                               ('logreg', LogisticRegression(random_state=42)),
                               ('svm',
                                SVC(C=1, probability=True, random_state=42))],
                   final_estimator=KNeighborsClassifier())
RandomForestClassifier(random_state=42)
LogisticRegression(random_state=42)
SVC(C=1, probability=True, random_state=42)
KNeighborsClassifier()
In [36]:
# Predictions on test data
y_pred_stacking_knn = stacking_knn_classifier.predict(X_test)
In [37]:
# Calculating Accuracy
accuracy_stacking_knn = accuracy_score(y_test, y_pred_stacking_knn)
print(f"Accuracy of Stacking Classifier with KNN: {accuracy_stacking_knn*100:.2f}%")

Accuracy of Stacking Classifier with KNN: 98.40%
In [38]:
# Classification report
print("Classification Report for Stacking Classifier with KNN:")
print(classification_report(y_test, y_pred_stacking_knn))

Classification Report for Stacking Classifier with KNN:
              precision    recall  f1-score   support

           0       0.88      1.00      0.94        15
           1       0.99      1.00      0.99      1935
           2       0.65      1.00      0.79        13
           3       0.80      0.60      0.69        20
           4       0.00      0.00      0.00         6
           5       0.00      0.00      0.00        11

    accuracy                           0.98      2000
   macro avg       0.55      0.60      0.57      2000
weighted avg       0.98      0.98      0.98      2000
In [39]:
# Confustion Matrix
conf_matrix_stacking_knn = confusion_matrix(y_test, y_pred_stacking_knn)

plt.figure(figsize=(8, 6))
sns.heatmap(conf_matrix_stacking_knn, annot=True, cmap='flare', fmt='d', cbar=False)
plt.title('Confusion Matrix - Stacking Classifier with KNN')
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.show()

1.4 Stacking classifier with Logistic Regression

In [40]:
# Initialize the Stacking Classifier with Logistic Regression
base_estimators = [
    ('rf', RandomForestClassifier(random_state=42)),
    ('svm', SVC(kernel='rbf', C=1, gamma='scale', random_state=42, probability=True)),
    ('knn', KNeighborsClassifier(n_neighbors=5))
]

stacking_logreg_classifier = StackingClassifier(estimators=base_estimators, final_estimator=LogisticRegression(random_state=42))
In [41]:
# Training the model
stacking_logreg_classifier.fit(X_train, y_train)
Out[41]:
StackingClassifier(estimators=[('rf', RandomForestClassifier(random_state=42)),
                               ('svm',
                                SVC(C=1, probability=True, random_state=42)),
                               ('knn', KNeighborsClassifier())],
                   final_estimator=LogisticRegression(random_state=42))
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
StackingClassifier(estimators=[('rf', RandomForestClassifier(random_state=42)),
                               ('svm',
                                SVC(C=1, probability=True, random_state=42)),
                               ('knn', KNeighborsClassifier())],
                   final_estimator=LogisticRegression(random_state=42))
RandomForestClassifier(random_state=42)
SVC(C=1, probability=True, random_state=42)
KNeighborsClassifier()
LogisticRegression(random_state=42)
In [42]:
# Making predictions on the test data
y_pred_stacking_logreg = stacking_logreg_classifier.predict(X_test)
In [43]:
# Calculating accuracy score
accuracy_stacking_logreg = accuracy_score(y_test, y_pred_stacking_logreg)
print(f"Accuracy of Stacking Classifier with Logistic Regression: {accuracy_stacking_logreg*100:.2f}%")

Accuracy of Stacking Classifier with Logistic Regression: 98.45%
In [44]:
# Classification report
print("Classification Report for Stacking Classifier with Logistic Regression:")
print(classification_report(y_test, y_pred_stacking_logreg))

Classification Report for Stacking Classifier with Logistic Regression:
              precision    recall  f1-score   support

           0       0.88      1.00      0.94        15
           1       0.99      1.00      0.99      1935
           2       0.82      0.69      0.75        13
           3       0.81      0.65      0.72        20
           4       0.00      0.00      0.00         6
           5       0.00      0.00      0.00        11

    accuracy                           0.98      2000
   macro avg       0.58      0.56      0.57      2000
weighted avg       0.98      0.98      0.98      2000
In [45]:
# Confusion Matrix
conf_matrix_stacking_logreg = confusion_matrix(y_test, y_pred_stacking_logreg)

plt.figure(figsize=(8, 6))
sns.heatmap(conf_matrix_stacking_logreg, annot=True, cmap='flare', fmt='d', cbar=False)
plt.title('Confusion Matrix - Stacking Classifier with Logistic Regression')
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.show()

2. Bagging Classifier¶

2.1 Bagging Classifier with Random Forest

In [46]:
# Initialize the Bagging Classifier with Random Forest
rf_bagging_classifier = BaggingClassifier(base_estimator=RandomForestClassifier(random_state=42), n_estimators=10, random_state=42)
In [47]:
# Training the model
rf_bagging_classifier.fit(X_train, y_train)
Out[47]:
BaggingClassifier(base_estimator=RandomForestClassifier(random_state=42),
                  random_state=42)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
BaggingClassifier(base_estimator=RandomForestClassifier(random_state=42),
                  random_state=42)
RandomForestClassifier(random_state=42)
RandomForestClassifier(random_state=42)
In [48]:
# Making predictions
y_pred_rf_bagging = rf_bagging_classifier.predict(X_test)
In [49]:
# Accuracy Score
accuracy_rf_bagging = accuracy_score(y_test, y_pred_rf_bagging)
print(f"Accuracy of Bagging Classifier with Random Forest: {accuracy_rf_bagging*100:.2f}%")

Accuracy of Bagging Classifier with Random Forest: 98.10%
In [50]:
# Classification Report
print("Classification Report for Bagging Classifier with Random Forest:")
print(classification_report(y_test, y_pred_rf_bagging))

Classification Report for Bagging Classifier with Random Forest:
              precision    recall  f1-score   support

           0       0.83      1.00      0.91        15
           1       0.98      1.00      0.99      1935
           2       0.67      0.15      0.25        13
           3       0.78      0.70      0.74        20
           4       0.00      0.00      0.00         6
           5       0.00      0.00      0.00        11

    accuracy                           0.98      2000
   macro avg       0.54      0.48      0.48      2000
weighted avg       0.97      0.98      0.97      2000
In [51]:
# Confusion Matrix
conf_matrix_rf_bagging = confusion_matrix(y_test, y_pred_rf_bagging)

plt.figure(figsize=(8, 6))
sns.heatmap(conf_matrix_rf_bagging, annot=True, cmap='flare', fmt='d', cbar=False)
plt.title('Confusion Matrix - Bagging Classifier with Random Forest')
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.show()

2.2 Bagging Classifier with SVM

In [52]:
# Initialize the Bagging Classifier with SVM
svm_bagging_classifier = BaggingClassifier(base_estimator=SVC(kernel='rbf', C=1, gamma='scale', random_state=42, probability=True), n_estimators=10, random_state=42)
In [53]:
# Training the model
svm_bagging_classifier.fit(X_train, y_train)
Out[53]:
BaggingClassifier(base_estimator=SVC(C=1, probability=True, random_state=42),
                  random_state=42)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
BaggingClassifier(base_estimator=SVC(C=1, probability=True, random_state=42),
                  random_state=42)
SVC(C=1, probability=True, random_state=42)
SVC(C=1, probability=True, random_state=42)
In [54]:
# Making predictions
y_pred_svm_bagging = svm_bagging_classifier.predict(X_test)
In [55]:
# Calculating accuracy score
accuracy_svm_bagging = accuracy_score(y_test, y_pred_svm_bagging)
print(f"Accuracy of Bagging Classifier with SVM: {accuracy_svm_bagging*100:.2f}%")

Accuracy of Bagging Classifier with SVM: 97.10%
In [56]:
# Classification report
print("Classification Report for Bagging Classifier with SVM:")
print(classification_report(y_test, y_pred_svm_bagging))

Classification Report for Bagging Classifier with SVM:
              precision    recall  f1-score   support

           0       0.00      0.00      0.00        15
           1       0.97      1.00      0.99      1935
           2       0.50      0.31      0.38        13
           3       0.80      0.20      0.32        20
           4       0.00      0.00      0.00         6
           5       0.00      0.00      0.00        11

    accuracy                           0.97      2000
   macro avg       0.38      0.25      0.28      2000
weighted avg       0.95      0.97      0.96      2000
In [57]:
# Confusion Matrix
conf_matrix_svm_bagging = confusion_matrix(y_test, y_pred_svm_bagging)
plt.figure(figsize=(8, 6))
sns.heatmap(conf_matrix_svm_bagging, annot=True, cmap='flare', fmt='d', cbar=False)
plt.title('Confusion Matrix - Bagging Classifier with SVM')
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.show()

2.3 Bagging Classifier with KNN

In [58]:
# Initialize the Bagging Classifier with KNN
knn_bagging_classifier = BaggingClassifier(base_estimator=KNeighborsClassifier(n_neighbors=5), n_estimators=10, random_state=42)
In [59]:
# Training the model
knn_bagging_classifier.fit(X_train, y_train)
Out[59]:
BaggingClassifier(base_estimator=KNeighborsClassifier(), random_state=42)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
BaggingClassifier(base_estimator=KNeighborsClassifier(), random_state=42)
KNeighborsClassifier()
KNeighborsClassifier()
In [60]:
# Making predictions
y_pred_knn_bagging = knn_bagging_classifier.predict(X_test)
accuracy_knn_bagging = accuracy_score(y_test, y_pred_knn_bagging)
print(f"Accuracy of Bagging Classifier with KNN: {accuracy_knn_bagging*100:.2f}%")

Accuracy of Bagging Classifier with KNN: 96.75%
In [61]:
# Classification report
print("Classification Report for Bagging Classifier with KNN:")
print(classification_report(y_test, y_pred_knn_bagging))

Classification Report for Bagging Classifier with KNN:
              precision    recall  f1-score   support

           0       0.00      0.00      0.00        15
           1       0.97      1.00      0.98      1935
           2       0.50      0.15      0.24        13
           3       0.75      0.15      0.25        20
           4       0.00      0.00      0.00         6
           5       0.00      0.00      0.00        11

    accuracy                           0.97      2000
   macro avg       0.37      0.22      0.24      2000
weighted avg       0.95      0.97      0.96      2000
In [62]:
# Confusion Matrix
conf_matrix_knn_bagging = confusion_matrix(y_test, y_pred_knn_bagging)
plt.figure(figsize=(8, 6))
sns.heatmap(conf_matrix_knn_bagging, annot=True, cmap='flare', fmt='d', cbar=False)
plt.title('Confusion Matrix - Bagging Classifier with KNN')
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.show()

2.4 Bagging Classifier with Logistic Regression

In [63]:
# Initialize the Bagging Classifier with Logistic Regression
logreg_bagging_classifier = BaggingClassifier(base_estimator=LogisticRegression(random_state=42, max_iter=1000), n_estimators=10, random_state=42)
In [64]:
# Training the model
logreg_bagging_classifier.fit(X_train, y_train)
Out[64]:
BaggingClassifier(base_estimator=LogisticRegression(max_iter=1000,
                                                    random_state=42),
                  random_state=42)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
BaggingClassifier(base_estimator=LogisticRegression(max_iter=1000,
                                                    random_state=42),
                  random_state=42)
LogisticRegression(max_iter=1000, random_state=42)
LogisticRegression(max_iter=1000, random_state=42)
In [65]:
# Making prediction
y_pred_logreg_bagging = logreg_bagging_classifier.predict(X_test)
In [66]:
# Calculating accuracy score
accuracy_logreg_bagging = accuracy_score(y_test, y_pred_logreg_bagging)
print(f"Accuracy of Bagging Classifier with Logistic Regression: {accuracy_logreg_bagging*100:.2f}%")

Accuracy of Bagging Classifier with Logistic Regression: 97.40%
In [67]:
# Classification report
print("Classification Report for Bagging Classifier with Logistic Regression:")
print(classification_report(y_test, y_pred_logreg_bagging))

Classification Report for Bagging Classifier with Logistic Regression:
              precision    recall  f1-score   support

           0       0.43      0.20      0.27        15
           1       0.98      1.00      0.99      1935
           2       0.56      0.38      0.45        13
           3       0.82      0.70      0.76        20
           4       0.00      0.00      0.00         6
           5       0.00      0.00      0.00        11

    accuracy                           0.97      2000
   macro avg       0.46      0.38      0.41      2000
weighted avg       0.96      0.97      0.97      2000
In [68]:
# Confusion Matrix
conf_matrix_logreg_bagging = confusion_matrix(y_test, y_pred_logreg_bagging)
plt.figure(figsize=(8, 6))
sns.heatmap(conf_matrix_logreg_bagging, annot=True, cmap='flare', fmt='d', cbar=False)
plt.title('Confusion Matrix - Bagging Classifier with Logistic Regression')
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.show()

3. Voting Classifier¶

3.1 Voting Classifier with Random Forest

In [69]:
# Initialize the Voting Classifier with Random Forest
voting_rf_classifier = VotingClassifier(
    estimators=[
        ('logreg', LogisticRegression(random_state=42)),
        ('svm', SVC(kernel='rbf', C=1, gamma='scale', random_state=42, probability=True)),
        ('knn', KNeighborsClassifier(n_neighbors=5))
    ],
    voting='soft'
)
In [70]:
# Training the model
voting_rf_classifier.fit(X_train, y_train)
Out[70]:
VotingClassifier(estimators=[('logreg', LogisticRegression(random_state=42)),
                             ('svm',
                              SVC(C=1, probability=True, random_state=42)),
                             ('knn', KNeighborsClassifier())],
                 voting='soft')
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
VotingClassifier(estimators=[('logreg', LogisticRegression(random_state=42)),
                             ('svm',
                              SVC(C=1, probability=True, random_state=42)),
                             ('knn', KNeighborsClassifier())],
                 voting='soft')
LogisticRegression(random_state=42)
SVC(C=1, probability=True, random_state=42)
KNeighborsClassifier()
In [71]:
# Making prediction
y_pred_voting_rf = voting_rf_classifier.predict(X_test)
In [72]:
# Calculating accuracy
accuracy_voting_rf = accuracy_score(y_test, y_pred_voting_rf)
print(f"Accuracy of Voting Classifier with Random Forest Base Estimator: {accuracy_voting_rf*100:.2f}%")

Accuracy of Voting Classifier with Random Forest Base Estimator: 96.95%
In [73]:
# Classification report
print("Classification Report for Voting Classifier with Random Forest Base Estimator:")
print(classification_report(y_test, y_pred_voting_rf))

Classification Report for Voting Classifier with Random Forest Base Estimator:
              precision    recall  f1-score   support

           0       0.00      0.00      0.00        15
           1       0.97      1.00      0.99      1935
           2       0.43      0.23      0.30        13
           3       0.75      0.15      0.25        20
           4       0.00      0.00      0.00         6
           5       0.00      0.00      0.00        11

    accuracy                           0.97      2000
   macro avg       0.36      0.23      0.26      2000
weighted avg       0.95      0.97      0.96      2000
In [74]:
# Confusion Matrix
conf_matrix_voting_rf = confusion_matrix(y_test, y_pred_voting_rf)
plt.figure(figsize=(8, 6))
sns.heatmap(conf_matrix_voting_rf, annot=True, cmap='flare', fmt='d', cbar=False)
plt.title('Confusion Matrix - Voting Classifier with Random Forest Base Estimator')
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.show()

3.2 Voting Classifier with SVM

In [75]:
# Initialize the Voting Classifier with SVM
voting_svm_classifier = VotingClassifier(
    estimators=[
        ('rf', RandomForestClassifier(random_state=42)),
        ('logreg', LogisticRegression(random_state=42)),
        ('knn', KNeighborsClassifier(n_neighbors=5))
    ],
    voting='soft'
)
In [76]:
# Training the model
voting_svm_classifier.fit(X_train, y_train)
Out[76]:
VotingClassifier(estimators=[('rf', RandomForestClassifier(random_state=42)),
                             ('logreg', LogisticRegression(random_state=42)),
                             ('knn', KNeighborsClassifier())],
                 voting='soft')
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
VotingClassifier(estimators=[('rf', RandomForestClassifier(random_state=42)),
                             ('logreg', LogisticRegression(random_state=42)),
                             ('knn', KNeighborsClassifier())],
                 voting='soft')
RandomForestClassifier(random_state=42)
LogisticRegression(random_state=42)
KNeighborsClassifier()
In [77]:
# Making Predictions
y_pred_voting_svm = voting_svm_classifier.predict(X_test)
In [78]:
# Calculating accuracies
accuracy_voting_svm = accuracy_score(y_test, y_pred_voting_svm)
print(f"Accuracy of Voting Classifier with SVM Base Estimator: {accuracy_voting_svm*100:.2f}%")

Accuracy of Voting Classifier with SVM Base Estimator: 97.05%
In [79]:
# Classification report
print("Classification Report for Voting Classifier with SVM Base Estimator:")
print(classification_report(y_test, y_pred_voting_svm))

Classification Report for Voting Classifier with SVM Base Estimator:
              precision    recall  f1-score   support

           0       0.00      0.00      0.00        15
           1       0.97      1.00      0.99      1935
           2       0.57      0.31      0.40        13
           3       0.75      0.15      0.25        20
           4       0.00      0.00      0.00         6
           5       0.00      0.00      0.00        11

    accuracy                           0.97      2000
   macro avg       0.38      0.24      0.27      2000
weighted avg       0.95      0.97      0.96      2000
In [80]:
# Confusion Matrix
conf_matrix_voting_svm = confusion_matrix(y_test, y_pred_voting_svm)
plt.figure(figsize=(8, 6))
sns.heatmap(conf_matrix_voting_svm, annot=True, cmap='flare', fmt='d', cbar=False)
plt.title('Confusion Matrix - Voting Classifier with SVM Base Estimator')
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.show()

3.3 Voting Classifier with KNN

In [81]:
# Initialize the Voting Classifier with KNN
voting_knn_classifier = VotingClassifier(
    estimators=[
        ('rf', RandomForestClassifier(random_state=42)),
        ('logreg', LogisticRegression(random_state=42)),
        ('svm', SVC(kernel='rbf', C=1, gamma='scale', random_state=42, probability=True))
    ],
    voting='soft'
)
In [82]:
# Training the model
voting_knn_classifier.fit(X_train, y_train)
Out[82]:
VotingClassifier(estimators=[('rf', RandomForestClassifier(random_state=42)),
                             ('logreg', LogisticRegression(random_state=42)),
                             ('svm',
                              SVC(C=1, probability=True, random_state=42))],
                 voting='soft')
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
VotingClassifier(estimators=[('rf', RandomForestClassifier(random_state=42)),
                             ('logreg', LogisticRegression(random_state=42)),
                             ('svm',
                              SVC(C=1, probability=True, random_state=42))],
                 voting='soft')
RandomForestClassifier(random_state=42)
LogisticRegression(random_state=42)
SVC(C=1, probability=True, random_state=42)
In [83]:
# Making predictions
y_pred_voting_knn = voting_knn_classifier.predict(X_test)
In [84]:
# Calculating accuracy score
accuracy_voting_knn = accuracy_score(y_test, y_pred_voting_knn)
print(f"Accuracy of Voting Classifier with KNN Base Estimator: {accuracy_voting_knn*100:.2f}%")

Accuracy of Voting Classifier with KNN Base Estimator: 97.05%
In [85]:
# Classification report
print("Classification Report for Voting Classifier with KNN Base Estimator:")
print(classification_report(y_test, y_pred_voting_knn))

Classification Report for Voting Classifier with KNN Base Estimator:
              precision    recall  f1-score   support

           0       0.00      0.00      0.00        15
           1       0.97      1.00      0.99      1935
           2       0.43      0.23      0.30        13
           3       0.80      0.20      0.32        20
           4       0.00      0.00      0.00         6
           5       0.00      0.00      0.00        11

    accuracy                           0.97      2000
   macro avg       0.37      0.24      0.27      2000
weighted avg       0.95      0.97      0.96      2000
In [86]:
# Confusion Matrix
conf_matrix_voting_knn = confusion_matrix(y_test, y_pred_voting_knn)
plt.figure(figsize=(8, 6))
sns.heatmap(conf_matrix_voting_knn, annot=True, cmap='flare', fmt='d', cbar=False)
plt.title('Confusion Matrix - Voting Classifier with KNN Base Estimator')
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.show()

3.4 Voting Classifier with Logistic Regression

In [87]:
# Initialize the Voting Classifier with Logistic Regression
voting_logreg_classifier = VotingClassifier(
    estimators=[
        ('rf', RandomForestClassifier(random_state=42)),
        ('svm', SVC(kernel='rbf', C=1, gamma='scale', random_state=42, probability=True)),
        ('knn', KNeighborsClassifier(n_neighbors=5))
    ],
    voting='soft'
)
In [88]:
# Training the model
voting_logreg_classifier.fit(X_train, y_train)
Out[88]:
VotingClassifier(estimators=[('rf', RandomForestClassifier(random_state=42)),
                             ('svm',
                              SVC(C=1, probability=True, random_state=42)),
                             ('knn', KNeighborsClassifier())],
                 voting='soft')
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
VotingClassifier(estimators=[('rf', RandomForestClassifier(random_state=42)),
                             ('svm',
                              SVC(C=1, probability=True, random_state=42)),
                             ('knn', KNeighborsClassifier())],
                 voting='soft')
RandomForestClassifier(random_state=42)
SVC(C=1, probability=True, random_state=42)
KNeighborsClassifier()
In [89]:
# Making predictions
y_pred_voting_logreg = voting_logreg_classifier.predict(X_test)
In [90]:
# Calculating the accuracy score
accuracy_voting_logreg = accuracy_score(y_test, y_pred_voting_logreg)
print(f"Accuracy of Voting Classifier with Logistic Regression Base Estimator: {accuracy_voting_logreg*100:.2f}%")

Accuracy of Voting Classifier with Logistic Regression Base Estimator: 97.15%
In [91]:
# Classification report
print("Classification Report for Voting Classifier with Logistic Regression Base Estimator:")
print(classification_report(y_test, y_pred_voting_logreg))

Classification Report for Voting Classifier with Logistic Regression Base Estimator:
              precision    recall  f1-score   support

           0       0.00      0.00      0.00        15
           1       0.97      1.00      0.99      1935
           2       0.62      0.38      0.48        13
           3       0.80      0.20      0.32        20
           4       0.00      0.00      0.00         6
           5       0.00      0.00      0.00        11

    accuracy                           0.97      2000
   macro avg       0.40      0.26      0.30      2000
weighted avg       0.95      0.97      0.96      2000
In [92]:
# Confusion matrix
conf_matrix_voting_logreg = confusion_matrix(y_test, y_pred_voting_logreg)
plt.figure(figsize=(8, 6))
sns.heatmap(conf_matrix_voting_logreg, annot=True, cmap='flare', fmt='d', cbar=False)
plt.title('Confusion Matrix - Voting Classifier with Logistic Regression Base Estimator')
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.show()

Comparison Between The Models¶

In [93]:
# Accuracy Scores
print(f"Accuracy Scores:")
print(f"Accuracy of Stacking Classifier with Random Forest: {accuracy_stacking_rf*100:.2f}%")
print(f"Accuracy of Stacking Classifier with Logistic Regression: {accuracy_stacking_logreg*100:.2f}%")
print(f"Accuracy of Stacking Classifier with Support Vector Machine: {accuracy_stacking_svm*100:.2f}%")
print(f"Accuracy of Stacking Classifier with KNN: {accuracy_stacking_knn*100:.2f}%")
print(f"Accuracy of Bagging Classifier with Random Forest: {accuracy_rf_bagging*100:.2f}%")
print(f"Accuracy of Bagging Classifier with SVM: {accuracy_svm_bagging*100:.2f}%")
print(f"Accuracy of Bagging Classifier with KNN: {accuracy_knn_bagging*100:.2f}%")
print(f"Accuracy of Bagging Classifier with Logistic Regression: {accuracy_logreg_bagging*100:.2f}%")
print(f"Accuracy of Voting Classifier with Random Forest as Base Estimator: {accuracy_voting_rf*100:.2f}%")
print(f"Accuracy of Voting Classifier with SVM as Base Estimator: {accuracy_voting_svm*100:.2f}%")
print(f"Accuracy of Voting Classifier with KNN as Base Estimator: {accuracy_voting_knn*100:.2f}%")
print(f"Accuracy of Voting Classifier with Logistic Regression as Base Estimator: {accuracy_voting_logreg*100:.2f}%")

# Define models and accuracies
all_models = [
    'Stacking RF', 'Stacking LogReg', 'Stacking SVM', 'Stacking KNN',
    'Bagging RF', 'Bagging SVM', 'Bagging KNN', 'Bagging LogReg',
    'Voting RF', 'Voting SVM', 'Voting KNN', 'Voting LogReg'
]

all_accuracies = [
    accuracy_stacking_rf, accuracy_stacking_logreg, accuracy_stacking_svm, accuracy_stacking_knn,
    accuracy_rf_bagging, accuracy_svm_bagging, accuracy_knn_bagging, accuracy_logreg_bagging,
    accuracy_voting_rf, accuracy_voting_svm, accuracy_voting_knn, accuracy_voting_logreg
]

# Identify the best model
best_accuracy = max(all_accuracies)
best_model = all_models[all_accuracies.index(best_accuracy)]

print(f"\nBest Fitted Model: {best_model} with Accuracy: {best_accuracy*100:.2f}%")

Accuracy Scores:
Accuracy of Stacking Classifier with Random Forest: 97.05%
Accuracy of Stacking Classifier with Logistic Regression: 98.45%
Accuracy of Stacking Classifier with Support Vector Machine: 98.35%
Accuracy of Stacking Classifier with KNN: 98.40%
Accuracy of Bagging Classifier with Random Forest: 98.10%
Accuracy of Bagging Classifier with SVM: 97.10%
Accuracy of Bagging Classifier with KNN: 96.75%
Accuracy of Bagging Classifier with Logistic Regression: 97.40%
Accuracy of Voting Classifier with Random Forest as Base Estimator: 96.95%
Accuracy of Voting Classifier with SVM as Base Estimator: 97.05%
Accuracy of Voting Classifier with KNN as Base Estimator: 97.05%
Accuracy of Voting Classifier with Logistic Regression as Base Estimator: 97.15%

Best Fitted Model: Stacking LogReg with Accuracy: 98.45%
In [94]:
# Plot the bar chart
plt.figure(figsize=(12, 6))
colors = plt.cm.get_cmap('flare')(np.linspace(0, 1, len(all_models)))
bars = plt.barh(all_models, [accuracy * 100 for accuracy in all_accuracies], color=colors)
plt.xlabel('Accuracy (%)')
plt.title('Classifier Accuracy Comparison')
plt.grid(axis='x')

# Adding accuracy percentages
for index, value in enumerate(all_accuracies):
    plt.text(value * 100, index, f'{value * 100:.2f}%', va='center')

plt.show()
In [ ]: