Name: Teja Vamsi Devalla

Student ID: 2505091

Module: Artificial Intelligence (ACCA7026)

Assignment Title: Machine Learning Model

#importing all the required libraries

import pandas as pd #for datamanipulation and analysis
import numpy as np #for numerical computations
import matplotlib.pyplot as plt #for data visualisation
import seaborn as sns #for advanced visualisation

#importing preprocessing tools
from sklearn.preprocessing import StandardScaler #for standardising numerical columns
from sklearn.preprocessing import LabelEncoder #for converting catogorical columns to numerical columns

#importing model selection utilities.
from sklearn.model_selection import train_test_split #for splitting data into train and test data
from sklearn.utils.class_weight import compute_class_weight #for balancing inbalanced datasets

#importing machine learning algorithms
from sklearn.linear_model import LogisticRegression #for logistic regression classifier
from sklearn.svm import SVC #for support vector classifier
from sklearn.tree import DecisionTreeClassifier #for desision tree classifier
from sklearn.ensemble import BaggingClassifier #for bagging ensemble method
from sklearn.ensemble import RandomForestClassifier #for random forest classifier
from sklearn.ensemble import GradientBoostingClassifier #for gradient boosting classifier
from sklearn.ensemble import HistGradientBoostingClassifier


#importing model evolution metrics
from sklearn import metrics #for general metrics
from sklearn.metrics import confusion_matrix, classification_report, precision_recall_curve,recall_score,accuracy_score,make_scorer,f1_score

from sklearn import tree #for tree visualisation

#importing hypertuning tools
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import cross_val_score

import warnings
warnings.filterwarnings("ignore")
import time



 

Data exploration and preprocessing

Loading the dataset

#loading train dataset
dfTrain=pd.read_excel("kdd_train.xlsx")
dfTrain.head()

	duration	protocol_type	service	flag	src_bytes	dst_bytes	land	wrong_fragment	urgent	hot	...	dst_host_srv_count	dst_host_same_srv_rate	dst_host_diff_srv_rate	dst_host_same_src_port_rate	dst_host_srv_diff_host_rate	dst_host_serror_rate	dst_host_srv_serror_rate	dst_host_rerror_rate	dst_host_srv_rerror_rate	labels
0	0	tcp	ftp_data	SF	491	0	0	0	0	0	...	25	0.17	0.03	0.17	0.00	0.00	0.00	0.05	0.00	normal
1	0	udp	other	SF	146	0	0	0	0	0	...	1	0.00	0.60	0.88	0.00	0.00	0.00	0.00	0.00	normal
2	0	tcp	private	S0	0	0	0	0	0	0	...	26	0.10	0.05	0.00	0.00	1.00	1.00	0.00	0.00	neptune
3	0	tcp	http	SF	232	8153	0	0	0	0	...	255	1.00	0.00	0.03	0.04	0.03	0.01	0.00	0.01	normal
4	0	tcp	http	SF	199	420	0	0	0	0	...	255	1.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	normal

5 rows × 42 columns

#loading test dataset
dfTest=pd.read_excel("kdd_test.xlsx")
dfTest.head()

	duration	protocol_type	service	flag	src_bytes	dst_bytes	land	wrong_fragment	urgent	hot	...	dst_host_srv_count	dst_host_same_srv_rate	dst_host_diff_srv_rate	dst_host_same_src_port_rate	dst_host_srv_diff_host_rate	dst_host_serror_rate	dst_host_srv_serror_rate	dst_host_rerror_rate	dst_host_srv_rerror_rate	labels
0	5	tcp	smtp	SF	2429	475	0	0	0	0	...	57	0.11	0.02	0.01	0.04	0.0	0.0	0.87	0.0	normal
1	0	udp	domain_u	SF	45	134	0	0	0	0	...	155	0.80	0.02	0.01	0.00	0.0	0.0	0.00	0.0	normal
2	0	udp	domain_u	SF	45	80	0	0	0	0	...	255	1.00	0.00	0.01	0.00	0.0	0.0	0.00	0.0	normal
3	1979	udp	other	SF	145	105	0	0	0	0	...	2	0.01	0.84	1.00	0.00	0.0	0.0	0.00	0.0	normal
4	14462	tcp	other	RSTR	1	0	0	0	0	0	...	2	0.01	0.68	1.00	0.00	0.0	0.0	1.00	1.0	portsweep

5 rows × 42 columns

lets check number of unique values in each column

#number of unique values in each column from train dataset
dfTrain.nunique()

duration                       3161
protocol_type                     3
service                          70
flag                             11
src_bytes                      3392
dst_bytes                      9335
land                              2
wrong_fragment                    3
urgent                            4
hot                              28
num_failed_logins                 6
logged_in                         2
num_compromised                  90
root_shell                        2
su_attempted                      3
num_root                         83
num_file_creations               35
num_shells                        3
num_access_files                 10
num_outbound_cmds                 1
is_host_login                     2
is_guest_login                    2
count                           512
srv_count                       509
serror_rate                      92
srv_serror_rate                  91
rerror_rate                      88
srv_rerror_rate                  83
same_srv_rate                   101
diff_srv_rate                    95
srv_diff_host_rate               78
dst_host_count                  256
dst_host_srv_count              256
dst_host_same_srv_rate          101
dst_host_diff_srv_rate          101
dst_host_same_src_port_rate     101
dst_host_srv_diff_host_rate      75
dst_host_serror_rate            101
dst_host_srv_serror_rate        100
dst_host_rerror_rate            101
dst_host_srv_rerror_rate        101
labels                           37
dtype: int64

#number of unique values in each column from test dataset
dfTest.nunique()

duration                        704
protocol_type                     3
service                          66
flag                             11
src_bytes                      1517
dst_bytes                      3745
land                              2
wrong_fragment                    3
urgent                            3
hot                              18
num_failed_logins                 6
logged_in                         2
num_compromised                  29
root_shell                        2
su_attempted                      3
num_root                         27
num_file_creations               14
num_shells                        3
num_access_files                  7
num_outbound_cmds                 1
is_host_login                     2
is_guest_login                    2
count                           476
srv_count                       428
serror_rate                      85
srv_serror_rate                  66
rerror_rate                      85
srv_rerror_rate                  80
same_srv_rate                    97
diff_srv_rate                    92
srv_diff_host_rate               73
dst_host_count                  255
dst_host_srv_count              255
dst_host_same_srv_rate          101
dst_host_diff_srv_rate          101
dst_host_same_src_port_rate     101
dst_host_srv_diff_host_rate      60
dst_host_serror_rate            100
dst_host_srv_serror_rate        100
dst_host_rerror_rate            101
dst_host_srv_rerror_rate        101
labels                           36
dtype: int64

By observing number of columns in each , we can see that there is only one value in "num_outbound_cmds" column, a column which has only one value, doesn't give any information to the model. so we can remove it.

#dropping the unwanted columns from both train and test data sets 
dfTrain=dfTrain.drop(['num_outbound_cmds'],axis=1)
dfTest=dfTest.drop(['num_outbound_cmds'],axis=1)

By observing the number of unique values in the label column ,there are 23 unique values in train data set and 36 unique values in test data set. when the model trained on 23 labels in train dataset and tested on 36 labels in test data set , it doesn't predit well, which seems like overfitting but not exaclty that why we are adding some rows from test data set into the train data set which brings the missing labels in train dataset.

#adding rows to train dataset.
label_col='labels'
#Identify missing labels in train dataset
train_labels=set(dfTrain['labels'].unique())
test_labels=set(dfTest['labels'].unique())
missing_labels_in_train=sorted(test_labels-train_labels)
#selecting the required rows
rows_to_add=dfTest[dfTest['labels'].isin(missing_labels_in_train)].copy()
rows_to_add = rows_to_add[dfTrain.columns]
#concatanating to train dataset
dfTrain = pd.concat([dfTrain, rows_to_add], ignore_index=True)

Checking the info of the data set

#checking the information of the train dataset
dfTrain.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 127119 entries, 0 to 127118
Data columns (total 41 columns):
 #   Column                       Non-Null Count   Dtype  
---  ------                       --------------   -----  
 0   duration                     127119 non-null  int64  
 1   protocol_type                127119 non-null  object 
 2   service                      127119 non-null  object 
 3   flag                         127119 non-null  object 
 4   src_bytes                    127119 non-null  int64  
 5   dst_bytes                    127119 non-null  int64  
 6   land                         127119 non-null  int64  
 7   wrong_fragment               127119 non-null  int64  
 8   urgent                       127119 non-null  int64  
 9   hot                          127119 non-null  int64  
 10  num_failed_logins            127119 non-null  int64  
 11  logged_in                    127119 non-null  int64  
 12  num_compromised              127119 non-null  int64  
 13  root_shell                   127119 non-null  int64  
 14  su_attempted                 127119 non-null  int64  
 15  num_root                     127119 non-null  int64  
 16  num_file_creations           127119 non-null  int64  
 17  num_shells                   127119 non-null  int64  
 18  num_access_files             127119 non-null  int64  
 19  is_host_login                127119 non-null  int64  
 20  is_guest_login               127119 non-null  int64  
 21  count                        127119 non-null  int64  
 22  srv_count                    127119 non-null  int64  
 23  serror_rate                  127119 non-null  float64
 24  srv_serror_rate              127119 non-null  float64
 25  rerror_rate                  127119 non-null  float64
 26  srv_rerror_rate              127119 non-null  float64
 27  same_srv_rate                127119 non-null  float64
 28  diff_srv_rate                127119 non-null  float64
 29  srv_diff_host_rate           127119 non-null  float64
 30  dst_host_count               127119 non-null  int64  
 31  dst_host_srv_count           127119 non-null  int64  
 32  dst_host_same_srv_rate       127119 non-null  float64
 33  dst_host_diff_srv_rate       127119 non-null  float64
 34  dst_host_same_src_port_rate  127119 non-null  float64
 35  dst_host_srv_diff_host_rate  127119 non-null  float64
 36  dst_host_serror_rate         127119 non-null  float64
 37  dst_host_srv_serror_rate     127119 non-null  float64
 38  dst_host_rerror_rate         127119 non-null  float64
 39  dst_host_srv_rerror_rate     127119 non-null  float64
 40  labels                       127119 non-null  object 
dtypes: float64(15), int64(22), object(4)
memory usage: 39.8+ MB

---

Here we can check the information about the dataset,

• number of rows and columns,
• number of non-null values-->0
• dtype of the columns

we clearly say that there is no null values in our train data set

---

#checking the information of the test dataset
dfTest.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 22544 entries, 0 to 22543
Data columns (total 41 columns):
 #   Column                       Non-Null Count  Dtype  
---  ------                       --------------  -----  
 0   duration                     22544 non-null  int64  
 1   protocol_type                22544 non-null  object 
 2   service                      22544 non-null  object 
 3   flag                         22544 non-null  object 
 4   src_bytes                    22544 non-null  int64  
 5   dst_bytes                    22544 non-null  int64  
 6   land                         22544 non-null  int64  
 7   wrong_fragment               22544 non-null  int64  
 8   urgent                       22544 non-null  int64  
 9   hot                          22544 non-null  int64  
 10  num_failed_logins            22544 non-null  int64  
 11  logged_in                    22544 non-null  int64  
 12  num_compromised              22544 non-null  int64  
 13  root_shell                   22544 non-null  int64  
 14  su_attempted                 22544 non-null  int64  
 15  num_root                     22544 non-null  int64  
 16  num_file_creations           22544 non-null  int64  
 17  num_shells                   22544 non-null  int64  
 18  num_access_files             22544 non-null  int64  
 19  is_host_login                22544 non-null  int64  
 20  is_guest_login               22544 non-null  int64  
 21  count                        22544 non-null  int64  
 22  srv_count                    22544 non-null  int64  
 23  serror_rate                  22544 non-null  float64
 24  srv_serror_rate              22544 non-null  float64
 25  rerror_rate                  22544 non-null  float64
 26  srv_rerror_rate              22544 non-null  float64
 27  same_srv_rate                22544 non-null  float64
 28  diff_srv_rate                22544 non-null  float64
 29  srv_diff_host_rate           22544 non-null  float64
 30  dst_host_count               22544 non-null  int64  
 31  dst_host_srv_count           22544 non-null  int64  
 32  dst_host_same_srv_rate       22544 non-null  float64
 33  dst_host_diff_srv_rate       22544 non-null  float64
 34  dst_host_same_src_port_rate  22544 non-null  float64
 35  dst_host_srv_diff_host_rate  22544 non-null  float64
 36  dst_host_serror_rate         22544 non-null  float64
 37  dst_host_srv_serror_rate     22544 non-null  float64
 38  dst_host_rerror_rate         22544 non-null  float64
 39  dst_host_srv_rerror_rate     22544 non-null  float64
 40  labels                       22544 non-null  object 
dtypes: float64(15), int64(22), object(4)
memory usage: 7.1+ MB

Here we can check the information about the dataset,

• number of rows and columns,
• number of null values-->0
• dtype of the columns

we clearly say that there is no null values in our test data set

Here, by observing the dtypes of the columns, we can divide the columns into numerical columns and categorial columns. There are 4 catogorical columns and rest of the columns are numerical columns

#lets see all the columns in the train dataset
dfTrain.columns

Index(['duration', 'protocol_type', 'service', 'flag', 'src_bytes',
       'dst_bytes', 'land', 'wrong_fragment', 'urgent', 'hot',
       'num_failed_logins', 'logged_in', 'num_compromised', 'root_shell',
       'su_attempted', 'num_root', 'num_file_creations', 'num_shells',
       'num_access_files', 'is_host_login', 'is_guest_login', 'count',
       'srv_count', 'serror_rate', 'srv_serror_rate', 'rerror_rate',
       'srv_rerror_rate', 'same_srv_rate', 'diff_srv_rate',
       'srv_diff_host_rate', 'dst_host_count', 'dst_host_srv_count',
       'dst_host_same_srv_rate', 'dst_host_diff_srv_rate',
       'dst_host_same_src_port_rate', 'dst_host_srv_diff_host_rate',
       'dst_host_serror_rate', 'dst_host_srv_serror_rate',
       'dst_host_rerror_rate', 'dst_host_srv_rerror_rate', 'labels'],
      dtype='object')

#lets see all the columns in the test dataset
dfTest.columns

Index(['duration', 'protocol_type', 'service', 'flag', 'src_bytes',
       'dst_bytes', 'land', 'wrong_fragment', 'urgent', 'hot',
       'num_failed_logins', 'logged_in', 'num_compromised', 'root_shell',
       'su_attempted', 'num_root', 'num_file_creations', 'num_shells',
       'num_access_files', 'is_host_login', 'is_guest_login', 'count',
       'srv_count', 'serror_rate', 'srv_serror_rate', 'rerror_rate',
       'srv_rerror_rate', 'same_srv_rate', 'diff_srv_rate',
       'srv_diff_host_rate', 'dst_host_count', 'dst_host_srv_count',
       'dst_host_same_srv_rate', 'dst_host_diff_srv_rate',
       'dst_host_same_src_port_rate', 'dst_host_srv_diff_host_rate',
       'dst_host_serror_rate', 'dst_host_srv_serror_rate',
       'dst_host_rerror_rate', 'dst_host_srv_rerror_rate', 'labels'],
      dtype='object')

#assigning all the numerical columns into num_columns
num_columns=['duration', 'src_bytes',
       'dst_bytes', 'land', 'wrong_fragment', 'urgent', 'hot',
       'num_failed_logins', 'logged_in', 'num_compromised', 'root_shell',
       'su_attempted', 'num_root', 'num_file_creations', 'num_shells',
       'num_access_files', 'is_host_login',
       'is_guest_login', 'count', 'srv_count', 'serror_rate',
       'srv_serror_rate', 'rerror_rate', 'srv_rerror_rate', 'same_srv_rate',
       'diff_srv_rate', 'srv_diff_host_rate', 'dst_host_count',
       'dst_host_srv_count', 'dst_host_same_srv_rate',
       'dst_host_diff_srv_rate', 'dst_host_same_src_port_rate',
       'dst_host_srv_diff_host_rate', 'dst_host_serror_rate',
       'dst_host_srv_serror_rate', 'dst_host_rerror_rate',
       'dst_host_srv_rerror_rate']

#assigning all the categorical columns into cat_columns
cat_columns=['protocol_type', 'service', 'flag','labels']

#checking is there any null values in the columns of train dataset
dfTrain.isnull()

	duration	protocol_type	service	flag	src_bytes	dst_bytes	land	wrong_fragment	urgent	hot	...	dst_host_srv_count	dst_host_same_srv_rate	dst_host_diff_srv_rate	dst_host_same_src_port_rate	dst_host_srv_diff_host_rate	dst_host_serror_rate	dst_host_srv_serror_rate	dst_host_rerror_rate	dst_host_srv_rerror_rate	labels
0	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False
1	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False
2	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False
3	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False
4	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
127114	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False
127115	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False
127116	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False
127117	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False
127118	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False

127119 rows × 41 columns

#checking is any null values in the columns of test dataset
dfTest.isnull()

	duration	protocol_type	service	flag	src_bytes	dst_bytes	land	wrong_fragment	urgent	hot	...	dst_host_srv_count	dst_host_same_srv_rate	dst_host_diff_srv_rate	dst_host_same_src_port_rate	dst_host_srv_diff_host_rate	dst_host_serror_rate	dst_host_srv_serror_rate	dst_host_rerror_rate	dst_host_srv_rerror_rate	labels
0	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False
1	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False
2	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False
3	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False
4	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
22539	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False
22540	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False
22541	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False
22542	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False
22543	False	False	False	False	False	False	False	False	False	False	...	False	False	False	False	False	False	False	False	False	False

22544 rows × 41 columns

Exploratory data analysis (EDA)

#checking the summary statistics of the train dataset
dfTrain.describe()

	duration	src_bytes	dst_bytes	land	wrong_fragment	urgent	hot	num_failed_logins	logged_in	num_compromised	...	dst_host_count	dst_host_srv_count	dst_host_same_srv_rate	dst_host_diff_srv_rate	dst_host_same_src_port_rate	dst_host_srv_diff_host_rate	dst_host_serror_rate	dst_host_srv_serror_rate	dst_host_rerror_rate	dst_host_srv_rerror_rate
count	127119.000000	1.271190e+05	1.271190e+05	127119.000000	127119.000000	127119.000000	127119.000000	127119.000000	127119.000000	127119.000000	...	127119.000000	127119.000000	127119.000000	127119.00000	127119.000000	127119.000000	127119.000000	127119.000000	127119.000000	127119.000000
mean	294.975786	4.572862e+04	1.960429e+04	0.000197	0.022483	0.000134	0.202660	0.001243	0.394654	0.278377	...	182.449941	115.861862	0.521750	0.08319	0.147427	0.032374	0.283114	0.277326	0.120725	0.122202
std	2605.257621	5.846464e+06	4.003102e+06	0.014022	0.252394	0.015616	2.140381	0.046410	0.488778	23.838495	...	99.084718	110.603774	0.448394	0.18940	0.308273	0.112187	0.443690	0.444592	0.307496	0.320809
min	0.000000	0.000000e+00	0.000000e+00	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	...	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
25%	0.000000	0.000000e+00	0.000000e+00	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	...	83.000000	11.000000	0.050000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
50%	0.000000	4.400000e+01	0.000000e+00	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	...	255.000000	63.000000	0.510000	0.02000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
75%	0.000000	2.760000e+02	4.980000e+02	0.000000	0.000000	0.000000	0.000000	0.000000	1.000000	0.000000	...	255.000000	255.000000	1.000000	0.07000	0.060000	0.020000	1.000000	1.000000	0.000000	0.000000
max	42908.000000	1.379964e+09	1.309937e+09	1.000000	3.000000	3.000000	77.000000	5.000000	1.000000	7479.000000	...	255.000000	255.000000	1.000000	1.00000	1.000000	1.000000	1.000000	1.000000	1.000000	1.000000

8 rows × 37 columns

#checking the summary statistics of the test dataset
dfTest.describe()

	duration	src_bytes	dst_bytes	land	wrong_fragment	urgent	hot	num_failed_logins	logged_in	num_compromised	...	dst_host_count	dst_host_srv_count	dst_host_same_srv_rate	dst_host_diff_srv_rate	dst_host_same_src_port_rate	dst_host_srv_diff_host_rate	dst_host_serror_rate	dst_host_srv_serror_rate	dst_host_rerror_rate	dst_host_srv_rerror_rate
count	22544.000000	2.254400e+04	2.254400e+04	22544.000000	22544.000000	22544.000000	22544.000000	22544.000000	22544.000000	22544.000000	...	22544.000000	22544.000000	22544.000000	22544.000000	22544.000000	22544.000000	22544.000000	22544.000000	22544.000000	22544.000000
mean	258.858499	1.226873e+04	2.966787e+03	0.000089	0.016989	0.000177	0.155562	0.008783	0.406095	0.259359	...	186.878549	123.129613	0.545571	0.084905	0.141117	0.028275	0.229375	0.225729	0.155008	0.153560
std	2184.355090	4.665783e+05	6.293205e+04	0.009419	0.217507	0.016313	1.699324	0.104942	0.491114	11.370701	...	97.508604	111.577695	0.447079	0.199813	0.305538	0.105966	0.410665	0.412500	0.339255	0.350511
min	0.000000	0.000000e+00	0.000000e+00	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	...	1.000000	1.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
25%	0.000000	0.000000e+00	0.000000e+00	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	...	95.000000	12.000000	0.050000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
50%	0.000000	4.500000e+01	0.000000e+00	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	...	255.000000	85.000000	0.670000	0.020000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
75%	0.000000	2.750000e+02	5.950000e+02	0.000000	0.000000	0.000000	0.000000	0.000000	1.000000	0.000000	...	255.000000	255.000000	1.000000	0.070000	0.050000	0.010000	0.100000	0.010000	0.010000	0.000000
max	54451.000000	6.282565e+07	5.153771e+06	1.000000	3.000000	2.000000	30.000000	5.000000	1.000000	796.000000	...	255.000000	255.000000	1.000000	1.000000	1.000000	1.000000	1.000000	1.000000	1.000000	1.000000

8 rows × 37 columns

Univariate analysis for numerical columns

Lets see the distribution of each numerical columns in train dataset

#visual representation of each numerical column in train dataset by using histograms
dfTrain[num_columns].hist(figsize=(37,37))
plt.show()

---

Lets see the distribution of each numerical columns in test dataset

#visual representation of each numerical column in test dataset by using histograms
dfTest[num_columns].hist(figsize=(37,37))
plt.show()

---

Lets see the distribution of each categorical columns in train dataset

#let see the percentage of each value in each categorical column in train dataset
for i in cat_columns:
    print(dfTrain[i].value_counts (normalize=True))
    print('*'*40)

protocol_type
tcp     0.815425
udp     0.119101
icmp    0.065474
Name: proportion, dtype: float64
****************************************
service
http         0.319118
private      0.173137
domain_u     0.071138
smtp         0.058300
ftp_data     0.053997
               ...   
tftp_u       0.000024
http_8001    0.000016
aol          0.000016
harvest      0.000016
http_2784    0.000008
Name: proportion, Length: 70, dtype: float64
****************************************
flag
SF        0.593475
S0        0.275152
REJ       0.089743
RSTR      0.020288
RSTO      0.013200
S1        0.002879
SH        0.002132
S2        0.001015
S3        0.000944
RSTOS0    0.000810
OTH       0.000362
Name: proportion, dtype: float64
****************************************
labels
normal             0.529763
neptune            0.324216
satan              0.028580
ipsweep            0.028312
portsweep          0.023057
smurf              0.020815
nmap               0.011745
back               0.007521
teardrop           0.007017
warezclient        0.007001
mscan              0.002439
apache2            0.001794
processtable       0.001660
pod                0.001581
snmpguess          0.000779
mailbomb           0.000739
saint              0.000732
guess_passwd       0.000417
snmpgetattack      0.000338
httptunnel         0.000323
buffer_overflow    0.000236
warezmaster        0.000157
land               0.000142
imap               0.000087
rootkit            0.000079
loadmodule         0.000071
ps                 0.000071
sendmail           0.000063
ftp_write          0.000063
multihop           0.000055
xlock              0.000031
phf                0.000031
xterm              0.000024
perl               0.000024
spy                0.000016
named              0.000016
xsnoop             0.000008
Name: proportion, dtype: float64
****************************************

---

Lets see the distribution of each numerical columns in test dataset

#let see the percentage of each value in each categorical column in test dataset
for i in cat_columns:
    print(dfTest[i].value_counts (normalize=True))
    print('*'*40)

protocol_type
tcp     0.820972
udp     0.119810
icmp    0.059218
Name: proportion, dtype: float64
****************************************
service
http        0.324122
private     0.185637
domain_u    0.063653
smtp        0.052342
ftp_data    0.047330
              ...   
pm_dump     0.000222
tim_i       0.000222
urh_i       0.000133
red_i       0.000089
aol         0.000044
Name: proportion, Length: 66, dtype: float64
****************************************
flag
SF        0.612269
S0        0.221478
REJ       0.115153
RSTR      0.021691
RSTO      0.019828
S3        0.003504
SH        0.002440
S1        0.002040
S2        0.000887
RSTOS0    0.000532
OTH       0.000177
Name: proportion, dtype: float64
****************************************
labels
normal             0.498802
neptune            0.295156
satan              0.030962
smurf              0.023953
ipsweep            0.021247
guess_passwd       0.018231
portsweep          0.016457
mscan              0.013751
warezmaster        0.012376
apache2            0.010114
back               0.010069
processtable       0.009359
nmap               0.009138
warezclient        0.004746
teardrop           0.004613
snmpguess          0.004391
mailbomb           0.004170
saint              0.004125
snmpgetattack      0.001907
httptunnel         0.001819
pod                0.001553
buffer_overflow    0.000754
multihop           0.000399
ps                 0.000399
sendmail           0.000355
xlock              0.000177
loadmodule         0.000177
rootkit            0.000177
xterm              0.000133
land               0.000089
imap               0.000089
ftp_write          0.000089
named              0.000089
phf                0.000044
perl               0.000044
xsnoop             0.000044
Name: proportion, dtype: float64
****************************************

---

lets see the distribution of each categorical column in train dataset by using histogram

#creating histogram to show the distribution of each catogorical column
plt.figure(figsize=(16,4))
sns.countplot(x=dfTrain['protocol_type'])
plt.show()
plt.figure(figsize=(50,30))
sns.countplot(x=dfTrain['service'])
plt.show()
plt.figure(figsize=(8,4))
sns.countplot(x=dfTrain['flag'])
plt.show()
plt.figure(figsize=(25,6))
sns.countplot(x=dfTrain['labels'])
plt.show()

---

lets see the class distribution of target variable

#creating histogram to show how class distribution of target variable
plt.figure(figsize=(20,4))
sns.countplot(x='labels', data=dfTrain, order=dfTrain['labels'].value_counts().index)
plt.title("Class distribution of target variable")
plt.ylabel("Count")
plt.show()

Bivariate analysis of catogorical columns

lets see how target column is related to other categorical columns

#cearting histogram to check how each class in the label column is relation catogorical columns
for i in cat_columns:
    if i!='labels':
        (pd.crosstab(dfTrain[i],dfTrain['labels'],normalize='index')*100).plot(kind='bar',figsize=(60,16),stacked=True)
        plt.ylabel('percentage labels %')

Multivariate analysis of numerical columns

lets see the relation between different numerical columns by using heap map

#creating a heatmap to show relation between different numerical 
plt.figure(figsize=(60,30))
sns.heatmap(dfTrain[num_columns].corr(),annot=True,fmt='0.2f',cmap='Blues')

<Axes: >

Data preparation

#creating dummy variables
to_get_dummies_for=['protocol_type', 'service', 'flag']

one hot encoding

1.here , we are doing one hot encoding for ('protocol_type', 'service', 'flag') columns , the number of columns may increase now.
2.we have 2 datasets (train and test), some values in particular column only present in train dataset but not in test dataset.some values in paricular column only present in test dataset but not in train dataset.
3.so we are combining the datasets for encoding and then split back to train and test datasets.

#here we are combining the datasets, then encoding catogorical columns and then split back to train and test datasets
combined = pd.concat([dfTrain, dfTest], axis=0)
#One hot encode
combined = pd.get_dummies(combined, columns=to_get_dummies_for, drop_first=True)
combined.reset_index(drop=True, inplace=True)
# Splitting the combined dataset into train and test datasets 
dfTrain= combined.iloc[:len(dfTrain)]
dfTest= combined.iloc[len(dfTrain):]

here we are converting all the boolean columns into intergers , its better to be 0s and 1s for the model.

#selecting the boolean columns and Converting them into 0s and 1s in train dataset.
bool_cols_train = dfTrain.select_dtypes(include='bool').columns
dfTrain[bool_cols_train] = dfTrain[bool_cols_train].astype(int)

#selecting the boolean columns and Converting them into 0s and 1s in train dataset.
bool_cols_test = dfTest.select_dtypes(include='bool').columns
dfTest[bool_cols_test] = dfTest[bool_cols_test].astype(int)

The combined encoding is also needed for labels column,when we check the unique values particularly in labels column , number of unique values in train dataset is 23 and number of unique values in test dataset is 36. The model gets confuss if we only train on 23 labels and test on 36 labels that why we are encoding combinly.

# Combine labels from train and test
all_labels = pd.concat([dfTrain['labels'], dfTest['labels']], axis=0)
# Fit LabelEncoder on all labels
le = LabelEncoder()
le.fit(all_labels)

# Transform train and test labels
y_train = le.transform(dfTrain['labels'])
y_test = le.transform(dfTest['labels'])

dfTrain['labels'] = y_train
dfTest['labels'] = y_test

here we are taking all the training data into the x_all and all the tesing data into the y_all

#except target columns all the columns are assigning into x_all from both train and test dataset
x_all = pd.concat([
    dfTrain.drop(columns=['labels']),
    dfTest.drop(columns=['labels'])
], axis=0, ignore_index=True)

#target column is assigning into y_all from both train and test dataset
y_all = pd.concat([
    dfTrain['labels'],
    dfTest['labels']
], axis=0, ignore_index=True)

1.And from train dataset except traget column rest of the columns are in x_train
2.from train dataset the target column is in y_train
3.from test dataset except traget column rest of the columns are in x_test
4.from test dataset the target column is in y_test

x_train=dfTrain.drop(columns='labels')

y_train=dfTrain['labels']

x_test=dfTest.drop(columns='labels')

y_test=dfTest['labels']

#scaling the data
sc=StandardScaler()

#fit transform on train data
x_train_scaled=sc.fit_transform(x_train)
x_train_scaled=pd.DataFrame(x_train_scaled,columns=dfTrain.drop(columns=['labels']).columns)

#transform on test data
x_test_scaled=sc.transform(x_test)
x_test_scaled=pd.DataFrame(x_test_scaled,columns=dfTest.drop(columns=['labels']).columns)

here, we should not use fit_transform on test data,
because it would compute mean and standar deviation on test data
which leaks information to model an leeds to high accuracy which is unrealistic.

#creating metric fuction
def metrics_score_multiclass(actual, predicted, normalize=False):
    print(classification_report(actual, predicted))
    cm = confusion_matrix(actual, predicted)    
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
    classes = np.unique(np.concatenate((actual, predicted)))
    # Plot
    plt.figure(figsize=(14,10))
    sns.heatmap(cm, annot=True, fmt='.2f' if normalize else 'd', 
                cmap='Blues', xticklabels=classes, yticklabels=classes)
    plt.ylabel('Actual')
    plt.xlabel('Predicted')
    plt.title('Confusion Matrix')
    plt.xticks(rotation=90)
    plt.yticks(rotation=0)
    plt.show()

here, we are creating the metric_score to compare the actual and pedicted by using the heatmap

---

As we are dealing with the imbalanced datasets, the model may get biased, that why we are using classweigths.
low frequency - high classweight
high frequency - less classweight
this will balance the inbalanced dataset

---

creating the class weights by using the compute_class_weight() function .

#creating the classweights
classes = np.unique(dfTrain['labels'])
weights = compute_class_weight(
    class_weight='balanced',
    classes=classes,
    y=dfTrain['labels']
)
class_weights = dict(zip(classes, weights))
print(class_weights)

{0: 15.068634423897581, 1: 3.5937747370801763, 2: 114.52162162162162, 3: 429.4560810810811, 4: 64.82355940846507, 5: 83.79630850362558, 6: 312.33169533169536, 7: 0.9546120168515279, 8: 190.86936936936937, 9: 381.73873873873873, 10: 36.54945370902818, 11: 11.082737576285963, 12: 490.80694980694983, 13: 1717.8243243243244, 14: 0.08336120368439483, 15: 2.3011712315128254, 16: 0.051017160635086774, 17: 1145.2162162162163, 18: 858.9121621621622, 19: 17.09277934651069, 20: 1.17217627043625, 21: 16.282695017292173, 22: 381.73873873873873, 23: 343.5648648648649, 24: 36.942458587619875, 25: 0.9456781306492289, 26: 429.4560810810811, 27: 1.29843108414537, 28: 79.89880578252671, 29: 34.7035217035217, 30: 1717.8243243243244, 31: 3.851624045570234, 32: 3.8602793805040996, 33: 171.78243243243244, 34: 858.9121621621622, 35: 3435.6486486486488, 36: 1145.2162162162163}

Randomforest

1. Random Forest is a model which use many decision trees together.
2. Each tree learns from various parts of the data.
3. The final prediction will be taken from the majority vote of all trees.
4. random forest avoids overfitting and results will be stable.
5. and it give the most important features for making predictions.

#creating random forest classifier model
randomforest=RandomForestClassifier(class_weight=class_weights,random_state=1)

#fitting the random forest classifier on the train dataset
randomforest.fit(x_train,y_train)

RandomForestClassifier(class_weight={0: 15.068634423897581,
                                     1: 3.5937747370801763,
                                     2: 114.52162162162162,
                                     3: 429.4560810810811, 4: 64.82355940846507,
                                     5: 83.79630850362558,
                                     6: 312.33169533169536,
                                     7: 0.9546120168515279,
                                     8: 190.86936936936937,
                                     9: 381.73873873873873,
                                     10: 36.54945370902818,
                                     11: 11.082737576285963,
                                     12: 490.80694980694983,
                                     13: 1717.8243243243244,
                                     14: 0.08336120368439483,
                                     15: 2.3011712315128254,
                                     16: 0.051017160635086774,
                                     17: 1145.2162162162163,
                                     18: 858.9121621621622,
                                     19: 17.09277934651069,
                                     20: 1.17217627043625,
                                     21: 16.282695017292173,
                                     22: 381.73873873873873,
                                     23: 343.5648648648649,
                                     24: 36.942458587619875,
                                     25: 0.9456781306492289,
                                     26: 429.4560810810811,
                                     27: 1.29843108414537,
                                     28: 79.89880578252671,
                                     29: 34.7035217035217, ...},
                       random_state=1)

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

y_train_pred_randomforest=randomforest.predict(x_train)
metrics_score_multiclass(y_train,y_train_pred_randomforest)

              precision    recall  f1-score   support

           0       1.00      1.00      1.00       228
           1       1.00      1.00      1.00       956
           2       1.00      1.00      1.00        30
           3       1.00      1.00      1.00         8
           4       1.00      1.00      1.00        53
           5       1.00      1.00      1.00        41
           6       1.00      1.00      1.00        11
           7       1.00      1.00      1.00      3599
           8       0.82      1.00      0.90        18
           9       1.00      1.00      1.00         9
          10       1.00      1.00      1.00        94
          11       1.00      1.00      1.00       310
          12       1.00      1.00      1.00         7
          13       1.00      1.00      1.00         2
          14       1.00      1.00      1.00     41214
          15       0.99      1.00      1.00      1493
          16       1.00      1.00      1.00     67343
          17       1.00      1.00      1.00         3
          18       1.00      1.00      1.00         4
          19       1.00      1.00      1.00       201
          20       1.00      1.00      1.00      2931
          21       1.00      1.00      1.00       211
          22       1.00      1.00      1.00         9
          23       0.91      1.00      0.95        10
          24       0.99      1.00      0.99        93
          25       1.00      1.00      1.00      3633
          26       1.00      1.00      1.00         8
          27       1.00      1.00      1.00      2646
          28       0.90      1.00      0.95        43
          29       1.00      1.00      1.00        99
          30       1.00      1.00      1.00         2
          31       1.00      1.00      1.00       892
          32       1.00      1.00      1.00       890
          33       1.00      1.00      1.00        20
          34       1.00      1.00      1.00         4
          35       1.00      1.00      1.00         1
          36       1.00      1.00      1.00         3

    accuracy                           1.00    127119
   macro avg       0.99      1.00      0.99    127119
weighted avg       1.00      1.00      1.00    127119

y_test_pred_randomforest=randomforest.predict(x_test)
metrics_score_multiclass(y_test,y_test_pred_randomforest)

              precision    recall  f1-score   support

           0       1.00      1.00      1.00       228
           1       1.00      0.99      0.99       227
           2       1.00      0.41      0.58        17
           3       1.00      0.50      0.67         2
           4       1.00      0.02      0.04       411
           5       1.00      1.00      1.00        41
           6       1.00      1.00      1.00         2
           7       1.00      1.00      1.00       479
           8       1.00      1.00      1.00         2
           9       1.00      0.75      0.86         4
          10       1.00      1.00      1.00        94
          11       1.00      1.00      1.00       310
          12       1.00      0.11      0.20         9
          13       1.00      1.00      1.00         2
          14       1.00      1.00      1.00      6654
          15       1.00      1.00      1.00       206
          16       0.94      0.99      0.96     11245
          17       1.00      1.00      1.00         1
          18       1.00      1.00      1.00         1
          19       0.87      0.97      0.92        35
          20       0.97      1.00      0.98       371
          21       1.00      1.00      1.00       211
          22       1.00      1.00      1.00         9
          23       1.00      0.25      0.40         4
          24       0.88      1.00      0.93        93
          25       0.92      0.98      0.95       698
          26       1.00      1.00      1.00         8
          27       0.99      1.00      1.00       540
          28       0.64      1.00      0.78        43
          29       0.88      1.00      0.94        99
          31       0.89      1.00      0.94       104
          32       1.00      1.00      1.00       107
          33       1.00      0.01      0.02       279
          34       1.00      1.00      1.00         4
          35       1.00      1.00      1.00         1
          36       1.00      1.00      1.00         3

    accuracy                           0.96     22544
   macro avg       0.97      0.86      0.87     22544
weighted avg       0.96      0.96      0.95     22544

#lets display the important features of the model by using barsplot
importances=randomforest.feature_importances_
columns=x_all.columns
importance_df=pd.DataFrame(importances,index=columns,columns=['Importance']).sort_values(by='Importance',ascending=False)
plt.figure(figsize=(13,33))
sns.barplot(x=importance_df.Importance,y=importance_df.index)

<Axes: xlabel='Importance', ylabel='None'>

Observation

1. The Random Forest model performs well in training dataset, displaying the perfect precision, recall, and F1-scores.
2. Test accuracy is also high, but a few minority classes show lower recall, indicating slight overfitting.
3. The confusion matrix shows that most classes are correctly predicted with very few misclassifications.
4. Important features like src_bytes and dst_bytes will influence the model’s decisions.
5. Random Forest gives strong prediction and good generaling for intrusion detection.

Gradient boosting

1. Gradient Boosting builds trees one after another, each tree is correcting the errors of previous tree.
2. It learns slowly and carefully using a small learning rate.
3. Each new tree focuses on the mistakes made before, and improves the model step by step.
4. It can work well for complex patterns
5. With the right tuning, Gradient Boosting can become a strong and accurate prediction model.

#creating gradient boosting classifier
gradientboosting=GradientBoostingClassifier(n_estimators=30,learning_rate=0.1,subsample=0.7,max_features="sqrt",max_depth=2)

#training the gradient boosting model
gradientboosting.fit(x_train,y_train)

GradientBoostingClassifier(max_depth=2, max_features='sqrt', n_estimators=30,
                           subsample=0.7)

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

y_train_pred_gradientboosting=gradientboosting.predict(x_train)
metrics_score_multiclass(y_train,y_train_pred_gradientboosting)

              precision    recall  f1-score   support

           0       0.00      0.00      0.00       228
           1       0.00      0.00      0.00       956
           2       0.00      0.00      0.00        30
           3       0.00      0.00      0.00         8
           4       0.00      0.00      0.00        53
           5       0.00      0.00      0.00        41
           6       0.00      0.00      0.00        11
           7       0.00      0.00      0.00      3599
           8       0.00      0.00      0.00        18
           9       0.00      0.00      0.00         9
          10       0.00      0.00      0.00        94
          11       0.00      0.00      0.00       310
          12       0.00      0.00      0.00         7
          13       0.00      0.00      0.00         2
          14       0.00      0.00      0.00     41214
          15       0.00      0.00      0.00      1493
          16       0.00      0.00      0.00     67343
          17       0.00      0.00      0.00         3
          18       0.00      0.00      0.00         4
          19       0.00      0.00      0.00       201
          20       0.00      0.00      0.00      2931
          21       0.00      0.00      0.00       211
          22       0.09      0.11      0.10         9
          23       0.00      0.50      0.00        10
          24       0.00      0.00      0.00        93
          25       0.00      0.00      0.00      3633
          26       0.00      0.00      0.00         8
          27       0.00      0.00      0.00      2646
          28       0.00      0.00      0.00        43
          29       0.00      1.00      0.00        99
          30       0.00      0.00      0.00         2
          31       0.00      0.00      0.00       892
          32       0.00      0.00      0.00       890
          33       0.00      0.00      0.00        20
          34       0.00      0.00      0.00         4
          35       0.00      0.00      0.00         1
          36       0.00      0.00      0.00         3

    accuracy                           0.00    127119
   macro avg       0.00      0.04      0.00    127119
weighted avg       0.00      0.00      0.00    127119

y_test_pred_gradientboosting=gradientboosting.predict(x_test)
metrics_score_multiclass(y_test,y_test_pred_gradientboosting)

              precision    recall  f1-score   support

           0       0.00      0.00      0.00       228
           1       0.00      0.00      0.00       227
           2       0.00      0.00      0.00        17
           3       0.00      0.00      0.00         2
           4       0.00      0.00      0.00       411
           5       0.00      0.00      0.00        41
           6       0.00      0.00      0.00         2
           7       0.00      0.00      0.00       479
           8       0.00      0.00      0.00         2
           9       0.00      0.00      0.00         4
          10       0.00      0.00      0.00        94
          11       0.00      0.00      0.00       310
          12       0.00      0.00      0.00         9
          13       0.00      0.00      0.00         2
          14       0.00      0.00      0.00      6654
          15       0.00      0.00      0.00       206
          16       0.00      0.00      0.00     11245
          17       0.00      0.00      0.00         1
          18       0.00      0.00      0.00         1
          19       0.00      0.00      0.00        35
          20       0.00      0.00      0.00       371
          21       0.00      0.00      0.00       211
          22       0.17      0.11      0.13         9
          23       0.00      0.75      0.00         4
          24       0.00      0.00      0.00        93
          25       0.00      0.00      0.00       698
          26       0.00      0.00      0.00         8
          27       0.00      0.00      0.00       540
          28       0.00      0.00      0.00        43
          29       0.01      1.00      0.02        99
          31       0.00      0.00      0.00       104
          32       0.00      0.00      0.00       107
          33       0.00      0.00      0.00       279
          34       0.00      0.00      0.00         4
          35       0.00      0.00      0.00         1
          36       0.00      0.00      0.00         3

    accuracy                           0.00     22544
   macro avg       0.00      0.05      0.00     22544
weighted avg       0.00      0.00      0.00     22544

#lets diplay the important features of gradient boosting by using the barplots
importances=gradientboosting.feature_importances_
columns=x_all.columns
importance_df=pd.DataFrame(importances,index=columns,columns=['Importance']).sort_values(by='Importance',ascending=False)
plt.figure(figsize=(13,33))
sns.barplot(x=importance_df.Importance,y=importance_df.index)

<Axes: xlabel='Importance', ylabel='None'>

Observation

1. The Gradient Boosting model performs very bad on both training and test dataset.
2. more classes have zero precision, recall, and F1-score, meaning the model fails to learn patterns.
3. The confusion matrix shows that the model predicts almost only a few major classes.
4. Class imbalance heavily affects the model, causing it to ignore minority attack types.
5. Important features like src_bytes and serror_rate still appear meaningful, but the model cannot use them effectively.
6. i think gradient boosting cant handle huge amount of data , advanced models like XGBoosting,LightBoosting and etc may give better results but they wont give important features.

Hyperparameter tuning

1. Hyperparameter tuning helps find the best settings for a model.
2. It tests different combinations of parameters to improve performance.
3. RandomizedSearchCV is faster because it tries random sets instead of all possibilities.
4. Good tuning reduces overfitting and increases accuracy on new data.
5. It makes the model more reliable by choosing the most effective parameters.

#hyperstuning for the randomforest
randomforest_tuned=RandomForestClassifier(class_weight=class_weights,random_state=1)
params_randomforest={
    "n_estimators":[50,100],
    "min_samples_leaf":[1,2],
    "max_features":[0.7,'auto']
}
scorer=metrics.make_scorer(recall_score,pos_label=1)
random_search=RandomizedSearchCV(estimator=randomforest_tuned,param_distributions=params_randomforest,n_iter=10,scoring=scorer,cv=5,n_jobs=-1)
random_search=random_search.fit(x_train,y_train)

randomforest_tuned.fit(x_train,y_train)

RandomForestClassifier(class_weight={0: 15.068634423897581,
                                     1: 3.5937747370801763,
                                     2: 114.52162162162162,
                                     3: 429.4560810810811, 4: 64.82355940846507,
                                     5: 83.79630850362558,
                                     6: 312.33169533169536,
                                     7: 0.9546120168515279,
                                     8: 190.86936936936937,
                                     9: 381.73873873873873,
                                     10: 36.54945370902818,
                                     11: 11.082737576285963,
                                     12: 490.80694980694983,
                                     13: 1717.8243243243244,
                                     14: 0.08336120368439483,
                                     15: 2.3011712315128254,
                                     16: 0.051017160635086774,
                                     17: 1145.2162162162163,
                                     18: 858.9121621621622,
                                     19: 17.09277934651069,
                                     20: 1.17217627043625,
                                     21: 16.282695017292173,
                                     22: 381.73873873873873,
                                     23: 343.5648648648649,
                                     24: 36.942458587619875,
                                     25: 0.9456781306492289,
                                     26: 429.4560810810811,
                                     27: 1.29843108414537,
                                     28: 79.89880578252671,
                                     29: 34.7035217035217, ...},
                       random_state=1)

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

y_pred_train_randomforest_tuned=randomforest_tuned.predict(x_train)
metrics_score_multiclass(y_train,y_pred_train_randomforest_tuned)

              precision    recall  f1-score   support

           0       1.00      1.00      1.00       228
           1       1.00      1.00      1.00       956
           2       1.00      1.00      1.00        30
           3       1.00      1.00      1.00         8
           4       1.00      1.00      1.00        53
           5       1.00      1.00      1.00        41
           6       1.00      1.00      1.00        11
           7       1.00      1.00      1.00      3599
           8       0.82      1.00      0.90        18
           9       1.00      1.00      1.00         9
          10       1.00      1.00      1.00        94
          11       1.00      1.00      1.00       310
          12       1.00      1.00      1.00         7
          13       1.00      1.00      1.00         2
          14       1.00      1.00      1.00     41214
          15       0.99      1.00      1.00      1493
          16       1.00      1.00      1.00     67343
          17       1.00      1.00      1.00         3
          18       1.00      1.00      1.00         4
          19       1.00      1.00      1.00       201
          20       1.00      1.00      1.00      2931
          21       1.00      1.00      1.00       211
          22       1.00      1.00      1.00         9
          23       0.91      1.00      0.95        10
          24       0.99      1.00      0.99        93
          25       1.00      1.00      1.00      3633
          26       1.00      1.00      1.00         8
          27       1.00      1.00      1.00      2646
          28       0.90      1.00      0.95        43
          29       1.00      1.00      1.00        99
          30       1.00      1.00      1.00         2
          31       1.00      1.00      1.00       892
          32       1.00      1.00      1.00       890
          33       1.00      1.00      1.00        20
          34       1.00      1.00      1.00         4
          35       1.00      1.00      1.00         1
          36       1.00      1.00      1.00         3

    accuracy                           1.00    127119
   macro avg       0.99      1.00      0.99    127119
weighted avg       1.00      1.00      1.00    127119

y_pred_test_randomforest_tuned=randomforest_tuned.predict(x_test)
metrics_score_multiclass(y_test,y_pred_test_randomforest_tuned)

              precision    recall  f1-score   support

           0       1.00      1.00      1.00       228
           1       1.00      0.99      0.99       227
           2       1.00      0.41      0.58        17
           3       1.00      0.50      0.67         2
           4       1.00      0.02      0.04       411
           5       1.00      1.00      1.00        41
           6       1.00      1.00      1.00         2
           7       1.00      1.00      1.00       479
           8       1.00      1.00      1.00         2
           9       1.00      0.75      0.86         4
          10       1.00      1.00      1.00        94
          11       1.00      1.00      1.00       310
          12       1.00      0.11      0.20         9
          13       1.00      1.00      1.00         2
          14       1.00      1.00      1.00      6654
          15       1.00      1.00      1.00       206
          16       0.94      0.99      0.96     11245
          17       1.00      1.00      1.00         1
          18       1.00      1.00      1.00         1
          19       0.87      0.97      0.92        35
          20       0.97      1.00      0.98       371
          21       1.00      1.00      1.00       211
          22       1.00      1.00      1.00         9
          23       1.00      0.25      0.40         4
          24       0.88      1.00      0.93        93
          25       0.92      0.98      0.95       698
          26       1.00      1.00      1.00         8
          27       0.99      1.00      1.00       540
          28       0.64      1.00      0.78        43
          29       0.88      1.00      0.94        99
          31       0.89      1.00      0.94       104
          32       1.00      1.00      1.00       107
          33       1.00      0.01      0.02       279
          34       1.00      1.00      1.00         4
          35       1.00      1.00      1.00         1
          36       1.00      1.00      1.00         3

    accuracy                           0.96     22544
   macro avg       0.97      0.86      0.87     22544
weighted avg       0.96      0.96      0.95     22544

importances=randomforest_tuned.feature_importances_
columns=x_all.columns
importance_df=pd.DataFrame(importances,index=columns,columns=['Importance']).sort_values(by='Importance',ascending=False)
plt.figure(figsize=(13,33))
sns.barplot(x=importance_df.Importance,y=importance_df.index)

<Axes: xlabel='Importance', ylabel='None'>

#taking smaller subset of the dataset for tuning the gradient boosting because it is slow when we train on full dataset
x_train_subset = x_train.sample(n=min(2000, int(0.1 * len(x_train))), random_state=42)
y_train_subset = y_train.loc[x_train_subset.index]

gradientboosting_tuned=GradientBoostingClassifier(random_state=1)
params_gradientboosting = {
    "n_estimators": [80, 120, 160],     
    "learning_rate": [0.08, 0.12, 0.16],
    "max_depth": [3, 4],                
    "subsample": [0.7, 0.9],           
    "max_features": [0.5, "sqrt", None] 
}
scorer_recall_macro = make_scorer(recall_score, average='macro')
random_search = RandomizedSearchCV(
    estimator=gradientboosting_tuned,
    param_distributions=params_gradientboosting,
    n_iter=8,                  
    scoring=scorer_recall_macro,
    cv=3,
    n_jobs=-1,              
    random_state=1,
    verbose=1
)
random_search=random_search.fit(x_train_subset,y_train_subset)

Fitting 3 folds for each of 8 candidates, totalling 24 fits

gradientboosting_tuned.fit(x_train,y_train)

y_pred_train_gradientboosting_tuned=gradientboosting_tuned.predict(x_train)
metrics_score_multiclass(y_train,y_pred_train_gradientboosting_tuned)

              precision    recall  f1-score   support

           0       1.00      1.00      1.00       228
           1       1.00      1.00      1.00       956
           2       0.50      0.20      0.29        30
           3       1.00      0.38      0.55         8
           4       1.00      0.98      0.99        53
           5       1.00      0.02      0.05        41
           6       1.00      0.91      0.95        11
           7       0.82      0.31      0.45      3599
           8       0.80      0.22      0.35        18
           9       0.03      0.22      0.05         9
          10       1.00      1.00      1.00        94
          11       1.00      1.00      1.00       310
          12       1.00      0.14      0.25         7
          13       1.00      1.00      1.00         2
          14       1.00      1.00      1.00     41214
          15       1.00      0.99      0.99      1493
          16       0.96      0.99      0.97     67343
          17       0.00      0.00      0.00         3
          18       0.00      0.00      0.00         4
          19       0.99      0.71      0.83       201
          20       1.00      0.99      1.00      2931
          21       1.00      1.00      1.00       211
          22       0.02      0.44      0.04         9
          23       0.01      0.20      0.02        10
          24       0.97      0.32      0.48        93
          25       0.98      0.97      0.98      3633
          26       0.01      0.12      0.02         8
          27       1.00      1.00      1.00      2646
          28       0.20      0.02      0.04        43
          29       1.00      0.88      0.94        99
          30       1.00      0.50      0.67         2
          31       1.00      1.00      1.00       892
          32       0.99      0.97      0.98       890
          33       1.00      0.90      0.95        20
          34       0.00      0.00      0.00         4
          35       0.00      0.00      0.00         1
          36       0.33      0.33      0.33         3

    accuracy                           0.97    127119
   macro avg       0.72      0.59      0.60    127119
weighted avg       0.97      0.97      0.97    127119

y_pred_test_gradientboosting_tuned=gradientboosting_tuned.predict(x_test)
metrics_score_multiclass(y_test,y_pred_test_gradientboosting_tuned)

              precision    recall  f1-score   support

           0       0.92      1.00      0.96       228
           1       1.00      0.93      0.96       227
           2       0.25      0.06      0.10        17
           3       0.50      0.50      0.50         2
           4       1.00      0.05      0.10       411
           5       1.00      0.02      0.05        41
           6       1.00      0.50      0.67         2
           7       0.83      0.38      0.52       479
           8       1.00      0.50      0.67         2
           9       0.00      0.00      0.00         4
          10       1.00      1.00      1.00        94
          11       0.59      1.00      0.74       310
          12       0.00      0.00      0.00         9
          13       1.00      1.00      1.00         2
          14       1.00      1.00      1.00      6654
          15       1.00      0.96      0.98       206
          16       0.93      0.98      0.96     11245
          17       0.00      0.00      0.00         1
          18       0.00      0.00      0.00         1
          19       0.92      0.69      0.79        35
          20       0.97      0.99      0.98       371
          21       1.00      1.00      1.00       211
          22       0.09      0.44      0.15         9
          23       0.00      0.00      0.00         4
          24       1.00      0.32      0.49        93
          25       0.87      0.99      0.92       698
          26       0.07      0.12      0.09         8
          27       0.99      1.00      1.00       540
          28       0.12      0.02      0.04        43
          29       1.00      0.88      0.94        99
          31       0.89      1.00      0.94       104
          32       0.57      0.98      0.72       107
          33       0.75      0.01      0.02       279
          34       0.00      0.00      0.00         4
          35       0.00      0.00      0.00         1
          36       1.00      0.33      0.50         3

    accuracy                           0.94     22544
   macro avg       0.65      0.52      0.52     22544
weighted avg       0.94      0.94      0.92     22544

#lets displaay the important features of gradientboosting_tuned by using barplots 
importances=gradientboosting_tuned.feature_importances_
columns=x_all.columns
importance_df=pd.DataFrame(importances,index=columns,columns=['Importance']).sort_values(by='Importance',ascending=False)
plt.figure(figsize=(13,33))
sns.barplot(x=importance_df.Importance,y=importance_df.index)

<Axes: xlabel='Importance', ylabel='None'>

---

Computational efficiency analysis

1. here we are messuring how well both models perform in terms of training time and precision time
2. we can analyse which model is faster to train and which model is faster in predicting

start = time.time()
randomforest.fit(x_train, y_train)
randomforest_train_time = time.time() - start
start = time.time()
randomforest_preds = randomforest.predict(x_test)
randomforest_pred_time = time.time() - start

start = time.time()
gradientboosting.fit(x_train, y_train)
gradientboosting_train_time = time.time() - start
start = time.time()
gradientboosting_preds = gradientboosting.predict(x_test)
gradientboosting_pred_time = time.time() - start


print(randomforest_train_time)
print(gradientboosting_train_time)
print(randomforest_pred_time)
print(gradientboosting_pred_time)

7.291335344314575
21.458756685256958
0.3218984603881836
0.2384355068206787

models = ['RandomForest', 'GradientBoosting']
train_times = [randomforest_train_time, gradientboosting_train_time]
pred_times = [randomforest_pred_time, gradientboosting_pred_time]
time_df = pd.DataFrame({
    'Model': models * 2,
    'Time (seconds)': train_times + pred_times,
    'Type': ['Training']*2 + ['Prediction']*2
})
plt.figure(figsize=(8,5))
sns.barplot(x='Model', y='Time (seconds)', hue='Type', data=time_df)
plt.title('Training vs Prediction Time for Models')
plt.ylabel('Time (seconds)')
plt.show()

Observation

1. random forest takes less time to training than the gradient boosting.
2. gradient boosting takes less time in testing than the random forest.

---