In [5]:
# Importing required libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import LabelEncoder
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.model_selection import RandomizedSearchCV, train_test_split
from sklearn.metrics import accuracy_score, f1_score,precision_score,recall_score
from sklearn.metrics import classification_report, confusion_matrix
import warnings
warnings.filterwarnings("ignore")
In [6]:
# Import test dataset
tf=pd.read_csv('kdd_test.csv')
In [7]:
#Explore the dataset
tf.head(20)
Out[7]:
| duration | protocol_type | service | flag | src_bytes | dst_bytes | land | wrong_fragment | urgent | hot | ... | dst_host_srv_count | dst_host_same_srv_rate | dst_host_diff_srv_rate | dst_host_same_src_port_rate | dst_host_srv_diff_host_rate | dst_host_serror_rate | dst_host_srv_serror_rate | dst_host_rerror_rate | dst_host_srv_rerror_rate | labels | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 5 | tcp | smtp | SF | 2429 | 475 | 0 | 0 | 0 | 0 | ... | 57 | 0.11 | 0.02 | 0.01 | 0.04 | 0.00 | 0.0 | 0.87 | 0.0 | normal |
| 1 | 0 | udp | domain_u | SF | 45 | 134 | 0 | 0 | 0 | 0 | ... | 155 | 0.80 | 0.02 | 0.01 | 0.00 | 0.00 | 0.0 | 0.00 | 0.0 | normal |
| 2 | 0 | udp | domain_u | SF | 45 | 80 | 0 | 0 | 0 | 0 | ... | 255 | 1.00 | 0.00 | 0.01 | 0.00 | 0.00 | 0.0 | 0.00 | 0.0 | normal |
| 3 | 1979 | udp | other | SF | 145 | 105 | 0 | 0 | 0 | 0 | ... | 2 | 0.01 | 0.84 | 1.00 | 0.00 | 0.00 | 0.0 | 0.00 | 0.0 | normal |
| 4 | 14462 | tcp | other | RSTR | 1 | 0 | 0 | 0 | 0 | 0 | ... | 2 | 0.01 | 0.68 | 1.00 | 0.00 | 0.00 | 0.0 | 1.00 | 1.0 | portsweep |
| 5 | 0 | tcp | private | S0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 9 | 0.04 | 0.07 | 0.00 | 0.00 | 1.00 | 1.0 | 0.00 | 0.0 | neptune |
| 6 | 0 | tcp | http | SF | 294 | 5499 | 0 | 0 | 0 | 0 | ... | 255 | 1.00 | 0.00 | 0.50 | 0.04 | 0.00 | 0.0 | 0.00 | 0.0 | normal |
| 7 | 0 | tcp | http | S0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 44 | 0.17 | 0.03 | 0.01 | 0.00 | 1.00 | 1.0 | 0.00 | 0.0 | neptune |
| 8 | 0 | icmp | ecr_i | SF | 1032 | 0 | 0 | 0 | 0 | 0 | ... | 255 | 1.00 | 0.00 | 1.00 | 0.00 | 0.00 | 0.0 | 0.00 | 0.0 | smurf |
| 9 | 0 | udp | private | SF | 1 | 0 | 0 | 0 | 0 | 0 | ... | 32 | 0.13 | 0.45 | 0.66 | 0.00 | 0.03 | 0.0 | 0.31 | 0.0 | satan |
| 10 | 0 | tcp | mtp | REJ | 0 | 0 | 0 | 0 | 0 | 0 | ... | 6 | 0.02 | 0.08 | 0.00 | 0.00 | 0.00 | 0.0 | 1.00 | 1.0 | neptune |
| 11 | 0 | tcp | netstat | RSTR | 0 | 0 | 0 | 0 | 0 | 0 | ... | 1 | 0.01 | 0.17 | 0.13 | 0.00 | 0.00 | 0.0 | 0.12 | 1.0 | portsweep |
| 12 | 0 | tcp | private | S0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 6 | 0.02 | 0.09 | 0.00 | 0.00 | 1.00 | 1.0 | 0.00 | 0.0 | neptune |
| 13 | 0 | tcp | private | S0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 9 | 0.04 | 0.07 | 0.00 | 0.00 | 1.00 | 1.0 | 0.00 | 0.0 | neptune |
| 14 | 0 | tcp | http_443 | S0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 20 | 0.08 | 0.05 | 0.00 | 0.00 | 1.00 | 1.0 | 0.00 | 0.0 | neptune |
| 15 | 0 | tcp | http | SF | 291 | 670 | 0 | 0 | 0 | 0 | ... | 255 | 1.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.0 | 0.00 | 0.0 | normal |
| 16 | 0 | tcp | private | S0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 5 | 0.02 | 0.06 | 0.00 | 0.00 | 1.00 | 1.0 | 0.00 | 0.0 | neptune |
| 17 | 0 | tcp | http | SF | 340 | 851 | 0 | 0 | 0 | 0 | ... | 255 | 1.00 | 0.00 | 0.08 | 0.03 | 0.00 | 0.0 | 0.00 | 0.0 | normal |
| 18 | 0 | tcp | smtp | SF | 1201 | 330 | 0 | 0 | 0 | 0 | ... | 201 | 0.74 | 0.02 | 0.01 | 0.01 | 0.00 | 0.0 | 0.00 | 0.0 | normal |
| 19 | 0 | tcp | http | SF | 192 | 3717 | 0 | 0 | 0 | 0 | ... | 188 | 1.00 | 0.00 | 1.00 | 0.04 | 0.00 | 0.0 | 0.00 | 0.0 | normal |
20 rows × 42 columns
In [8]:
#Show the last top 5 data from dataset
tf.tail()
Out[8]:
| duration | protocol_type | service | flag | src_bytes | dst_bytes | land | wrong_fragment | urgent | hot | ... | dst_host_srv_count | dst_host_same_srv_rate | dst_host_diff_srv_rate | dst_host_same_src_port_rate | dst_host_srv_diff_host_rate | dst_host_serror_rate | dst_host_srv_serror_rate | dst_host_rerror_rate | dst_host_srv_rerror_rate | labels | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 22539 | 0 | tcp | private | REJ | 0 | 0 | 0 | 0 | 0 | 0 | ... | 1 | 0.0 | 1.00 | 0.00 | 0.00 | 0.09 | 0.0 | 0.91 | 1.0 | satan |
| 22540 | 0 | tcp | http | SF | 254 | 2277 | 0 | 0 | 0 | 0 | ... | 255 | 1.0 | 0.00 | 0.04 | 0.04 | 0.00 | 0.0 | 0.00 | 0.0 | normal |
| 22541 | 0 | tcp | private | S0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 25 | 0.1 | 0.07 | 0.00 | 0.00 | 1.00 | 1.0 | 0.00 | 0.0 | neptune |
| 22542 | 0 | tcp | private | RSTR | 0 | 0 | 0 | 0 | 0 | 0 | ... | 1 | 0.0 | 0.58 | 0.57 | 0.00 | 0.00 | 0.0 | 0.58 | 1.0 | portsweep |
| 22543 | 0 | udp | domain_u | SF | 44 | 129 | 0 | 0 | 0 | 0 | ... | 255 | 1.0 | 0.00 | 0.00 | 0.00 | 0.00 | 0.0 | 0.00 | 0.0 | normal |
5 rows × 42 columns
In [9]:
#Show all the information about dataset
tf.info()
<class 'pandas.core.frame.DataFrame'> RangeIndex: 22544 entries, 0 to 22543 Data columns (total 42 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 duration 22544 non-null int64 1 protocol_type 22544 non-null object 2 service 22544 non-null object 3 flag 22544 non-null object 4 src_bytes 22544 non-null int64 5 dst_bytes 22544 non-null int64 6 land 22544 non-null int64 7 wrong_fragment 22544 non-null int64 8 urgent 22544 non-null int64 9 hot 22544 non-null int64 10 num_failed_logins 22544 non-null int64 11 logged_in 22544 non-null int64 12 num_compromised 22544 non-null int64 13 root_shell 22544 non-null int64 14 su_attempted 22544 non-null int64 15 num_root 22544 non-null int64 16 num_file_creations 22544 non-null int64 17 num_shells 22544 non-null int64 18 num_access_files 22544 non-null int64 19 num_outbound_cmds 22544 non-null int64 20 is_host_login 22544 non-null int64 21 is_guest_login 22544 non-null int64 22 count 22544 non-null int64 23 srv_count 22544 non-null int64 24 serror_rate 22544 non-null float64 25 srv_serror_rate 22544 non-null float64 26 rerror_rate 22544 non-null float64 27 srv_rerror_rate 22544 non-null float64 28 same_srv_rate 22544 non-null float64 29 diff_srv_rate 22544 non-null float64 30 srv_diff_host_rate 22544 non-null float64 31 dst_host_count 22544 non-null int64 32 dst_host_srv_count 22544 non-null int64 33 dst_host_same_srv_rate 22544 non-null float64 34 dst_host_diff_srv_rate 22544 non-null float64 35 dst_host_same_src_port_rate 22544 non-null float64 36 dst_host_srv_diff_host_rate 22544 non-null float64 37 dst_host_serror_rate 22544 non-null float64 38 dst_host_srv_serror_rate 22544 non-null float64 39 dst_host_rerror_rate 22544 non-null float64 40 dst_host_srv_rerror_rate 22544 non-null float64 41 labels 22544 non-null object dtypes: float64(15), int64(23), object(4) memory usage: 7.2+ MB
In [10]:
# Import test dataset
df = pd.read_csv('kdd_train.csv')
In [11]:
#Explore the dataset
df.head(20)
Out[11]:
| duration | protocol_type | service | flag | src_bytes | dst_bytes | land | wrong_fragment | urgent | hot | ... | dst_host_srv_count | dst_host_same_srv_rate | dst_host_diff_srv_rate | dst_host_same_src_port_rate | dst_host_srv_diff_host_rate | dst_host_serror_rate | dst_host_srv_serror_rate | dst_host_rerror_rate | dst_host_srv_rerror_rate | labels | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 0 | tcp | ftp_data | SF | 491 | 0 | 0 | 0 | 0 | 0 | ... | 25 | 0.17 | 0.03 | 0.17 | 0.00 | 0.00 | 0.00 | 0.05 | 0.00 | normal |
| 1 | 0 | udp | other | SF | 146 | 0 | 0 | 0 | 0 | 0 | ... | 1 | 0.00 | 0.60 | 0.88 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | normal |
| 2 | 0 | tcp | private | S0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 26 | 0.10 | 0.05 | 0.00 | 0.00 | 1.00 | 1.00 | 0.00 | 0.00 | neptune |
| 3 | 0 | tcp | http | SF | 232 | 8153 | 0 | 0 | 0 | 0 | ... | 255 | 1.00 | 0.00 | 0.03 | 0.04 | 0.03 | 0.01 | 0.00 | 0.01 | normal |
| 4 | 0 | tcp | http | SF | 199 | 420 | 0 | 0 | 0 | 0 | ... | 255 | 1.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | normal |
| 5 | 0 | tcp | private | REJ | 0 | 0 | 0 | 0 | 0 | 0 | ... | 19 | 0.07 | 0.07 | 0.00 | 0.00 | 0.00 | 0.00 | 1.00 | 1.00 | neptune |
| 6 | 0 | tcp | private | S0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 9 | 0.04 | 0.05 | 0.00 | 0.00 | 1.00 | 1.00 | 0.00 | 0.00 | neptune |
| 7 | 0 | tcp | private | S0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 15 | 0.06 | 0.07 | 0.00 | 0.00 | 1.00 | 1.00 | 0.00 | 0.00 | neptune |
| 8 | 0 | tcp | remote_job | S0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 23 | 0.09 | 0.05 | 0.00 | 0.00 | 1.00 | 1.00 | 0.00 | 0.00 | neptune |
| 9 | 0 | tcp | private | S0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 13 | 0.05 | 0.06 | 0.00 | 0.00 | 1.00 | 1.00 | 0.00 | 0.00 | neptune |
| 10 | 0 | tcp | private | REJ | 0 | 0 | 0 | 0 | 0 | 0 | ... | 12 | 0.05 | 0.07 | 0.00 | 0.00 | 0.00 | 0.00 | 1.00 | 1.00 | neptune |
| 11 | 0 | tcp | private | S0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 13 | 0.05 | 0.07 | 0.00 | 0.00 | 1.00 | 1.00 | 0.00 | 0.00 | neptune |
| 12 | 0 | tcp | http | SF | 287 | 2251 | 0 | 0 | 0 | 0 | ... | 219 | 1.00 | 0.00 | 0.12 | 0.03 | 0.00 | 0.00 | 0.00 | 0.00 | normal |
| 13 | 0 | tcp | ftp_data | SF | 334 | 0 | 0 | 0 | 0 | 0 | ... | 20 | 1.00 | 0.00 | 1.00 | 0.20 | 0.00 | 0.00 | 0.00 | 0.00 | warezclient |
| 14 | 0 | tcp | name | S0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 1 | 0.00 | 0.07 | 0.00 | 0.00 | 1.00 | 1.00 | 0.00 | 0.00 | neptune |
| 15 | 0 | tcp | netbios_ns | S0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 2 | 0.01 | 0.06 | 0.00 | 0.00 | 1.00 | 1.00 | 0.00 | 0.00 | neptune |
| 16 | 0 | tcp | http | SF | 300 | 13788 | 0 | 0 | 0 | 0 | ... | 255 | 1.00 | 0.00 | 0.01 | 0.02 | 0.00 | 0.00 | 0.00 | 0.00 | normal |
| 17 | 0 | icmp | eco_i | SF | 18 | 0 | 0 | 0 | 0 | 0 | ... | 16 | 1.00 | 0.00 | 1.00 | 1.00 | 0.00 | 0.00 | 0.00 | 0.00 | ipsweep |
| 18 | 0 | tcp | http | SF | 233 | 616 | 0 | 0 | 0 | 0 | ... | 255 | 1.00 | 0.00 | 0.02 | 0.03 | 0.00 | 0.00 | 0.02 | 0.00 | normal |
| 19 | 0 | tcp | http | SF | 343 | 1178 | 0 | 0 | 0 | 0 | ... | 255 | 1.00 | 0.00 | 0.01 | 0.04 | 0.00 | 0.00 | 0.00 | 0.00 | normal |
20 rows × 42 columns
In [12]:
#Show the last top 5 data from dataset
df.tail()
Out[12]:
| duration | protocol_type | service | flag | src_bytes | dst_bytes | land | wrong_fragment | urgent | hot | ... | dst_host_srv_count | dst_host_same_srv_rate | dst_host_diff_srv_rate | dst_host_same_src_port_rate | dst_host_srv_diff_host_rate | dst_host_serror_rate | dst_host_srv_serror_rate | dst_host_rerror_rate | dst_host_srv_rerror_rate | labels | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 125968 | 0 | tcp | private | S0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 25 | 0.10 | 0.06 | 0.00 | 0.0 | 1.00 | 1.0 | 0.00 | 0.0 | neptune |
| 125969 | 8 | udp | private | SF | 105 | 145 | 0 | 0 | 0 | 0 | ... | 244 | 0.96 | 0.01 | 0.01 | 0.0 | 0.00 | 0.0 | 0.00 | 0.0 | normal |
| 125970 | 0 | tcp | smtp | SF | 2231 | 384 | 0 | 0 | 0 | 0 | ... | 30 | 0.12 | 0.06 | 0.00 | 0.0 | 0.72 | 0.0 | 0.01 | 0.0 | normal |
| 125971 | 0 | tcp | klogin | S0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 8 | 0.03 | 0.05 | 0.00 | 0.0 | 1.00 | 1.0 | 0.00 | 0.0 | neptune |
| 125972 | 0 | tcp | ftp_data | SF | 151 | 0 | 0 | 0 | 0 | 0 | ... | 77 | 0.30 | 0.03 | 0.30 | 0.0 | 0.00 | 0.0 | 0.00 | 0.0 | normal |
5 rows × 42 columns
In [13]:
#Show all the information about dataset
df.info()
<class 'pandas.core.frame.DataFrame'> RangeIndex: 125973 entries, 0 to 125972 Data columns (total 42 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 duration 125973 non-null int64 1 protocol_type 125973 non-null object 2 service 125973 non-null object 3 flag 125973 non-null object 4 src_bytes 125973 non-null int64 5 dst_bytes 125973 non-null int64 6 land 125973 non-null int64 7 wrong_fragment 125973 non-null int64 8 urgent 125973 non-null int64 9 hot 125973 non-null int64 10 num_failed_logins 125973 non-null int64 11 logged_in 125973 non-null int64 12 num_compromised 125973 non-null int64 13 root_shell 125973 non-null int64 14 su_attempted 125973 non-null int64 15 num_root 125973 non-null int64 16 num_file_creations 125973 non-null int64 17 num_shells 125973 non-null int64 18 num_access_files 125973 non-null int64 19 num_outbound_cmds 125973 non-null int64 20 is_host_login 125973 non-null int64 21 is_guest_login 125973 non-null int64 22 count 125973 non-null int64 23 srv_count 125973 non-null int64 24 serror_rate 125973 non-null float64 25 srv_serror_rate 125973 non-null float64 26 rerror_rate 125973 non-null float64 27 srv_rerror_rate 125973 non-null float64 28 same_srv_rate 125973 non-null float64 29 diff_srv_rate 125973 non-null float64 30 srv_diff_host_rate 125973 non-null float64 31 dst_host_count 125973 non-null int64 32 dst_host_srv_count 125973 non-null int64 33 dst_host_same_srv_rate 125973 non-null float64 34 dst_host_diff_srv_rate 125973 non-null float64 35 dst_host_same_src_port_rate 125973 non-null float64 36 dst_host_srv_diff_host_rate 125973 non-null float64 37 dst_host_serror_rate 125973 non-null float64 38 dst_host_srv_serror_rate 125973 non-null float64 39 dst_host_rerror_rate 125973 non-null float64 40 dst_host_srv_rerror_rate 125973 non-null float64 41 labels 125973 non-null object dtypes: float64(15), int64(23), object(4) memory usage: 40.4+ MB
In [14]:
# explore the unique types of data in each columns
df.nunique()
Out[14]:
duration 2981 protocol_type 3 service 70 flag 11 src_bytes 3341 dst_bytes 9326 land 2 wrong_fragment 3 urgent 4 hot 28 num_failed_logins 6 logged_in 2 num_compromised 88 root_shell 2 su_attempted 3 num_root 82 num_file_creations 35 num_shells 3 num_access_files 10 num_outbound_cmds 1 is_host_login 2 is_guest_login 2 count 512 srv_count 509 serror_rate 89 srv_serror_rate 86 rerror_rate 82 srv_rerror_rate 62 same_srv_rate 101 diff_srv_rate 95 srv_diff_host_rate 60 dst_host_count 256 dst_host_srv_count 256 dst_host_same_srv_rate 101 dst_host_diff_srv_rate 101 dst_host_same_src_port_rate 101 dst_host_srv_diff_host_rate 75 dst_host_serror_rate 101 dst_host_srv_serror_rate 100 dst_host_rerror_rate 101 dst_host_srv_rerror_rate 101 labels 23 dtype: int64
In [15]:
# drop a list of columns from dataset
df=df.drop(['land','wrong_fragment','urgent','hot','num_failed_logins','num_compromised','root_shell','su_attempted','num_root',
'num_file_creations','num_shells','num_access_files','num_outbound_cmds','is_host_login','is_guest_login'],
axis=1)
In [16]:
# Identify Numerical columns
num_columns = [ 'duration','src_bytes','dst_bytes','logged_in','count','srv_count','serror_rate','srv_serror_rate','rerror_rate',
'srv_rerror_rate','same_srv_rate','diff_srv_rate','srv_diff_host_rate','dst_host_count','dst_host_srv_count',
'dst_host_same_srv_rate','dst_host_diff_srv_rate','dst_host_same_src_port_rate','dst_host_srv_diff_host_rate',
'dst_host_serror_rate','dst_host_srv_serror_rate','dst_host_rerror_rate','dst_host_srv_rerror_rate' ]
# Identify categorical columns
categorical_cols = ['protocol_type', 'service', 'flag']
In [17]:
# Generate summary statistics for the numeric columns
df[num_columns].describe().T
Out[17]:
| count | mean | std | min | 25% | 50% | 75% | max | |
|---|---|---|---|---|---|---|---|---|
| duration | 125973.0 | 287.144650 | 2.604515e+03 | 0.0 | 0.00 | 0.00 | 0.00 | 4.290800e+04 |
| src_bytes | 125973.0 | 45566.743000 | 5.870331e+06 | 0.0 | 0.00 | 44.00 | 276.00 | 1.379964e+09 |
| dst_bytes | 125973.0 | 19779.114421 | 4.021269e+06 | 0.0 | 0.00 | 0.00 | 516.00 | 1.309937e+09 |
| logged_in | 125973.0 | 0.395736 | 4.890101e-01 | 0.0 | 0.00 | 0.00 | 1.00 | 1.000000e+00 |
| count | 125973.0 | 84.107555 | 1.145086e+02 | 0.0 | 2.00 | 14.00 | 143.00 | 5.110000e+02 |
| srv_count | 125973.0 | 27.737888 | 7.263584e+01 | 0.0 | 2.00 | 8.00 | 18.00 | 5.110000e+02 |
| serror_rate | 125973.0 | 0.284485 | 4.464556e-01 | 0.0 | 0.00 | 0.00 | 1.00 | 1.000000e+00 |
| srv_serror_rate | 125973.0 | 0.282485 | 4.470225e-01 | 0.0 | 0.00 | 0.00 | 1.00 | 1.000000e+00 |
| rerror_rate | 125973.0 | 0.119958 | 3.204355e-01 | 0.0 | 0.00 | 0.00 | 0.00 | 1.000000e+00 |
| srv_rerror_rate | 125973.0 | 0.121183 | 3.236472e-01 | 0.0 | 0.00 | 0.00 | 0.00 | 1.000000e+00 |
| same_srv_rate | 125973.0 | 0.660928 | 4.396229e-01 | 0.0 | 0.09 | 1.00 | 1.00 | 1.000000e+00 |
| diff_srv_rate | 125973.0 | 0.063053 | 1.803144e-01 | 0.0 | 0.00 | 0.00 | 0.06 | 1.000000e+00 |
| srv_diff_host_rate | 125973.0 | 0.097322 | 2.598305e-01 | 0.0 | 0.00 | 0.00 | 0.00 | 1.000000e+00 |
| dst_host_count | 125973.0 | 182.148945 | 9.920621e+01 | 0.0 | 82.00 | 255.00 | 255.00 | 2.550000e+02 |
| dst_host_srv_count | 125973.0 | 115.653005 | 1.107027e+02 | 0.0 | 10.00 | 63.00 | 255.00 | 2.550000e+02 |
| dst_host_same_srv_rate | 125973.0 | 0.521242 | 4.489494e-01 | 0.0 | 0.05 | 0.51 | 1.00 | 1.000000e+00 |
| dst_host_diff_srv_rate | 125973.0 | 0.082951 | 1.889218e-01 | 0.0 | 0.00 | 0.02 | 0.07 | 1.000000e+00 |
| dst_host_same_src_port_rate | 125973.0 | 0.148379 | 3.089971e-01 | 0.0 | 0.00 | 0.00 | 0.06 | 1.000000e+00 |
| dst_host_srv_diff_host_rate | 125973.0 | 0.032542 | 1.125638e-01 | 0.0 | 0.00 | 0.00 | 0.02 | 1.000000e+00 |
| dst_host_serror_rate | 125973.0 | 0.284452 | 4.447841e-01 | 0.0 | 0.00 | 0.00 | 1.00 | 1.000000e+00 |
| dst_host_srv_serror_rate | 125973.0 | 0.278485 | 4.456691e-01 | 0.0 | 0.00 | 0.00 | 1.00 | 1.000000e+00 |
| dst_host_rerror_rate | 125973.0 | 0.118832 | 3.065575e-01 | 0.0 | 0.00 | 0.00 | 0.00 | 1.000000e+00 |
| dst_host_srv_rerror_rate | 125973.0 | 0.120240 | 3.194594e-01 | 0.0 | 0.00 | 0.00 | 0.00 | 1.000000e+00 |
In [18]:
# Generate Histrograms
df[num_columns].hist(figsize=(14,14))
plt.show()
In [19]:
# For each categorical column, print the normalized value counts
for i in categorical_cols:
print(df[i].value_counts (normalize=True))
print('*'*40)
protocol_type
tcp 0.815167
udp 0.119018
icmp 0.065816
Name: proportion, dtype: float64
****************************************
service
http 0.320211
private 0.173474
domain_u 0.071785
smtp 0.058052
ftp_data 0.054456
...
tftp_u 0.000024
http_8001 0.000016
aol 0.000016
harvest 0.000016
http_2784 0.000008
Name: proportion, Length: 70, dtype: float64
****************************************
flag
SF 0.594929
S0 0.276655
REJ 0.089170
RSTR 0.019218
RSTO 0.012399
S1 0.002897
SH 0.002151
S2 0.001008
RSTOS0 0.000818
S3 0.000389
OTH 0.000365
Name: proportion, dtype: float64
****************************************
In [20]:
# For each categorical column (except 'protocol_type'), create a stacked bar chart
for i in categorical_cols:
if i!='protocol_type':
(pd.crosstab(df[i],df['protocol_type'],normalize='index')*100).plot(kind='bar',figsize=(8,4),stacked=True)
plt.ylabel('Percentage Attrition %')
In [21]:
# create heatmap for numeric columns
plt.figure(figsize=(15,8))
sns.heatmap(df[num_columns].corr(), annot=True, fmt="0.2f", cmap='Blues')
Out[21]:
<Axes: >
In [22]:
# One-hot encode categorical variables
to_get_dummies_for = ['protocol_type','service','flag']
df = pd.get_dummies(df, columns=['protocol_type','service','flag'], drop_first=True)
In [23]:
# Split labels
Y = df['labels']
X = df.drop(columns=['labels'])
In [24]:
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size = 0.3, random_state = 1, stratify = Y)
In [25]:
# Scale numeric features
sc=StandardScaler()
X_train_scaled=sc.fit_transform(X_train)
X_train_scaled=pd.DataFrame(X_train_scaled, columns=X.columns)
X_test_scaled=sc.transform(X_test)
X_test_scaled=pd.DataFrame(X_test_scaled, columns=X.columns)
In [26]:
#Model 1: Random Forest Classifier
rf_model = RandomForestClassifier(n_estimators=100, random_state=42, class_weight='balanced')
rf_model.fit(X_train_scaled, y_train)
y_pred = rf_model.predict(X_test_scaled)
print(classification_report(y_test, y_pred))
precision recall f1-score support
back 1.00 1.00 1.00 287
buffer_overflow 1.00 0.44 0.62 9
ftp_write 1.00 0.50 0.67 2
guess_passwd 1.00 0.88 0.93 16
imap 1.00 0.67 0.80 3
ipsweep 0.99 1.00 1.00 1080
land 0.60 0.60 0.60 5
loadmodule 0.00 0.00 0.00 3
multihop 0.00 0.00 0.00 2
neptune 1.00 1.00 1.00 12364
nmap 0.99 0.99 0.99 448
normal 1.00 1.00 1.00 20203
perl 0.00 0.00 0.00 1
phf 0.00 0.00 0.00 1
pod 0.98 1.00 0.99 60
portsweep 1.00 1.00 1.00 879
rootkit 0.00 0.00 0.00 3
satan 1.00 0.99 0.99 1090
smurf 1.00 1.00 1.00 794
spy 0.00 0.00 0.00 1
teardrop 1.00 1.00 1.00 268
warezclient 0.99 0.97 0.98 267
warezmaster 1.00 0.67 0.80 6
accuracy 1.00 37792
macro avg 0.72 0.64 0.67 37792
weighted avg 1.00 1.00 1.00 37792
In [27]:
results = []
param_list = [
{"n_estimators": 50, "max_depth": 10},
{"n_estimators": 100, "max_depth": 10},
{"n_estimators": 100, "max_depth": 20},
{"n_estimators": 200, "max_depth": 20},
{"n_estimators": 300, "max_depth": None}
]
for params in param_list:
print(f"\nTraining with: {params}")
rf = RandomForestClassifier(
n_estimators=params["n_estimators"],
max_depth=params["max_depth"],
random_state=42,
n_jobs=1
)
rf.fit(X_train_scaled, y_train)
y_pred = rf.predict(X_test_scaled)
acc = accuracy_score(y_test, y_pred)
print(classification_report(y_test, y_pred))
results.append({
"n_estimators": params["n_estimators"],
"max_depth": params["max_depth"],
"accuracy": acc
})
Training with: {'n_estimators': 50, 'max_depth': 10}
precision recall f1-score support
back 1.00 0.96 0.98 287
buffer_overflow 0.00 0.00 0.00 9
ftp_write 0.00 0.00 0.00 2
guess_passwd 1.00 0.81 0.90 16
imap 1.00 0.67 0.80 3
ipsweep 0.99 0.98 0.99 1080
land 0.60 0.60 0.60 5
loadmodule 0.00 0.00 0.00 3
multihop 0.00 0.00 0.00 2
neptune 1.00 1.00 1.00 12364
nmap 1.00 0.95 0.97 448
normal 0.99 1.00 0.99 20203
perl 0.00 0.00 0.00 1
phf 0.00 0.00 0.00 1
pod 1.00 0.68 0.81 60
portsweep 1.00 0.99 1.00 879
rootkit 0.00 0.00 0.00 3
satan 1.00 0.98 0.99 1090
smurf 1.00 1.00 1.00 794
spy 0.00 0.00 0.00 1
teardrop 1.00 1.00 1.00 268
warezclient 0.99 0.57 0.72 267
warezmaster 0.00 0.00 0.00 6
accuracy 0.99 37792
macro avg 0.59 0.53 0.55 37792
weighted avg 0.99 0.99 0.99 37792
Training with: {'n_estimators': 100, 'max_depth': 10}
precision recall f1-score support
back 1.00 0.95 0.97 287
buffer_overflow 0.00 0.00 0.00 9
ftp_write 0.00 0.00 0.00 2
guess_passwd 1.00 0.81 0.90 16
imap 1.00 0.67 0.80 3
ipsweep 0.99 0.98 0.99 1080
land 0.60 0.60 0.60 5
loadmodule 0.00 0.00 0.00 3
multihop 0.00 0.00 0.00 2
neptune 1.00 1.00 1.00 12364
nmap 1.00 0.95 0.97 448
normal 0.99 1.00 0.99 20203
perl 0.00 0.00 0.00 1
phf 0.00 0.00 0.00 1
pod 1.00 0.83 0.91 60
portsweep 1.00 0.99 1.00 879
rootkit 0.00 0.00 0.00 3
satan 1.00 0.98 0.99 1090
smurf 1.00 1.00 1.00 794
spy 0.00 0.00 0.00 1
teardrop 1.00 1.00 1.00 268
warezclient 0.97 0.53 0.68 267
warezmaster 0.00 0.00 0.00 6
accuracy 0.99 37792
macro avg 0.59 0.53 0.56 37792
weighted avg 0.99 0.99 0.99 37792
Training with: {'n_estimators': 100, 'max_depth': 20}
precision recall f1-score support
back 1.00 1.00 1.00 287
buffer_overflow 1.00 0.67 0.80 9
ftp_write 0.00 0.00 0.00 2
guess_passwd 1.00 0.88 0.93 16
imap 1.00 0.67 0.80 3
ipsweep 0.99 1.00 0.99 1080
land 0.60 0.60 0.60 5
loadmodule 1.00 0.33 0.50 3
multihop 0.00 0.00 0.00 2
neptune 1.00 1.00 1.00 12364
nmap 0.99 0.99 0.99 448
normal 1.00 1.00 1.00 20203
perl 0.00 0.00 0.00 1
phf 0.00 0.00 0.00 1
pod 0.98 0.98 0.98 60
portsweep 1.00 1.00 1.00 879
rootkit 0.00 0.00 0.00 3
satan 1.00 0.99 1.00 1090
smurf 1.00 1.00 1.00 794
spy 0.00 0.00 0.00 1
teardrop 1.00 1.00 1.00 268
warezclient 0.99 0.97 0.98 267
warezmaster 1.00 0.83 0.91 6
accuracy 1.00 37792
macro avg 0.72 0.65 0.67 37792
weighted avg 1.00 1.00 1.00 37792
Training with: {'n_estimators': 200, 'max_depth': 20}
precision recall f1-score support
back 1.00 1.00 1.00 287
buffer_overflow 1.00 0.67 0.80 9
ftp_write 0.00 0.00 0.00 2
guess_passwd 1.00 0.88 0.93 16
imap 1.00 0.67 0.80 3
ipsweep 0.99 1.00 0.99 1080
land 0.60 0.60 0.60 5
loadmodule 1.00 0.33 0.50 3
multihop 0.00 0.00 0.00 2
neptune 1.00 1.00 1.00 12364
nmap 0.98 0.99 0.99 448
normal 1.00 1.00 1.00 20203
perl 0.00 0.00 0.00 1
phf 0.00 0.00 0.00 1
pod 0.98 0.98 0.98 60
portsweep 1.00 1.00 1.00 879
rootkit 0.00 0.00 0.00 3
satan 1.00 0.99 1.00 1090
smurf 1.00 1.00 1.00 794
spy 0.00 0.00 0.00 1
teardrop 1.00 1.00 1.00 268
warezclient 0.99 0.97 0.98 267
warezmaster 1.00 0.83 0.91 6
accuracy 1.00 37792
macro avg 0.72 0.65 0.67 37792
weighted avg 1.00 1.00 1.00 37792
Training with: {'n_estimators': 300, 'max_depth': None}
precision recall f1-score support
back 1.00 1.00 1.00 287
buffer_overflow 1.00 0.67 0.80 9
ftp_write 1.00 0.50 0.67 2
guess_passwd 1.00 0.88 0.93 16
imap 1.00 0.67 0.80 3
ipsweep 0.99 1.00 0.99 1080
land 0.60 0.60 0.60 5
loadmodule 1.00 0.33 0.50 3
multihop 0.00 0.00 0.00 2
neptune 1.00 1.00 1.00 12364
nmap 0.98 0.99 0.99 448
normal 1.00 1.00 1.00 20203
perl 0.00 0.00 0.00 1
phf 0.00 0.00 0.00 1
pod 0.98 0.97 0.97 60
portsweep 1.00 1.00 1.00 879
rootkit 0.00 0.00 0.00 3
satan 1.00 0.99 1.00 1090
smurf 1.00 1.00 1.00 794
spy 0.00 0.00 0.00 1
teardrop 1.00 1.00 1.00 268
warezclient 0.99 0.98 0.98 267
warezmaster 1.00 0.83 0.91 6
accuracy 1.00 37792
macro avg 0.76 0.67 0.70 37792
weighted avg 1.00 1.00 1.00 37792
In [28]:
# THIS is trained model:
results_df = pd.DataFrame(results)
results_df.sort_values(by="accuracy", ascending=False)
Out[28]:
| n_estimators | max_depth | accuracy | |
|---|---|---|---|
| 2 | 100 | 20.0 | 0.998465 |
| 4 | 300 | NaN | 0.998439 |
| 3 | 200 | 20.0 | 0.998386 |
| 0 | 50 | 10.0 | 0.992961 |
| 1 | 100 | 10.0 | 0.992670 |
In [29]:
#Checking Performance on the Training and Testing Data
best_rf = RandomForestClassifier(
n_estimators=200,
max_depth=20,
random_state=42,
n_jobs=1
)
best_rf.fit(X_train_scaled, y_train)
y_pred = best_rf.predict(X_test_scaled)
print(classification_report(y_test, y_pred))
precision recall f1-score support
back 1.00 1.00 1.00 287
buffer_overflow 1.00 0.67 0.80 9
ftp_write 0.00 0.00 0.00 2
guess_passwd 1.00 0.88 0.93 16
imap 1.00 0.67 0.80 3
ipsweep 0.99 1.00 0.99 1080
land 0.60 0.60 0.60 5
loadmodule 1.00 0.33 0.50 3
multihop 0.00 0.00 0.00 2
neptune 1.00 1.00 1.00 12364
nmap 0.98 0.99 0.99 448
normal 1.00 1.00 1.00 20203
perl 0.00 0.00 0.00 1
phf 0.00 0.00 0.00 1
pod 0.98 0.98 0.98 60
portsweep 1.00 1.00 1.00 879
rootkit 0.00 0.00 0.00 3
satan 1.00 0.99 1.00 1090
smurf 1.00 1.00 1.00 794
spy 0.00 0.00 0.00 1
teardrop 1.00 1.00 1.00 268
warezclient 0.99 0.97 0.98 267
warezmaster 1.00 0.83 0.91 6
accuracy 1.00 37792
macro avg 0.72 0.65 0.67 37792
weighted avg 1.00 1.00 1.00 37792
In [30]:
#Random Forest Confusion Matrix
cm = confusion_matrix(y_test, y_pred)
plt.figure(figsize=(8,6))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
xticklabels=best_rf.classes_,
yticklabels=best_rf.classes_)
plt.title("Random Forest Confusion Matrix")
plt.xlabel("Predicted Label")
plt.ylabel("True Label")
plt.show()
In [31]:
#Normalized Confusion Matrix
plt.figure(figsize=(8,6))
cm_norm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
sns.heatmap(cm_norm, annot=True, fmt=".2f", cmap="Blues",
xticklabels=best_rf.classes_,
yticklabels=best_rf.classes_)
plt.title("Normalized Confusion Matrix")
plt.ylabel("True label")
plt.xlabel("Predicted label")
plt.show()
In [32]:
# Feature Importances
importances = best_rf.feature_importances_
features = X_train.columns
fi = pd.DataFrame({
'Feature': features,
'Importance': importances
})
fi = fi.sort_values(by='Importance', ascending=False).head(15)
plt.figure(figsize=(10,6))
plt.barh(fi['Feature'], fi['Importance'])
plt.gca().invert_yaxis()
plt.xlabel("Importance Score")
plt.title("Top 15 Important Features (Random Forest)")
plt.tight_layout()
plt.show()
In [33]:
#Model 2: Gradient Boosting
gb = GradientBoostingClassifier(
n_estimators=150,
learning_rate=0.1,
max_depth=3,
random_state=42
)
gb.fit(X_train_scaled, y_train)
y_pred_gb = gb.predict(X_test_scaled)
print(classification_report(y_test, y_pred_gb))
precision recall f1-score support
back 0.99 0.99 0.99 287
buffer_overflow 0.17 0.22 0.19 9
ftp_write 0.00 0.00 0.00 2
guess_passwd 0.00 0.00 0.00 16
imap 1.00 1.00 1.00 3
ipsweep 0.96 0.98 0.97 1080
land 0.33 0.40 0.36 5
loadmodule 0.12 0.33 0.18 3
multihop 0.00 0.00 0.00 2
neptune 1.00 0.96 0.98 12364
nmap 0.99 0.99 0.99 448
normal 0.98 0.99 0.98 20203
perl 0.00 0.00 0.00 1
phf 0.50 1.00 0.67 1
pod 0.97 0.98 0.98 60
portsweep 1.00 0.99 0.99 879
rootkit 0.00 0.00 0.00 3
satan 0.80 0.98 0.88 1090
smurf 1.00 1.00 1.00 794
spy 0.00 0.00 0.00 1
teardrop 1.00 1.00 1.00 268
warezclient 0.99 0.76 0.86 267
warezmaster 0.05 0.67 0.10 6
accuracy 0.98 37792
macro avg 0.56 0.62 0.57 37792
weighted avg 0.98 0.98 0.98 37792
In [34]:
#Gradient Boosting Confusion Matrix
cm_gb = confusion_matrix(y_test, y_pred_gb)
plt.figure(figsize=(7,6))
sns.heatmap(cm_gb, annot=True, fmt="d", cmap="Greens",
xticklabels=gb.classes_,
yticklabels=gb.classes_)
plt.xlabel("Predicted")
plt.ylabel("Actual")
plt.title("Confusion Matrix - Gradient Boosting")
plt.show()
In [35]:
# Gradient Boosting Normalized Confusion Matrix
cm_norm_gb = cm_gb.astype('float') / cm_gb.sum(axis=1)[:, np.newaxis]
plt.figure(figsize=(7,6))
sns.heatmap(cm_norm_gb, annot=True, fmt=".2f", cmap="Greens",
xticklabels=gb.classes_,
yticklabels=gb.classes_)
plt.xlabel("Predicted")
plt.ylabel("Actual")
plt.title("Normalized Confusion Matrix - Gradient Boosting")
plt.show()
In [36]:
# Feature Importances
fi_gb = pd.DataFrame({
"Feature": X_train.columns,
"Importance": gb.feature_importances_
}).sort_values(by="Importance", ascending=False).head(15)
plt.figure(figsize=(9,5))
plt.barh(fi_gb["Feature"], fi_gb["Importance"])
plt.gca().invert_yaxis()
plt.title("Top 15 Features - Gradient Boosting")
plt.tight_layout()
plt.show()
In [37]:
#Checking Performance
configs = [
(100, 0.1, 2),
(150, 0.1, 3),
]
for n, lr, md in configs:
print(f"\nTraining -> n={n}, lr={lr}, depth={md}")
gb = GradientBoostingClassifier(
n_estimators=n,
learning_rate=lr,
max_depth=md,
random_state=42
)
gb.fit(X_train_scaled, y_train)
y_pred = gb.predict(X_test_scaled)
print("Accuracy:", accuracy_score(y_test, y_pred))
Training -> n=100, lr=0.1, depth=2 Accuracy: 0.9937023708721423 Training -> n=150, lr=0.1, depth=3 Accuracy: 0.9759737510584251
In [38]:
models = ["Random Forest", "Gradient Boosting"]
accuracy = [0.93, 0.9353]
plt.bar(models, accuracy)
plt.ylabel("Accuracy")
plt.title("Model Accuracy Comparison")
plt.ylim(0.85, 1.0)
plt.show()
In [ ]: