NSL-KDD Intrusion Detection¶
IMPORT ALL LIBARARY¶
In [1]:
# Data manipulation
import pandas as pd
import numpy as np
# Visualisation
import matplotlib.pyplot as plt
import seaborn as sns
plt.rcParams['figure.dpi'] = 130
# Pre-processing
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.model_selection import train_test_split, StratifiedKFold, GridSearchCV
from sklearn.pipeline import Pipeline
# Ensemble models
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
# Evaluation metrics
from sklearn.metrics import (
accuracy_score, precision_score, recall_score, f1_score,
confusion_matrix, classification_report, ConfusionMatrixDisplay
)
# Reproducibility seed — ensures results are the same every run
SEED = 42
np.random.seed(SEED)
# Standard library
import os
os.makedirs('output', exist_ok=True) # create output directory for plots and results
import warnings
warnings.filterwarnings('ignore') # suppress minor sklearn warnings
Load the NSL-KDD Dataset¶
In [2]:
COL_NAMES = [
'duration','protocol_type','service','flag','src_bytes',
'dst_bytes','land','wrong_fragment','urgent','hot',
'num_failed_logins','logged_in','num_compromised','root_shell',
'su_attempted','num_root','num_file_creations','num_shells',
'num_access_files','num_outbound_cmds','is_host_login',
'is_guest_login','count','srv_count','serror_rate',
'srv_serror_rate','rerror_rate','srv_rerror_rate',
'same_srv_rate','diff_srv_rate','srv_diff_host_rate',
'dst_host_count','dst_host_srv_count','dst_host_same_srv_rate',
'dst_host_diff_srv_rate','dst_host_same_src_port_rate',
'dst_host_srv_diff_host_rate','dst_host_serror_rate',
'dst_host_srv_serror_rate','dst_host_rerror_rate',
'dst_host_srv_rerror_rate','label','difficulty_level'
]
In [3]:
def load_nslkdd(path):
# Peek at the very first value of the file
peek = pd.read_csv(path, header=None, nrows=1).iloc[0, 0]
has_header = (str(peek).strip().lower() == 'duration')
df = pd.read_csv(
path,
header = 0 if has_header else None,
names = None if has_header else COL_NAMES
)
# Rename columns if file had its own header
if has_header:
df.columns = COL_NAMES[:len(df.columns)]
return df
df_train = load_nslkdd('kdd_train.csv')
df_test = load_nslkdd('kdd_test.csv')
In [4]:
# Drop metadata column
for df in [df_train, df_test]:
if 'difficulty_level' in df.columns:
df.drop(columns=['difficulty_level'], inplace=True)
In [5]:
# force all feature columns to numeric Ensures no stray strings survive into X_train / X_test
CAT_COLS = ['protocol_type', 'service', 'flag', 'label']
NUM_COLS = [c for c in df_train.columns if c not in CAT_COLS]
for col in NUM_COLS:
df_train[col] = pd.to_numeric(df_train[col], errors='coerce')
df_test[col] = pd.to_numeric(df_test[col], errors='coerce')
In [6]:
# Fill any NaN produced by coercion with column median
df_train[NUM_COLS] = df_train[NUM_COLS].fillna(df_train[NUM_COLS].median())
df_test[NUM_COLS] = df_test[NUM_COLS].fillna(df_train[NUM_COLS].median())
print(f"Train : {df_train.shape} | Test : {df_test.shape}")
print(f"Dtypes:\n{df_train.dtypes.value_counts()}")
print("Sample row:\n", df_train.head(2))
Train : (125973, 42) | Test : (22544, 42)
Dtypes:
int64 23
float64 15
object 4
Name: count, dtype: int64
Sample row:
duration protocol_type service flag src_bytes dst_bytes land \
0 0 tcp ftp_data SF 491 0 0
1 0 udp other SF 146 0 0
wrong_fragment urgent hot ... dst_host_srv_count \
0 0 0 0 ... 25
1 0 0 0 ... 1
dst_host_same_srv_rate dst_host_diff_srv_rate \
0 0.17 0.03
1 0.00 0.60
dst_host_same_src_port_rate dst_host_srv_diff_host_rate \
0 0.17 0.0
1 0.88 0.0
dst_host_serror_rate dst_host_srv_serror_rate dst_host_rerror_rate \
0 0.0 0.0 0.05
1 0.0 0.0 0.00
dst_host_srv_rerror_rate label
0 0.0 normal
1 0.0 normal
[2 rows x 42 columns]
Exploratory Data Analysis (EDA)¶
In [7]:
# Basic statistics
print("=== Data Types ===")
print(df_train.dtypes.value_counts())
print("\n=== Missing Values ===")
print(df_train.isnull().sum().sum(), "missing values found")
print("\n=== Descriptive Statistics (numeric) ===")
print(df_train.describe().T.round(2))
=== Data Types ===
int64 23
float64 15
object 4
Name: count, dtype: int64
=== Missing Values ===
0 missing values found
=== Descriptive Statistics (numeric) ===
count mean std min 25% \
duration 125973.0 287.14 2604.52 0.0 0.00
src_bytes 125973.0 45566.74 5870331.18 0.0 0.00
dst_bytes 125973.0 19779.11 4021269.15 0.0 0.00
land 125973.0 0.00 0.01 0.0 0.00
wrong_fragment 125973.0 0.02 0.25 0.0 0.00
urgent 125973.0 0.00 0.01 0.0 0.00
hot 125973.0 0.20 2.15 0.0 0.00
num_failed_logins 125973.0 0.00 0.05 0.0 0.00
logged_in 125973.0 0.40 0.49 0.0 0.00
num_compromised 125973.0 0.28 23.94 0.0 0.00
root_shell 125973.0 0.00 0.04 0.0 0.00
su_attempted 125973.0 0.00 0.05 0.0 0.00
num_root 125973.0 0.30 24.40 0.0 0.00
num_file_creations 125973.0 0.01 0.48 0.0 0.00
num_shells 125973.0 0.00 0.02 0.0 0.00
num_access_files 125973.0 0.00 0.10 0.0 0.00
num_outbound_cmds 125973.0 0.00 0.00 0.0 0.00
is_host_login 125973.0 0.00 0.00 0.0 0.00
is_guest_login 125973.0 0.01 0.10 0.0 0.00
count 125973.0 84.11 114.51 0.0 2.00
srv_count 125973.0 27.74 72.64 0.0 2.00
serror_rate 125973.0 0.28 0.45 0.0 0.00
srv_serror_rate 125973.0 0.28 0.45 0.0 0.00
rerror_rate 125973.0 0.12 0.32 0.0 0.00
srv_rerror_rate 125973.0 0.12 0.32 0.0 0.00
same_srv_rate 125973.0 0.66 0.44 0.0 0.09
diff_srv_rate 125973.0 0.06 0.18 0.0 0.00
srv_diff_host_rate 125973.0 0.10 0.26 0.0 0.00
dst_host_count 125973.0 182.15 99.21 0.0 82.00
dst_host_srv_count 125973.0 115.65 110.70 0.0 10.00
dst_host_same_srv_rate 125973.0 0.52 0.45 0.0 0.05
dst_host_diff_srv_rate 125973.0 0.08 0.19 0.0 0.00
dst_host_same_src_port_rate 125973.0 0.15 0.31 0.0 0.00
dst_host_srv_diff_host_rate 125973.0 0.03 0.11 0.0 0.00
dst_host_serror_rate 125973.0 0.28 0.44 0.0 0.00
dst_host_srv_serror_rate 125973.0 0.28 0.45 0.0 0.00
dst_host_rerror_rate 125973.0 0.12 0.31 0.0 0.00
dst_host_srv_rerror_rate 125973.0 0.12 0.32 0.0 0.00
50% 75% max
duration 0.00 0.00 4.290800e+04
src_bytes 44.00 276.00 1.379964e+09
dst_bytes 0.00 516.00 1.309937e+09
land 0.00 0.00 1.000000e+00
wrong_fragment 0.00 0.00 3.000000e+00
urgent 0.00 0.00 3.000000e+00
hot 0.00 0.00 7.700000e+01
num_failed_logins 0.00 0.00 5.000000e+00
logged_in 0.00 1.00 1.000000e+00
num_compromised 0.00 0.00 7.479000e+03
root_shell 0.00 0.00 1.000000e+00
su_attempted 0.00 0.00 2.000000e+00
num_root 0.00 0.00 7.468000e+03
num_file_creations 0.00 0.00 4.300000e+01
num_shells 0.00 0.00 2.000000e+00
num_access_files 0.00 0.00 9.000000e+00
num_outbound_cmds 0.00 0.00 0.000000e+00
is_host_login 0.00 0.00 1.000000e+00
is_guest_login 0.00 0.00 1.000000e+00
count 14.00 143.00 5.110000e+02
srv_count 8.00 18.00 5.110000e+02
serror_rate 0.00 1.00 1.000000e+00
srv_serror_rate 0.00 1.00 1.000000e+00
rerror_rate 0.00 0.00 1.000000e+00
srv_rerror_rate 0.00 0.00 1.000000e+00
same_srv_rate 1.00 1.00 1.000000e+00
diff_srv_rate 0.00 0.06 1.000000e+00
srv_diff_host_rate 0.00 0.00 1.000000e+00
dst_host_count 255.00 255.00 2.550000e+02
dst_host_srv_count 63.00 255.00 2.550000e+02
dst_host_same_srv_rate 0.51 1.00 1.000000e+00
dst_host_diff_srv_rate 0.02 0.07 1.000000e+00
dst_host_same_src_port_rate 0.00 0.06 1.000000e+00
dst_host_srv_diff_host_rate 0.00 0.02 1.000000e+00
dst_host_serror_rate 0.00 1.00 1.000000e+00
dst_host_srv_serror_rate 0.00 1.00 1.000000e+00
dst_host_rerror_rate 0.00 0.00 1.000000e+00
dst_host_srv_rerror_rate 0.00 0.00 1.000000e+00
In [8]:
# Map fine-grained attack labels → 5 super-categories
dos_attacks = {'back','land','neptune','pod','smurf','teardrop',
'apache2','udpstorm','processtable','worm','mailbomb'}
probe_attacks = {'ipsweep','nmap','portsweep','satan','mscan','saint'}
r2l_attacks = {'ftp_write','guess_passwd','imap','multihop','phf',
'spy','warezclient','warezmaster','sendmail','named',
'snmpgetattack','snmpguess','xlock','xsnoop','httptunnel'}
u2r_attacks = {'buffer_overflow','loadmodule','perl','rootkit',
'ps','sqlattack','xterm'}
def map_label(label):
"""Convert raw NSL-KDD label to 5-class category."""
label = label.lower().strip()
if label == 'normal': return 'Normal'
elif label in dos_attacks: return 'DoS'
elif label in probe_attacks: return 'Probe'
elif label in r2l_attacks: return 'R2L'
elif label in u2r_attacks: return 'U2R'
else: return 'DoS'
for df in [df_train, df_test]:
df['label'] = df['label'].apply(map_label)
In [9]:
# Class distribution plot
fig, axes = plt.subplots(1, 2, figsize=(14, 5))
for ax, df, title in zip(axes,
[df_train, df_test],
['Training Set', 'Test Set']):
counts = df['label'].value_counts()
sns.barplot(x=counts.index, y=counts.values,
palette='viridis', ax=ax)
ax.set_title(f"{title} — Class Distribution", fontsize=13)
ax.set_xlabel('Attack Category')
ax.set_ylabel('Sample Count')
for bar in ax.patches:
ax.text(bar.get_x() + bar.get_width()/2,
bar.get_height() + 100,
f"{int(bar.get_height()):,}",
ha='center', va='bottom', fontsize=9)
plt.tight_layout()
plt.show()
In [10]:
# Correlation heatmap (numeric features)
df_train_numeric = df_train.drop(columns=['label', 'protocol_type',
'service', 'flag'],
errors='ignore')
In [11]:
# Coerce to numeric; non-numeric cells become NaN
df_train_numeric = df_train_numeric.apply(pd.to_numeric, errors='coerce')
df_train_numeric = df_train_numeric.dropna(axis=1, how='all') # drop all-NaN cols
print(f"Numeric columns found: {len(df_train_numeric.columns)}") # should be ~38
if df_train_numeric.empty or df_train_numeric.shape[1] < 2:
print("No numeric columns — skipping heatmap.")
else:
corr = df_train_numeric.corr()
fig, ax = plt.subplots(figsize=(18, 14))
sns.heatmap(corr, cmap='coolwarm', center=0, linewidths=.3,
ax=ax, annot=False)
ax.set_title('Feature Correlation Heatmap', fontsize=14)
plt.tight_layout()
plt.show()
Numeric columns found: 38
Data Preprocessing¶
In [12]:
# Preprocessing: Categorical feature encoding
CAT_COLS = ['protocol_type', 'service', 'flag']
cat_encoders = {}
for col in CAT_COLS:
le = LabelEncoder()
combined = pd.concat([df_train[col], df_test[col]])
le.fit(combined)
df_train[col] = le.transform(df_train[col])
df_test[col] = le.transform(df_test[col])
cat_encoders[col] = le
print(f"{col}: {le.classes_}")
protocol_type: ['icmp' 'tcp' 'udp'] service: ['IRC' 'X11' 'Z39_50' 'aol' 'auth' 'bgp' 'courier' 'csnet_ns' 'ctf' 'daytime' 'discard' 'domain' 'domain_u' 'echo' 'eco_i' 'ecr_i' 'efs' 'exec' 'finger' 'ftp' 'ftp_data' 'gopher' 'harvest' 'hostnames' 'http' 'http_2784' 'http_443' 'http_8001' 'imap4' 'iso_tsap' 'klogin' 'kshell' 'ldap' 'link' 'login' 'mtp' 'name' 'netbios_dgm' 'netbios_ns' 'netbios_ssn' 'netstat' 'nnsp' 'nntp' 'ntp_u' 'other' 'pm_dump' 'pop_2' 'pop_3' 'printer' 'private' 'red_i' 'remote_job' 'rje' 'shell' 'smtp' 'sql_net' 'ssh' 'sunrpc' 'supdup' 'systat' 'telnet' 'tftp_u' 'tim_i' 'time' 'urh_i' 'urp_i' 'uucp' 'uucp_path' 'vmnet' 'whois'] flag: ['OTH' 'REJ' 'RSTO' 'RSTOS0' 'RSTR' 'S0' 'S1' 'S2' 'S3' 'SF' 'SH']
In [13]:
# Target label encoding
label_enc = LabelEncoder()
all_labels = pd.concat([df_train['label'], df_test['label']])
label_enc.fit(all_labels)
CLASS_NAMES = label_enc.classes_
print("\nClasses:", CLASS_NAMES)
Classes: ['DoS' 'Normal' 'Probe' 'R2L' 'U2R']
In [14]:
# Feature / target split
X_train = df_train.drop(columns=['label']).values
y_train = label_enc.transform(df_train['label'])
In [15]:
X_test = df_test.drop(columns=['label']).values
y_test = label_enc.transform(df_test['label'])
In [16]:
# Feature normalisation (StandardScaler)
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train) # fit + transform train
X_test = scaler.transform(X_test)
print(f"\nX_train shape : {X_train.shape}")
print(f"X_test shape : {X_test.shape}")
X_train shape : (125973, 41) X_test shape : (22544, 41)
Random Forest Classifier & Hyperparameter Tuning¶
In [17]:
# Define hyperparameter search grid
rf_param_grid = {
'n_estimators' : [100, 200], # tree count candidates
'max_depth' : [None, 20, 30], # None = grow fully
'min_samples_split': [2, 5], # minimum split size
'max_features' : ['sqrt', 'log2'], # feature subset strategy
}
In [18]:
# Stratified k-fold cross-validation
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)
In [19]:
# Grid search
rf_base = RandomForestClassifier(random_state=SEED, n_jobs=-1)
rf_grid = GridSearchCV(
estimator = rf_base,
param_grid = rf_param_grid,
cv = cv,
scoring = 'f1_weighted',
n_jobs = -1,
verbose = 1
)
rf_grid.fit(X_train, y_train)
Fitting 5 folds for each of 24 candidates, totalling 120 fits
Out[19]:
GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),
estimator=RandomForestClassifier(n_jobs=-1, random_state=42),
n_jobs=-1,
param_grid={'max_depth': [None, 20, 30],
'max_features': ['sqrt', 'log2'],
'min_samples_split': [2, 5],
'n_estimators': [100, 200]},
scoring='f1_weighted', verbose=1)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
Parameters
| estimator | RandomForestC...ndom_state=42) | |
| param_grid | {'max_depth': [None, 20, ...], 'max_features': ['sqrt', 'log2'], 'min_samples_split': [2, 5], 'n_estimators': [100, 200]} | |
| scoring | 'f1_weighted' | |
| n_jobs | -1 | |
| refit | True | |
| cv | StratifiedKFo... shuffle=True) | |
| verbose | 1 | |
| pre_dispatch | '2*n_jobs' | |
| error_score | nan | |
| return_train_score | False |
RandomForestClassifier(n_estimators=200, n_jobs=-1, random_state=42)
Parameters
| n_estimators | 200 | |
| criterion | 'gini' | |
| max_depth | None | |
| min_samples_split | 2 | |
| min_samples_leaf | 1 | |
| min_weight_fraction_leaf | 0.0 | |
| max_features | 'sqrt' | |
| max_leaf_nodes | None | |
| min_impurity_decrease | 0.0 | |
| bootstrap | True | |
| oob_score | False | |
| n_jobs | -1 | |
| random_state | 42 | |
| verbose | 0 | |
| warm_start | False | |
| class_weight | None | |
| ccp_alpha | 0.0 | |
| max_samples | None | |
| monotonic_cst | None |
In [20]:
# Best model
best_rf = rf_grid.best_estimator_
print("Best RF hyperparameters:", rf_grid.best_params_)
print(f"Best CV F1 (weighted): {rf_grid.best_score_:.4f}")
Best RF hyperparameters: {'max_depth': None, 'max_features': 'sqrt', 'min_samples_split': 2, 'n_estimators': 200}
Best CV F1 (weighted): 0.9988
Model Performance Evaluation & Comparison¶
In [21]:
# We evaluate both models using five complementary metrics for multi-class classification
import time
best_gb = GradientBoostingClassifier(
n_estimators=100,
learning_rate=0.1,
max_depth=3,
random_state=42
)
best_gb.fit(X_train, y_train)
def evaluate_model(model, X_tr, y_tr, X_te, y_te, name):
"""
Fit model on training data, predict on test data,
and return a metrics dictionary with timing.
"""
# Training time measurement
t0 = time.time()
model.fit(X_tr, y_tr)
train_time = time.time() - t0
# Inference
t0 = time.time()
y_pred = model.predict(X_te)
infer_time = time.time() - t0
# Metrics
acc = accuracy_score(y_te, y_pred)
pr = precision_score(y_te, y_pred, average='weighted', zero_division=0)
rc = recall_score(y_te, y_pred, average='weighted', zero_division=0)
f1 = f1_score(y_te, y_pred, average='weighted', zero_division=0)
print(f"\n{'='*55}")
print(f" {name}")
print(f"{'='*55}")
print(f" Accuracy : {acc:.4f}")
print(f" Precision : {pr:.4f} (weighted)")
print(f" Recall : {rc:.4f} (weighted)")
print(f" F1-Score : {f1:.4f} (weighted)")
print(f" Train time: {train_time:.2f}s")
print(f" Infer time: {infer_time:.4f}s\n")
print(classification_report(y_te, y_pred,
target_names=CLASS_NAMES,
zero_division=0))
return {
'model': name, 'acc': acc, 'prec': pr, 'rec': rc, 'f1': f1,
'train_time': train_time, 'infer_time': infer_time,
'y_pred': y_pred
}
# Run evaluations
rf_results = evaluate_model(best_rf, X_train, y_train,
X_test, y_test, 'Random Forest')
gb_results = evaluate_model(best_gb, X_train, y_train,
X_test, y_test, 'Gradient Boosting')
# Confusion matrices
fig, axes = plt.subplots(1, 2, figsize=(16, 6))
for ax, res, title in zip(axes,
[rf_results, gb_results],
['Random Forest', 'Gradient Boosting']):
cm = confusion_matrix(y_test, res['y_pred'])
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
xticklabels=CLASS_NAMES, yticklabels=CLASS_NAMES, ax=ax)
ax.set_title(f"{title} — Confusion Matrix", fontsize=13)
ax.set_xlabel('Predicted'); ax.set_ylabel('Actual')
plt.tight_layout()
plt.show()
=======================================================
Random Forest
=======================================================
Accuracy : 0.9207
Precision : 0.9288 (weighted)
Recall : 0.9207 (weighted)
F1-Score : 0.9074 (weighted)
Train time: 22.23s
Infer time: 0.2919s
precision recall f1-score support
DoS 0.99 0.93 0.96 8095
Normal 0.87 0.99 0.93 11245
Probe 0.96 0.88 0.92 2157
R2L 0.99 0.14 0.24 1009
U2R 1.00 0.32 0.48 38
accuracy 0.92 22544
macro avg 0.96 0.65 0.71 22544
weighted avg 0.93 0.92 0.91 22544
=======================================================
Gradient Boosting
=======================================================
Accuracy : 0.9264
Precision : 0.9319 (weighted)
Recall : 0.9264 (weighted)
F1-Score : 0.9149 (weighted)
Train time: 237.25s
Infer time: 0.3096s
precision recall f1-score support
DoS 0.99 0.95 0.97 8095
Normal 0.88 0.99 0.93 11245
Probe 0.95 0.87 0.91 2157
R2L 0.98 0.18 0.31 1009
U2R 0.76 0.34 0.47 38
accuracy 0.93 22544
macro avg 0.91 0.67 0.72 22544
weighted avg 0.93 0.93 0.91 22544
In [22]:
# Metric comparison bar chart
metrics = ['Accuracy', 'Precision', 'Recall', 'F1-Score']
rf_vals = [rf_results[k] for k in ['acc','prec','rec','f1']]
gb_vals = [gb_results[k] for k in ['acc','prec','rec','f1']]
x = np.arange(len(metrics))
width = 0.35
fig, ax = plt.subplots(figsize=(10, 5))
bars1 = ax.bar(x - width/2, rf_vals, width, label='Random Forest', color='#4C8BE0')
bars2 = ax.bar(x + width/2, gb_vals, width, label='Gradient Boosting', color='#E07B4C')
ax.set_xticks(x); ax.set_xticklabels(metrics)
ax.set_ylim(0.80, 1.0)
ax.set_ylabel('Score'); ax.set_title('Model Metric Comparison (Test Set)')
ax.legend()
ax.bar_label(bars1, fmt='%.3f', padding=3, fontsize=9)
ax.bar_label(bars2, fmt='%.3f', padding=3, fontsize=9)
plt.tight_layout()
plt.show()
Overfitting / Underfitting Analysi¶
In [23]:
# Overfitting : model memorises training data but fails on new data => large gap between train and test accuracy
# Underfitting : model is too simple to capture patterns => low train AND test accuracy
def train_test_f1_gap(model, X_tr, y_tr, X_te, y_te, name):
"""Compute F1 on train AND test; report gap."""
f1_tr = f1_score(y_tr, model.predict(X_tr), average='weighted')
f1_te = f1_score(y_te, model.predict(X_te), average='weighted')
gap = f1_tr - f1_te
print(f"{name:25s} Train F1={f1_tr:.4f} Test F1={f1_te:.4f} Gap={gap:.4f}")
return f1_tr, f1_te
print("=== Overfitting Diagnostic ===")
rf_tr, rf_te = train_test_f1_gap(best_rf, X_train, y_train,
X_test, y_test, 'Random Forest')
gb_tr, gb_te = train_test_f1_gap(best_gb, X_train, y_train,
X_test, y_test, 'Gradient Boosting')
=== Overfitting Diagnostic === Random Forest Train F1=0.9999 Test F1=0.9074 Gap=0.0925 Gradient Boosting Train F1=0.9978 Test F1=0.9149 Gap=0.0829
In [24]:
# Visual: learning curve proxy
labels_ov = ['RF Train', 'RF Test', 'GB Train', 'GB Test']
vals_ov = [rf_tr, rf_te, gb_tr, gb_te]
colors_ov = ['#4C8BE0', '#8BB8F5', '#E07B4C', '#F5B38B']
fig, ax = plt.subplots(figsize=(8, 5))
bars = ax.bar(labels_ov, vals_ov, color=colors_ov)
ax.set_ylim(0.8, 1.01)
ax.set_ylabel('Weighted F1-Score')
ax.set_title('Train vs Test F1 — Overfitting Analysis')
ax.bar_label(bars, fmt='%.4f', padding=3)
plt.tight_layout()
plt.show()
Feature Importance Analysis¶
In [25]:
# Both RF and GB expose feature_importances
FEATURE_NAMES = (df_train.drop(columns=['label']).columns.tolist())
def plot_feature_importance(model, feature_names, title, top_n=15):
importances = model.feature_importances_
indices = np.argsort(importances)[::-1][:top_n] # top N descending
top_names = [feature_names[i] for i in indices]
top_vals = importances[indices]
fig, ax = plt.subplots(figsize=(10, 6))
sns.barplot(x=top_vals, y=top_names, palette='magma', ax=ax)
ax.set_title(f"{title} — Top {top_n} Feature Importances", fontsize=13)
ax.set_xlabel('Mean Decrease in Impurity (Importance Score)')
ax.set_ylabel('Feature')
plt.tight_layout()
filename = title.lower().replace(' ', '_') + '_feature_importance.png'
plt.savefig(f'output/{filename}', bbox_inches='tight')
plt.show()
return dict(zip(top_names, top_vals))
rf_fi = plot_feature_importance(best_rf, FEATURE_NAMES, 'Random Forest')
gb_fi = plot_feature_importance(best_gb, FEATURE_NAMES, 'Gradient Boosting')
In [26]:
# Shared top features across both models
shared_top10 = set(list(rf_fi)[:10]) & set(list(gb_fi)[:10])
print(f"\nFeatures in BOTH models' top-10: {shared_top10}")
Features in BOTH models' top-10: {'dst_host_diff_srv_rate', 'count', 'same_srv_rate', 'dst_bytes', 'dst_host_serror_rate', 'src_bytes', 'diff_srv_rate'}
Final Comparison Summary¶
In [29]:
# Print a formatted comparison table of all evaluation metrics and computational efficiency for the two models.
summary = pd.DataFrame([
{'Model' : 'Random Forest',
'Accuracy' : f"{rf_results['acc']:.4f}",
'Precision' : f"{rf_results['prec']:.4f}",
'Recall' : f"{rf_results['rec']:.4f}",
'F1-Score' : f"{rf_results['f1']:.4f}",
'Train (s)' : f"{rf_results['train_time']:.2f}",
'Infer (s)' : f"{rf_results['infer_time']:.4f}"},
{'Model' : 'Gradient Boosting',
'Accuracy' : f"{gb_results['acc']:.4f}",
'Precision' : f"{gb_results['prec']:.4f}",
'Recall' : f"{gb_results['rec']:.4f}",
'F1-Score' : f"{gb_results['f1']:.4f}",
'Train (s)' : f"{gb_results['train_time']:.2f}",
'Infer (s)' : f"{gb_results['infer_time']:.4f}"},
])
print("\n=== Final Model Comparison ===")
print(summary.to_string(index=False))
summary.to_csv('output/model_comparison_summary.csv', index=False)
=== Final Model Comparison ===
Model Accuracy Precision Recall F1-Score Train (s) Infer (s)
Random Forest 0.9207 0.9288 0.9207 0.9074 22.23 0.2919
Gradient Boosting 0.9264 0.9319 0.9264 0.9149 237.25 0.3096
In [ ]: