{ "cells": [ { "cell_type": "markdown", "id": "310c258e", "metadata": {}, "source": [ "# Part 1:- Machine Learning Activity: Feature Selection and Model Comparison\n", "\n", "In this notebook, you will train a **Random Forest classifier** using the **Diabetes** or **Titanic** dataset.\n", "You will analyze the effect of **feature selection** and compare **Random Forest, Logistic Regression, and Naïve Bayes classifiers**.\n", "\n", "\n" ] }, { "cell_type": "markdown", "id": "fa8e49f7-96f9-4eed-8559-0d6877397345", "metadata": {}, "source": [ "## Step 1. Load Diabetes dataset " ] }, { "cell_type": "code", "execution_count": 88, "id": "f77f9100", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "RangeIndex: 768 entries, 0 to 767\n", "Data columns (total 10 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 PatientId 768 non-null int64 \n", " 1 Pregnancies 768 non-null int64 \n", " 2 Glucose 768 non-null int64 \n", " 3 BloodPressure 768 non-null int64 \n", " 4 SkinThickness 768 non-null int64 \n", " 5 Insulin 768 non-null int64 \n", " 6 BMI 768 non-null float64\n", " 7 DiabetesPedigreeFunction 768 non-null float64\n", " 8 Age 768 non-null int64 \n", " 9 Outcome 768 non-null int64 \n", "dtypes: float64(2), int64(8)\n", "memory usage: 60.1 KB\n", "None\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
PatientIdPregnanciesGlucoseBloodPressureSkinThicknessInsulinBMIDiabetesPedigreeFunctionAgeOutcome
0161487235033.60.627501
121856629026.60.351310
238183640023.30.672321
3418966239428.10.167210
450137403516843.12.288331
\n", "
" ], "text/plain": [ " PatientId Pregnancies Glucose BloodPressure SkinThickness Insulin \\\n", "0 1 6 148 72 35 0 \n", "1 2 1 85 66 29 0 \n", "2 3 8 183 64 0 0 \n", "3 4 1 89 66 23 94 \n", "4 5 0 137 40 35 168 \n", "\n", " BMI DiabetesPedigreeFunction Age Outcome \n", "0 33.6 0.627 50 1 \n", "1 26.6 0.351 31 0 \n", "2 23.3 0.672 32 1 \n", "3 28.1 0.167 21 0 \n", "4 43.1 2.288 33 1 " ] }, "execution_count": 88, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pandas as pd\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.ensemble import RandomForestClassifier\n", "from sklearn.decomposition import PCA\n", "from sklearn.linear_model import LogisticRegression\n", "from sklearn.naive_bayes import GaussianNB\n", "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, classification_report\n", "from sklearn.decomposition import PCA\n", "from sklearn.preprocessing import StandardScaler\n", "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n", "\n", "# Load dataset (Uncomment the one you want to use)\n", "df = pd.read_csv('diabetes.csv') # Diabetes dataset\n", "#df = pd.read_csv('titanic.csv') # Titanic dataset\n", "\n", "# Display basic info\n", "print(df.info())\n", "\n", "df.head()\n", "\n" ] }, { "cell_type": "markdown", "id": "eb998120-f778-43c9-bba1-d6a5d50bdb86", "metadata": {}, "source": [ "## Step 2. check missing values in this dataset " ] }, { "cell_type": "code", "execution_count": 91, "id": "ef428d8d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Missing Values in Each Column:\n", " PatientId 0\n", "Pregnancies 0\n", "Glucose 0\n", "BloodPressure 0\n", "SkinThickness 0\n", "Insulin 0\n", "BMI 0\n", "DiabetesPedigreeFunction 0\n", "Age 0\n", "Outcome 0\n", "dtype: int64\n" ] } ], "source": [ "# Check for missing values\n", "print(\"\\nMissing Values in Each Column:\\n\", df.isnull().sum())" ] }, { "cell_type": "markdown", "id": "e604c858-59dc-4df7-a47c-8220802a123d", "metadata": {}, "source": [ "## Handling missing values " ] }, { "cell_type": "code", "execution_count": 94, "id": "550a3571-4b48-401b-a027-795f7c258ae8", "metadata": {}, "outputs": [], "source": [ "# Replace missing values with column mean (if any)\n", "df.fillna(df.mean(), inplace=True)" ] }, { "cell_type": "markdown", "id": "1809a72b-39eb-42b1-8f9d-cdbf4b908c1d", "metadata": {}, "source": [ "## Step 3. Data Overview and Preprocessing" ] }, { "cell_type": "code", "execution_count": 97, "id": "5dd4e53d-d190-414d-a330-265594a4a808", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Dataset contains 768 rows and 10 columns.\n", "\n", "Data Types:\n", " PatientId int64\n", "Pregnancies int64\n", "Glucose int64\n", "BloodPressure int64\n", "SkinThickness int64\n", "Insulin int64\n", "BMI float64\n", "DiabetesPedigreeFunction float64\n", "Age int64\n", "Outcome int64\n", "dtype: object\n", "\n", "Summary Statistics:\n", " PatientId Pregnancies Glucose BloodPressure SkinThickness \\\n", "count 768.000000 768.000000 768.000000 768.000000 768.000000 \n", "mean 384.500000 3.845052 120.894531 69.105469 20.536458 \n", "std 221.846794 3.369578 31.972618 19.355807 15.952218 \n", "min 1.000000 0.000000 0.000000 0.000000 0.000000 \n", "25% 192.750000 1.000000 99.000000 62.000000 0.000000 \n", "50% 384.500000 3.000000 117.000000 72.000000 23.000000 \n", "75% 576.250000 6.000000 140.250000 80.000000 32.000000 \n", "max 768.000000 17.000000 199.000000 122.000000 99.000000 \n", "\n", " Insulin BMI DiabetesPedigreeFunction Age \\\n", "count 768.000000 768.000000 768.000000 768.000000 \n", "mean 79.799479 31.992578 0.471876 33.240885 \n", "std 115.244002 7.884160 0.331329 11.760232 \n", "min 0.000000 0.000000 0.078000 21.000000 \n", "25% 0.000000 27.300000 0.243750 24.000000 \n", "50% 30.500000 32.000000 0.372500 29.000000 \n", "75% 127.250000 36.600000 0.626250 41.000000 \n", "max 846.000000 67.100000 2.420000 81.000000 \n", "\n", " Outcome \n", "count 768.000000 \n", "mean 0.348958 \n", "std 0.476951 \n", "min 0.000000 \n", "25% 0.000000 \n", "50% 0.000000 \n", "75% 1.000000 \n", "max 1.000000 \n" ] } ], "source": [ "# Check dataset shape\n", "print(f\"Dataset contains {df.shape[0]} rows and {df.shape[1]} columns.\")\n", "\n", "# Check column data types\n", "print(\"\\nData Types:\\n\", df.dtypes)\n", "\n", "# Summary statistics of numerical features\n", "print(\"\\nSummary Statistics:\\n\", df.describe())" ] }, { "cell_type": "markdown", "id": "84790066-e8eb-445a-807a-d6d0ef0e72d9", "metadata": {}, "source": [ "## Step 4. Check for Zero Values in Key Features" ] }, { "cell_type": "code", "execution_count": 100, "id": "58fb4844-600a-4763-a7ec-ad7dafefc80d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Zero Value Check After Replacement:\n", " Glucose 0\n", "BloodPressure 0\n", "SkinThickness 0\n", "Insulin 0\n", "BMI 0\n", "dtype: int64\n" ] } ], "source": [ "# List of features where zero is not a valid value\n", "invalid_zero_features = [\"Glucose\", \"BloodPressure\", \"SkinThickness\", \"Insulin\", \"BMI\"]\n", "\n", "# Replace zero values with the column mean (except for Outcome)\n", "for column in invalid_zero_features:\n", " df[column] = df[column].replace(0, df[column].mean())\n", "\n", "# Confirm no more zero values\n", "print(\"\\nZero Value Check After Replacement:\\n\", (df[invalid_zero_features] == 0).sum())" ] }, { "cell_type": "markdown", "id": "2c9ce2a8-17ba-4f26-9ed4-5a7f4efa99b0", "metadata": {}, "source": [ "## Step 5. Using techniques & trains the Random Forest classifier (Split the Data Features (X) and Target (y))" ] }, { "cell_type": "code", "execution_count": 103, "id": "7b411b30-aba0-448f-92b9-5fb4564fbab9", "metadata": {}, "outputs": [], "source": [ "# Define features and target variable\n", "X = df.drop(columns=['Outcome']) # Features\n", "y = df['Outcome'] # Target" ] }, { "cell_type": "code", "execution_count": 105, "id": "d3017a2a-d36c-4bc4-ab37-3fb7ab9d4cd6", "metadata": {}, "outputs": [], "source": [ "# Split into 80% train and 20% test data\n", "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n" ] }, { "cell_type": "code", "execution_count": 107, "id": "36bb2ed2-3f6d-4e71-a573-1940099e6514", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Training set shape: (614, 9), Testing set shape: (154, 9)\n" ] } ], "source": [ "# Print shape of training and testing sets\n", "print(f\"Training set shape: {X_train.shape}, Testing set shape: {X_test.shape}\")" ] }, { "cell_type": "markdown", "id": "56157b51-5ea0-4888-abc4-2517190c341c", "metadata": {}, "source": [ "## Step 5.1 Train a Random Forest Classifier" ] }, { "cell_type": "code", "execution_count": 110, "id": "c56a5a7c-8a2a-47c9-a144-71cf45c4171e", "metadata": {}, "outputs": [], "source": [ "# Initialize Random Forest model\n", "rf_model = RandomForestClassifier(n_estimators=100, random_state=42)" ] }, { "cell_type": "code", "execution_count": 112, "id": "af2e2f06-4726-4518-875f-5618ed300d36", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
RandomForestClassifier(random_state=42)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" ], "text/plain": [ "RandomForestClassifier(random_state=42)" ] }, "execution_count": 112, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Train the model\n", "rf_model.fit(X_train, y_train)" ] }, { "cell_type": "code", "execution_count": 114, "id": "175df083-5b4e-45f1-a674-c51c7cccede4", "metadata": {}, "outputs": [], "source": [ "# Make predictions on test set\n", "y_pred = rf_model.predict(X_test)" ] }, { "cell_type": "markdown", "id": "0ceba0eb-34e0-49cb-9608-166b5a607510", "metadata": {}, "source": [ "## Step 6. Evaluate Model Performance" ] }, { "cell_type": "code", "execution_count": 117, "id": "6f3c63fd-1852-43ef-b188-fe0b06f8d975", "metadata": {}, "outputs": [], "source": [ "# Compute evaluation metrics\n", "accuracy = accuracy_score(y_test, y_pred)\n", "precision = precision_score(y_test, y_pred)\n", "recall = recall_score(y_test, y_pred)\n", "f1 = f1_score(y_test, y_pred)" ] }, { "cell_type": "code", "execution_count": 119, "id": "265ed632-ee7a-413a-9c31-372005adf341", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Classification Report:\n", " precision recall f1-score support\n", "\n", " 0 0.82 0.79 0.80 99\n", " 1 0.64 0.69 0.67 55\n", "\n", " accuracy 0.75 154\n", " macro avg 0.73 0.74 0.74 154\n", "weighted avg 0.76 0.75 0.76 154\n", "\n" ] } ], "source": [ "# Print the classification report\n", "print(\"\\nClassification Report:\\n\", classification_report(y_test, y_pred))" ] }, { "cell_type": "code", "execution_count": 170, "id": "975b6d22-0844-4630-a6a0-b21861dead85", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "👌Accuracy: 0.7532\n", "👌Precision: 0.6441\n", "👌Recall: 0.6909\n", "👌F1-score: 0.6667\n" ] } ], "source": [ "# Print key metrics\n", "print(f\"👌Accuracy: {accuracy:.3f}\")\n", "print(f\"👌Precision: {precision:.3f}\")\n", "print(f\"👌Recall: {recall:.3f}\")\n", "print(f\"👌F1-score: {f1:.3f}\")" ] }, { "cell_type": "markdown", "id": "9e87f768-2722-48ef-820d-f930603a4cea", "metadata": {}, "source": [ "## Step 7. Manually Select Important Features- \n", "### {Extract Feature Importance Scores}" ] }, { "cell_type": "code", "execution_count": 128, "id": "362d4e45-986e-44ef-894a-79b9cb231631", "metadata": {}, "outputs": [], "source": [ "# Extract feature importances\n", "feature_importances = rf_model.feature_importances_" ] }, { "cell_type": "code", "execution_count": 131, "id": "7f8dc3d5-19ce-4eb7-9d57-73779fed43f5", "metadata": {}, "outputs": [], "source": [ "# Create a DataFrame for visualization\n", "feature_importance_df = pd.DataFrame({'Feature': X.columns, 'Importance': feature_importances})\n" ] }, { "cell_type": "code", "execution_count": 134, "id": "ebca95bf-642a-4b9b-abb8-33e58b016406", "metadata": {}, "outputs": [], "source": [ "# Sort features by importance\n", "feature_importance_df = feature_importance_df.sort_values(by='Importance', ascending=False)" ] }, { "cell_type": "code", "execution_count": 138, "id": "91bdb40f-6d44-42fa-b732-139416813d38", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " Feature Importance\n", "2 Glucose 0.269512\n", "6 BMI 0.146728\n", "8 Age 0.125184\n", "7 DiabetesPedigreeFunction 0.106558\n", "0 PatientId 0.090439\n", "3 BloodPressure 0.071946\n", "5 Insulin 0.069960\n", "1 Pregnancies 0.062901\n", "4 SkinThickness 0.056772\n" ] } ], "source": [ "# Display feature importances\n", "print(feature_importance_df)" ] }, { "cell_type": "markdown", "id": "6c062b6a-f629-4eda-889c-ccc58c4326ec", "metadata": {}, "source": [ "## Step 8. Select the Top N Features" ] }, { "cell_type": "code", "execution_count": 144, "id": "7a0a0630-1197-421f-8121-0843ef8da0cf", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Selected Top 5 Features: ['Glucose', 'BMI', 'Age', 'DiabetesPedigreeFunction', 'PatientId']\n" ] } ], "source": [ "# Select the top N features\n", "top_n = 5 # Change this value if needed\n", "top_features = feature_importance_df['Feature'][:top_n].tolist()\n", "\n", "print(f\"Selected Top {top_n} Features: {top_features}\")" ] }, { "cell_type": "markdown", "id": "3c2f636e-db71-4654-b26e-8208c9210ff5", "metadata": {}, "source": [ "## Step 9. Retrain the Model with Selected Features" ] }, { "cell_type": "code", "execution_count": 149, "id": "c2adf5a3-7d2c-4f22-b463-3b191b41f4fa", "metadata": {}, "outputs": [], "source": [ "# Create a new dataset with only top N features\n", "X_train_selected = X_train[top_features]\n", "X_test_selected = X_test[top_features]" ] }, { "cell_type": "code", "execution_count": 152, "id": "74692416-09ae-4b0f-b2cb-33bb0e31d4fa", "metadata": {}, "outputs": [], "source": [ "# Initialize a new Random Forest model\n", "rf_model_selected = RandomForestClassifier(n_estimators=100, random_state=42)" ] }, { "cell_type": "code", "execution_count": 155, "id": "8e9da041-6a92-43f6-baff-6fe789dbd9df", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
RandomForestClassifier(random_state=42)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" ], "text/plain": [ "RandomForestClassifier(random_state=42)" ] }, "execution_count": 155, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Train the model on selected features\n", "rf_model_selected.fit(X_train_selected, y_train)" ] }, { "cell_type": "code", "execution_count": 158, "id": "a9a6d61b-33fa-4f0f-960b-be315bc64934", "metadata": {}, "outputs": [], "source": [ "# Make predictions\n", "y_pred_selected = rf_model_selected.predict(X_test_selected)" ] }, { "cell_type": "markdown", "id": "f6e46709-311b-4fcf-a0dc-1069a6217192", "metadata": {}, "source": [ "## Step 10. Compare Model Performance Before and After Feature Selection" ] }, { "cell_type": "code", "execution_count": 164, "id": "893c325c-9ed3-4582-a0ab-0fc036774b0c", "metadata": {}, "outputs": [], "source": [ "# Compute new evaluation metrics\n", "accuracy_selected = accuracy_score(y_test, y_pred_selected)\n", "precision_selected = precision_score(y_test, y_pred_selected)\n", "recall_selected = recall_score(y_test, y_pred_selected)\n", "f1_selected = f1_score(y_test, y_pred_selected)" ] }, { "cell_type": "code", "execution_count": 212, "id": "4230c428-7012-4988-b561-e96bbf45c18e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Model Performance Before and After Feature Selection:\n", "\n", " **Original Model** (All Features)\n", "👌 Accuracy: 0.7532, Precision: 0.6441, Recall: 0.6909, F1-score: 0.6667\n", "\n", "----------------------------------\n", "\n", " **Feature-Selected Model** (Top 5 Features)\n", "👌 Accuracy: 0.7532, Precision: 0.6491, Recall: 0.6727, F1-score: 0.6607\n" ] } ], "source": [ "# Print comparison results\n", "print(\"\\nModel Performance Before and After Feature Selection:\\n\")\n", "print(f\" **Original Model** (All Features)\")\n", "print(f\"👌 Accuracy: {accuracy:.3f}, Precision: {precision:.3f}, Recall: {recall:.3f}, F1-score: {f1:.3f}\")\n", "print(\"\\n----------------------------------\")\n", "print(f\"\\n **Feature-Selected Model** (Top {top_n} Features)\")\n", "print(f\"👌 Accuracy: {accuracy_selected:.3f}, Precision: {precision_selected:.3f}, Recall: {recall_selected:.3f}, F1-score: {f1_selected:.3f}\")" ] }, { "cell_type": "markdown", "id": "a4dfd48c-06dd-4f2d-9920-3526b149898e", "metadata": {}, "source": [ "## Step11. Apply Principal Component Analysis (PCA)- \n", "### {Apply PCA}" ] }, { "cell_type": "code", "execution_count": 176, "id": "7bc180b2-2253-440a-a4f8-22ef9a942da9", "metadata": {}, "outputs": [], "source": [ "# Standardize the dataset (important for PCA)\n", "scaler = StandardScaler()\n", "X_train_scaled = scaler.fit_transform(X_train)\n", "X_test_scaled = scaler.transform(X_test)" ] }, { "cell_type": "code", "execution_count": 179, "id": "22cf36d6-5e02-4111-94a4-be0b940e213d", "metadata": {}, "outputs": [], "source": [ "# Apply PCA and keep enough components to explain 95% of variance\n", "pca = PCA(n_components=0.95) # Keeps 95% of variance\n", "X_train_pca = pca.fit_transform(X_train_scaled)\n", "X_test_pca = pca.transform(X_test_scaled)" ] }, { "cell_type": "code", "execution_count": 182, "id": "9ed5df4a-80b0-49b8-baa2-72dea5c2ac16", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Original features: 9, PCA components: 8\n" ] } ], "source": [ "# Print the number of selected components\n", "print(f\"Original features: {X_train.shape[1]}, PCA components: {X_train_pca.shape[1]}\")" ] }, { "cell_type": "markdown", "id": "afac7afa-6f90-4b72-88df-467b1d9c13bf", "metadata": {}, "source": [ "## Step 12. Train Random Forest on PCA-Reduced Data" ] }, { "cell_type": "code", "execution_count": 192, "id": "f20192d7-0cc0-475c-b886-21151bc789dd", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
RandomForestClassifier(random_state=42)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" ], "text/plain": [ "RandomForestClassifier(random_state=42)" ] }, "execution_count": 192, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Train a Random Forest model on PCA-transformed data\n", "rf_model_pca = RandomForestClassifier(n_estimators=100, random_state=42)\n", "rf_model_pca.fit(X_train_pca, y_train)" ] }, { "cell_type": "code", "execution_count": 195, "id": "ec08119c-d58a-4d98-8b97-2c79afa32dc6", "metadata": {}, "outputs": [], "source": [ "# Make predictions\n", "y_pred_pca = rf_model_pca.predict(X_test_pca)" ] }, { "cell_type": "markdown", "id": "386cdaea-fd9b-4af3-863a-89430ffdb516", "metadata": {}, "source": [ "## Step 13. Compare the Model Performance " ] }, { "cell_type": "code", "execution_count": 201, "id": "80728aee-d93c-4f47-9f50-a10fde17987b", "metadata": {}, "outputs": [], "source": [ "# Compute evaluation metrics for PCA model\n", "accuracy_pca = accuracy_score(y_test, y_pred_pca)\n", "precision_pca = precision_score(y_test, y_pred_pca)\n", "recall_pca = recall_score(y_test, y_pred_pca)\n", "f1_pca = f1_score(y_test, y_pred_pca)" ] }, { "cell_type": "code", "execution_count": 210, "id": "66c4a865-6247-44f4-b3b3-4ed499dd9385", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "-----------Model Performance Comparison-----------\n", "👌Original Model (All Features): Accuracy: 0.7532, Precision: 0.6441, Recall: 0.6909, F1-score: 0.6667\n", "\n", "--------------------------------------------------\n", "👌Feature-Selected Model (Top Features): Accuracy: 0.7532, Precision: 0.6491, Recall: 0.6727, F1-score: 0.6607\n", "\n", "--------------------------------------------------\n", "👌PCA-Reduced Model: Accuracy: 0.7208, Precision: 0.6154, Recall: 0.5818, F1-score: 0.5981\n" ] } ], "source": [ "# Print comparison results\n", "print(\"\\n-----------Model Performance Comparison-----------\")\n", "print(f\"👌Original Model (All Features): Accuracy: {accuracy:.3f}, Precision: {precision:.3f}, Recall: {recall:.3f}, F1-score: {f1:.3f}\")\n", "print(\"\\n--------------------------------------------------\")\n", "print(f\"👌Feature-Selected Model (Top Features): Accuracy: {accuracy_selected:.3f}, Precision: {precision_selected:.3f}, Recall: {recall_selected:.3f}, F1-score: {f1_selected:.3f}\")\n", "print(\"\\n--------------------------------------------------\")\n", "print(f\"👌PCA-Reduced Model: Accuracy: {accuracy_pca:.3f}, Precision: {precision_pca:.3f}, Recall: {recall_pca:.3f}, F1-score: {f1_pca:.3f}\")" ] }, { "cell_type": "markdown", "id": "bbd8dfa7-a018-49c6-94e3-1fe59785144f", "metadata": {}, "source": [ "## Part 2:-\n", "## Step 1. Train Logistic Regression Model" ] }, { "cell_type": "code", "execution_count": 236, "id": "48c8a1ff-9ca9-4774-ae3f-081c40a5ab34", "metadata": {}, "outputs": [], "source": [ "# Create Logistic regression Model \n", "logistic_model = LogisticRegression(random_state=42, max_iter=500)" ] }, { "cell_type": "code", "execution_count": 238, "id": "e8441b29-1d62-493b-9c0f-7380aca6d2e5", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
LogisticRegression(max_iter=500, random_state=42)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" ], "text/plain": [ "LogisticRegression(max_iter=500, random_state=42)" ] }, "execution_count": 238, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Train the model\n", "logistic_model.fit(X_train, y_train)" ] }, { "cell_type": "code", "execution_count": 240, "id": "b4a93d1f-5d80-4415-a50d-f4d731afcfec", "metadata": {}, "outputs": [], "source": [ "# Make predictions\n", "y_pred_logistic = logistic_model.predict(X_test)" ] }, { "cell_type": "markdown", "id": "acd5aa98-5eb0-4747-8ad1-b667a68880ea", "metadata": {}, "source": [ "## Step 2. Train Naive Bayes Model" ] }, { "cell_type": "code", "execution_count": 243, "id": "e5ba1874-6e43-4b66-8fbb-69bd4a11bf8e", "metadata": {}, "outputs": [], "source": [ "# Create Naive Bayes Model\n", "naive_bayes_model = GaussianNB()" ] }, { "cell_type": "code", "execution_count": 246, "id": "a49d07d9-2b41-4144-ad70-f625df65c59f", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
GaussianNB()
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" ], "text/plain": [ "GaussianNB()" ] }, "execution_count": 246, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Train the model\n", "naive_bayes_model.fit(X_train, y_train)" ] }, { "cell_type": "code", "execution_count": 249, "id": "6ce30e21-3665-4f12-818f-c586226c05dd", "metadata": {}, "outputs": [], "source": [ "# Make predictions\n", "y_pred_nb = naive_bayes_model.predict(X_test)" ] }, { "cell_type": "markdown", "id": "a69390bf-a388-4cd7-a086-2cccfc5f0802", "metadata": {}, "source": [ "## Step 3. Compare Model Performance -" ] }, { "cell_type": "markdown", "id": "41c78db7-e077-419a-b646-0665b2f23402", "metadata": {}, "source": [ "### 1). Random Forest (Original & Feature Selected)" ] }, { "cell_type": "markdown", "id": "ecdcd26e-28b7-46a1-926c-6d9e26ce6077", "metadata": {}, "source": [ "### 2). PCA-Reduced Random Forest" ] }, { "cell_type": "markdown", "id": "a8559812-c90e-4acf-91a5-79f10389bbc1", "metadata": {}, "source": [ "### 3). Logistic Regression" ] }, { "cell_type": "markdown", "id": "d53eb3c8-aa87-4b67-b6a6-9dacbe60669d", "metadata": {}, "source": [ "### 4). Naive Baye" ] }, { "cell_type": "code", "execution_count": 278, "id": "895eb57c-c12c-42f2-8be3-44a057a1e99a", "metadata": {}, "outputs": [], "source": [ "# Compare classification metrics for Logistic Regression\n", "accuracy_logistic = accuracy_score(y_test, y_pred_logistic)\n", "precision_logistic = precision_score(y_test, y_pred_logistic)\n", "recall_logistic = recall_score(y_test, y_pred_logistic)\n", "f1_logistic = f1_score(y_test, y_pred_logistic)" ] }, { "cell_type": "code", "execution_count": 303, "id": "04176676-055e-4460-ab08-9de503442249", "metadata": {}, "outputs": [], "source": [ "# Compare classification metrics for Naïve Bayes\n", "accuracy_nb = accuracy_score(y_test, y_pred_nb)\n", "precision_nb = precision_score(y_test, y_pred_nb)\n", "recall_nb = recall_score(y_test, y_pred_nb)\n", "f1_nb = f1_score(y_test, y_pred_nb)" ] }, { "cell_type": "code", "execution_count": 313, "id": "94aebf7c-c42e-4814-a4fa-e30df4f3ea07", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "**Model Performance Comparison**\n", "-----------------\n", "Random Forest (All Features): Accuracy: 0.753, Precision: 0.644, Recall: 0.691, F1-score: 0.667\n", "--------------------------\n", "Random Forest (Feature Selection): Accuracy: 0.753, Precision: 0.649, Recall: 0.673, F1-score: 0.661\n", "-------------------\n", "Random Forest (PCA-Reduced): Accuracy: 0.721, Precision: 0.615, Recall: 0.582, F1-score: 0.598\n", "---------------------------\n", "Logistic Regression: Accuracy: 0.760, Precision: 0.673, Recall: 0.636, F1-score: 0.654\n", "-------------------------\n", "Naïve Bayes: Accuracy: 0.753, Precision: 0.644, Recall: 0.691, F1-score: 0.667\n", "-----------------------------\n" ] } ], "source": [ "# Print model performance comparison\n", "print(\"\\n**Model Performance Comparison**\\n-----------------\")\n", "print(f\"Random Forest (All Features): Accuracy: {accuracy:.3f}, Precision: {precision:.3f}, Recall: {recall:.3f}, F1-score: {f1:.3f}\\n--------------------------\")\n", "print(f\"Random Forest (Feature Selection): Accuracy: {accuracy_selected:.3f}, Precision: {precision_selected:.3f}, Recall: {recall_selected:.3f}, F1-score: {f1_selected:.3f}\\n-------------------\")\n", "print(f\"Random Forest (PCA-Reduced): Accuracy: {accuracy_pca:.3f}, Precision: {precision_pca:.3f}, Recall: {recall_pca:.3f}, F1-score: {f1_pca:.3f}\\n---------------------------\")\n", "print(f\"Logistic Regression: Accuracy: {accuracy_logistic:.3f}, Precision: {precision_logistic:.3f}, Recall: {recall_logistic:.3f}, F1-score: {f1_logistic:.3f}\\n-------------------------\")\n", "print(f\"Naïve Bayes: Accuracy: {accuracy_nb:.3f}, Precision: {precision_nb:.3f}, Recall: {recall_nb:.3f}, F1-score: {f1_nb:.3f}\\n-----------------------------\")" ] }, { "cell_type": "code", "execution_count": null, "id": "ba498e40-bd85-41cb-8af1-2d85b5af5a9b", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "anaconda-panel-2023.05-py310", "language": "python", "name": "conda-env-anaconda-panel-2023.05-py310-py" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.5" } }, "nbformat": 4, "nbformat_minor": 5 }