Interpretable Multi-Class Mental Health Detection from Reddit Text¶
Support Vector Machine, Random Forest and Long Short-Term Memory models compared on the six class Reddit corpus, with SHAP explanation and a fairness audit.
Run every cell in order from the top. Section 1 mounts Drive and expects the three
both_*.csv files in the folder named below.
Section 1. Environment and data loading¶
In [ ]:
import os
import re
import time
import random
import warnings
import numpy as np
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
warnings.filterwarnings("ignore")
matplotlib.rcParams["font.family"] = "serif"
SEED = 42
random.seed(SEED)
np.random.seed(SEED)
DATA_DIR = "/content/drive/MyDrive/dissertation data set"
TRAIN_FILE = os.path.join(DATA_DIR, "both_train.csv")
VAL_FILE = os.path.join(DATA_DIR, "both_val.csv")
TEST_FILE = os.path.join(DATA_DIR, "both_test.csv")
print("Data directory:", DATA_DIR)
Data directory: /content/drive/MyDrive/dissertation data set
In [ ]:
from google.colab import drive
drive.mount("/content/drive")
if os.path.isdir(DATA_DIR):
print("Folder found. Contents:")
for name in sorted(os.listdir(DATA_DIR)):
print(" ", name)
else:
print("Folder not found. Folders available in MyDrive:")
for name in sorted(os.listdir("/content/drive/MyDrive")):
if os.path.isdir(os.path.join("/content/drive/MyDrive", name)):
print(" ", name)
Mounted at /content/drive
Folder found. Contents:
both_test.csv
both_train.csv
both_val.csv
In [ ]:
OUT_DIR = os.path.join(DATA_DIR, "outputs")
os.makedirs(OUT_DIR, exist_ok=True)
print("Figures and tables will be written to:")
print(" ", OUT_DIR)
Figures and tables will be written to:
/content/drive/MyDrive/dissertation data set/outputs
In [ ]:
train_df = pd.read_csv(TRAIN_FILE)
val_df = pd.read_csv(VAL_FILE)
test_df = pd.read_csv(TEST_FILE)
print("Rows loaded")
print(" train:", len(train_df))
print(" val: ", len(val_df))
print(" test: ", len(test_df))
print("\nColumns:", list(train_df.columns))
print("\nMissing values:")
print(train_df.isnull().sum())
Rows loaded train: 13727 val: 1488 test: 1488 Columns: ['ID', 'title', 'post', 'class_name', 'class_id'] Missing values: ID 0 title 0 post 0 class_name 0 class_id 0 dtype: int64
Section 2. Text assembly and partition hygiene¶
The corpus supplies the post title and body as separate fields. They are joined into a single text column, since the literature reports that the combination outperforms either field alone. Records that appear in more than one partition are then removed, so that no training instance can reach the test set.
In [ ]:
LABEL_COL = "class_id"
NAME_COL = "class_name"
def combine_text(df):
"""Join title and post into a single text field for classification."""
out = df.copy()
out["text"] = out["title"].fillna("").str.strip() + " " + out["post"].fillna("").str.strip()
out["text"] = out["text"].str.strip()
return out[["text", LABEL_COL, NAME_COL]]
train_df = combine_text(train_df)
val_df = combine_text(val_df)
test_df = combine_text(test_df)
id_to_name = (
pd.concat([train_df, val_df, test_df])
.drop_duplicates(LABEL_COL)
.sort_values(LABEL_COL)
.set_index(LABEL_COL)[NAME_COL]
.to_dict()
)
CLASS_NAMES = [id_to_name[i] for i in sorted(id_to_name)]
N_CLASSES = len(CLASS_NAMES)
print("Classes:", CLASS_NAMES)
print("\nSample combined text:")
print(train_df["text"].iloc[1][:280])
Classes: ['adhd', 'anxiety', 'bipolar', 'depression', 'ptsd', 'none'] Sample combined text: the epitome of my life i've recently requested testing accommodations at my university and was given letters to personally give to my professors. so i gave the first one i saw his and he was very nice about it. the next day i brought the next professor his (he probably thinks i'm
In [ ]:
def drop_internal_duplicates(df, name):
"""Remove empty and repeated records within a single partition."""
before = len(df)
df = df[df["text"].str.len() > 0]
df = df.drop_duplicates(subset="text")
print(f"{name:<12} {before} rows in, {len(df)} out, {before - len(df)} removed")
return df.reset_index(drop=True)
print("Removing duplicates within each partition")
train_df = drop_internal_duplicates(train_df, "train")
val_df = drop_internal_duplicates(val_df, "validation")
test_df = drop_internal_duplicates(test_df, "test")
Removing duplicates within each partition train 13727 rows in, 13724 out, 3 removed validation 1488 rows in, 1488 out, 0 removed test 1488 rows in, 1488 out, 0 removed
In [ ]:
# A post occurring in two partitions would leak training information into the
# evaluation, so the copy outside the training set is removed.
train_texts = set(train_df["text"])
before_val = len(val_df)
val_df = val_df[~val_df["text"].isin(train_texts)].reset_index(drop=True)
val_texts = set(val_df["text"])
before_test = len(test_df)
test_df = test_df[~test_df["text"].isin(train_texts | val_texts)].reset_index(drop=True)
print(f"validation {before_val} rows in, {len(val_df)} out")
print(f"test {before_test} rows in, {len(test_df)} out")
print("\nOverlap after removal")
print(" train and validation:", len(set(train_df["text"]) & set(val_df["text"])))
print(" train and test: ", len(set(train_df["text"]) & set(test_df["text"])))
print(" validation and test: ", len(set(val_df["text"]) & set(test_df["text"])))
validation 1488 rows in, 1488 out test 1488 rows in, 1485 out Overlap after removal train and validation: 0 train and test: 0 validation and test: 0
Section 3. Descriptive statistics¶
Class counts and post length are recorded here. Post length is retained for the fairness audit in Section 10, where performance is compared across length strata.
In [ ]:
dist = pd.DataFrame({
"Train": train_df[NAME_COL].value_counts(),
"Validation": val_df[NAME_COL].value_counts(),
"Test": test_df[NAME_COL].value_counts(),
}).reindex(CLASS_NAMES)
dist["Total"] = dist.sum(axis=1)
print("Class distribution after cleaning")
print(dist)
dist.to_csv(os.path.join(OUT_DIR, "table_class_distribution.csv"))
for df in (train_df, val_df, test_df):
df["word_count"] = df["text"].str.split().str.len()
print("\nPost length in words, training set")
print(train_df["word_count"].describe(percentiles=[0.5, 0.75, 0.9, 0.95]).round(1))
Class distribution after cleaning
Train Validation Test Total
class_name
adhd 2465 248 248 2961
anxiety 2422 248 248 2918
bipolar 2406 248 247 2901
depression 2450 248 248 2946
ptsd 2000 248 247 2495
none 1981 248 247 2476
Post length in words, training set
count 13724.0
mean 209.9
std 250.5
min 29.0
50% 135.0
75% 245.0
90% 422.0
95% 595.7
max 6973.0
Name: word_count, dtype: float64
In [ ]:
PALETTE = ["#2b6a8f", "#4f9dc4", "#a8cfe3"]
fig, ax = plt.subplots(figsize=(8.5, 4.6))
plot_df = dist[["Train", "Validation", "Test"]]
plot_df.plot(kind="bar", ax=ax, color=PALETTE, edgecolor="#1c1c1c", linewidth=0.6, width=0.78)
for container in ax.containers:
ax.bar_label(container, fmt="%d", padding=2, fontsize=7.5, color="#333333")
ax.set_xlabel("Mental health class", fontsize=10, labelpad=8)
ax.set_ylabel("Number of posts", fontsize=10, labelpad=8)
ax.set_title("Class distribution across the three partitions", fontsize=11.5, pad=12)
ax.set_ylim(0, plot_df.values.max() * 1.16)
ax.legend(title="Partition", frameon=False, fontsize=9, title_fontsize=9)
ax.grid(axis="y", linestyle=":", linewidth=0.6, color="#b0b0b0")
ax.set_axisbelow(True)
ax.spines["top"].set_visible(False)
ax.spines["right"].set_visible(False)
plt.xticks(rotation=0, fontsize=9.5)
plt.tight_layout()
plt.savefig(os.path.join(OUT_DIR, "fig_class_distribution.png"), dpi=300, bbox_inches="tight")
plt.show()
In [ ]:
fig, ax = plt.subplots(figsize=(8.5, 4.6))
ax.hist(train_df["word_count"].clip(upper=1000), bins=60,
color="#4f9dc4", edgecolor="#1c1c1c", linewidth=0.4)
median_len = train_df["word_count"].median()
p95_len = train_df["word_count"].quantile(0.95)
ax.axvline(median_len, color="#c1440e", linestyle="--", linewidth=1.4,
label=f"Median, {median_len:.0f} words")
ax.axvline(p95_len, color="#7a4b9c", linestyle=":", linewidth=1.6,
label=f"95th percentile, {p95_len:.0f} words")
ax.set_xlabel("Post length in words, capped at 1000 for display", fontsize=10, labelpad=8)
ax.set_ylabel("Number of posts", fontsize=10, labelpad=8)
ax.set_title("Distribution of post length in the training set", fontsize=11.5, pad=12)
ax.legend(frameon=False, fontsize=9)
ax.grid(axis="y", linestyle=":", linewidth=0.6, color="#b0b0b0")
ax.set_axisbelow(True)
ax.spines["top"].set_visible(False)
ax.spines["right"].set_visible(False)
plt.tight_layout()
plt.savefig(os.path.join(OUT_DIR, "fig_post_length.png"), dpi=300, bbox_inches="tight")
plt.show()
Section 4. Preprocessing¶
Two pipelines are produced from the same source text. The traditional models receive lemmatised text with stop words removed, since a sparse bag of words representation benefits from a reduced vocabulary. The recurrent model receives normalised text only, because stop words carry syntactic information a sequential model can use, and naive stop word removal destroys negation.
In [ ]:
import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
nltk.download("stopwords", quiet=True)
nltk.download("wordnet", quiet=True)
STOP_WORDS = set(stopwords.words("english"))
lemmatiser = WordNetLemmatizer()
def normalise(text):
"""Lowercase and strip everything that is not a letter or a space."""
text = str(text).lower()
text = re.sub(r"&#x200b;", " ", text)
text = re.sub(r"http\S+|www\.\S+", " ", text)
text = re.sub(r"[^a-z\s]", " ", text)
return re.sub(r"\s+", " ", text).strip()
def clean_for_traditional(text):
"""Normalise, remove stop words and lemmatise for the TF-IDF models."""
tokens = normalise(text).split()
tokens = [lemmatiser.lemmatize(t) for t in tokens if t not in STOP_WORDS and len(t) > 2]
return " ".join(tokens)
def clean_for_deep(text):
"""Normalise only. Stop words and inflection are kept for the LSTM."""
return normalise(text)
for name, df in [("train", train_df), ("validation", val_df), ("test", test_df)]:
print(f"Processing {name}")
df["text_ml"] = df["text"].apply(clean_for_traditional)
df["text_dl"] = df["text"].apply(clean_for_deep)
print(f" {len(df)} rows done")
print("\nTraditional pipeline output:")
print(train_df["text_ml"].iloc[1][:220])
print("\nDeep pipeline output:")
print(train_df["text_dl"].iloc[1][:220])
Processing train 13724 rows done Processing validation 1488 rows done Processing test 1485 rows done Traditional pipeline output: epitome life recently requested testing accommodation university given letter personally give professor gave first one saw nice next day brought next professor probably think nut always screwing stuff class also nice sho Deep pipeline output: the epitome of my life i ve recently requested testing accommodations at my university and was given letters to personally give to my professors so i gave the first one i saw his and he was very nice about it the next da
In [ ]:
def drop_empty(df, name):
before = len(df)
df = df[(df["text_ml"].str.len() > 0) & (df["text_dl"].str.len() > 0)]
print(f"{name:<12} {before} rows in, {len(df)} out")
return df.reset_index(drop=True)
train_df = drop_empty(train_df, "train")
val_df = drop_empty(val_df, "validation")
test_df = drop_empty(test_df, "test")
y_train = train_df[LABEL_COL].values
y_val = val_df[LABEL_COL].values
y_test = test_df[LABEL_COL].values
print("\nFinal sizes")
print(" train:", len(train_df))
print(" val: ", len(val_df))
print(" test: ", len(test_df))
train 13724 rows in, 13724 out validation 1488 rows in, 1488 out test 1485 rows in, 1485 out Final sizes train: 13724 val: 1488 test: 1485
Section 5. Selected configurations¶
The values below were chosen on the validation partition by grid search. They are
recorded explicitly so that the search does not have to be repeated on every run.
Set RUN_GRID_SEARCH to True in Section 6 to reproduce the search.
In [ ]:
SVM_FEATURES = 10000
SVM_C = 0.1
RF_CONFIG = {
"n_estimators": 200,
"max_depth": None,
"min_samples_split": 2,
}
LSTM_MAX_LEN = 400
LSTM_EMBED = 200
LSTM_UNITS = 160
LSTM_DROPOUT = 0.4
LSTM_BATCH = 32
LSTM_EPOCHS = 30
print("Selected configurations")
print(f" SVM features {SVM_FEATURES}, C {SVM_C}")
print(f" RF {RF_CONFIG}")
print(f" LSTM length {LSTM_MAX_LEN}, embedding {LSTM_EMBED}, units {LSTM_UNITS},"
f" dropout {LSTM_DROPOUT}, batch {LSTM_BATCH}")
Selected configurations
SVM features 10000, C 0.1
RF {'n_estimators': 200, 'max_depth': None, 'min_samples_split': 2}
LSTM length 400, embedding 200, units 160, dropout 0.4, batch 32
Section 6. Feature extraction and optional grid search¶
The vectoriser is fitted on the training partition only. Fitting it on the combined data would let the vocabulary of the evaluation partitions influence the representation, which would inflate the measured performance.
In [ ]:
from sklearn.feature_extraction.text import TfidfVectorizer
def build_tfidf(max_features):
"""Fit on the training partition, then transform all three partitions."""
vec = TfidfVectorizer(
max_features=max_features,
ngram_range=(1, 3),
min_df=3,
sublinear_tf=True,
)
Xtr = vec.fit_transform(train_df["text_ml"].values)
Xva = vec.transform(val_df["text_ml"].values)
Xte = vec.transform(test_df["text_ml"].values)
return vec, Xtr, Xva, Xte
vectoriser, X_train_tf, X_val_tf, X_test_tf = build_tfidf(SVM_FEATURES)
print(f"TF-IDF matrix: {X_train_tf.shape[0]} documents, {X_train_tf.shape[1]} features")
TF-IDF matrix: 13724 documents, 10000 features
In [ ]:
# Set to True to reproduce the search reported in Section 5. It takes several
# minutes and is not required for the results below.
RUN_GRID_SEARCH = False
if RUN_GRID_SEARCH:
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import f1_score
rows = []
start = time.time()
combos = [(f, c) for f in [1000, 5000, 10000, 20000] for c in [0.1, 0.5, 1.0, 2.0]]
for i, (n_feat, c) in enumerate(combos, start=1):
_, Xtr, Xva, _ = build_tfidf(n_feat)
m = LinearSVC(C=c, random_state=SEED, max_iter=5000).fit(Xtr, y_train)
s = f1_score(y_val, m.predict(Xva), average="macro")
rows.append({"max_features": n_feat, "C": c, "val_macro_f1": s})
print(f"[{i}/{len(combos)}] features {n_feat:>5} C {c:<4} "
f"val macro F1 {s:.4f} elapsed {time.time() - start:.0f}s")
svm_grid = pd.DataFrame(rows).sort_values("val_macro_f1", ascending=False)
svm_grid.to_csv(os.path.join(OUT_DIR, "table_svm_grid.csv"), index=False)
print(svm_grid.head().to_string(index=False))
else:
print("Grid search skipped. Using the configurations recorded in Section 5.")
Grid search skipped. Using the configurations recorded in Section 5.
Section 7. Training the traditional models¶
In [ ]:
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import f1_score
print("Training Support Vector Machine")
start = time.time()
svm_model = LinearSVC(C=SVM_C, random_state=SEED, max_iter=5000)
svm_model.fit(X_train_tf, y_train)
print(f" done in {time.time() - start:.0f}s")
print("Training Random Forest")
start = time.time()
rf_model = RandomForestClassifier(random_state=SEED, n_jobs=-1, **RF_CONFIG)
rf_model.fit(X_train_tf, y_train)
print(f" done in {time.time() - start:.0f}s")
svm_val_f1 = f1_score(y_val, svm_model.predict(X_val_tf), average="macro")
rf_val_f1 = f1_score(y_val, rf_model.predict(X_val_tf), average="macro")
print(f"\nValidation macro F1")
print(f" SVM {svm_val_f1:.4f}")
print(f" Random Forest {rf_val_f1:.4f}")
Training Support Vector Machine done in 1s Training Random Forest done in 47s Validation macro F1 SVM 0.8442 Random Forest 0.8114
Section 8. Training the recurrent model¶
The embedding layer is trained from random initialisation rather than loaded from pretrained vectors, since the vocabulary of this domain includes clinical and community specific usage that general corpora represent poorly. Masking is enabled so that the padding does not reach the recurrent computation.
In [ ]:
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Embedding, LSTM, Dense, Dropout, SpatialDropout1D
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
from tensorflow.keras.optimizers import Adam
print("Devices available:", tf.config.list_physical_devices())
MAX_WORDS = 20000
tokeniser = Tokenizer(num_words=MAX_WORDS, oov_token="<unk>")
tokeniser.fit_on_texts(train_df["text_dl"].values)
def to_sequences(texts):
"""Post-padding is required by cuDNN when masking is enabled."""
return pad_sequences(
tokeniser.texts_to_sequences(texts),
maxlen=LSTM_MAX_LEN,
padding="post",
truncating="post",
)
X_train_seq = to_sequences(train_df["text_dl"].values)
X_val_seq = to_sequences(val_df["text_dl"].values)
X_test_seq = to_sequences(test_df["text_dl"].values)
vocab_size = min(MAX_WORDS, len(tokeniser.word_index) + 1)
print(f"\nVocabulary size: {vocab_size}")
print(f"Sequence shape: {X_train_seq.shape}")
Devices available: [PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU'), PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')] Vocabulary size: 20000 Sequence shape: (13724, 400)
In [ ]:
def build_lstm():
"""Embedding with masking, one recurrent layer, softmax over six classes."""
model = Sequential([
Embedding(input_dim=vocab_size, output_dim=LSTM_EMBED, mask_zero=True),
SpatialDropout1D(LSTM_DROPOUT),
LSTM(LSTM_UNITS, dropout=LSTM_DROPOUT),
Dropout(LSTM_DROPOUT),
Dense(64, activation="relu"),
Dropout(LSTM_DROPOUT),
Dense(N_CLASSES, activation="softmax"),
])
model.compile(
optimizer=Adam(learning_rate=1e-3),
loss="sparse_categorical_crossentropy",
metrics=["accuracy"],
)
return model
tf.keras.utils.set_random_seed(SEED)
lstm_model = build_lstm()
lstm_model.build(input_shape=(None, LSTM_MAX_LEN))
lstm_model.summary()
Model: "sequential"
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓ ┃ Layer (type) ┃ Output Shape ┃ Param # ┃ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩ │ embedding (Embedding) │ (None, 400, 200) │ 4,000,000 │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ spatial_dropout1d │ (None, 400, 200) │ 0 │ │ (SpatialDropout1D) │ │ │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ lstm (LSTM) │ (None, 160) │ 231,040 │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ dropout (Dropout) │ (None, 160) │ 0 │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ dense (Dense) │ (None, 64) │ 10,304 │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ dropout_1 (Dropout) │ (None, 64) │ 0 │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ dense_1 (Dense) │ (None, 6) │ 390 │ └─────────────────────────────────┴────────────────────────┴───────────────┘
Total params: 4,241,734 (16.18 MB)
Trainable params: 4,241,734 (16.18 MB)
Non-trainable params: 0 (0.00 B)
In [ ]:
callbacks = [
EarlyStopping(monitor="val_loss", patience=5, restore_best_weights=True, verbose=1),
ReduceLROnPlateau(monitor="val_loss", factor=0.5, patience=2, min_lr=1e-5, verbose=1),
]
start = time.time()
history = lstm_model.fit(
X_train_seq, y_train,
validation_data=(X_val_seq, y_val),
epochs=LSTM_EPOCHS,
batch_size=LSTM_BATCH,
callbacks=callbacks,
verbose=2,
)
print(f"\nTraining finished in {time.time() - start:.0f} seconds")
lstm_val_f1 = f1_score(
y_val, lstm_model.predict(X_val_seq, batch_size=LSTM_BATCH, verbose=0).argmax(axis=1),
average="macro",
)
print(f"LSTM validation macro F1: {lstm_val_f1:.4f}")
Epoch 1/30 429/429 - 16s - 37ms/step - accuracy: 0.2789 - loss: 1.6058 - val_accuracy: 0.3273 - val_loss: 1.4520 - learning_rate: 0.0010 Epoch 2/30 429/429 - 10s - 24ms/step - accuracy: 0.3757 - loss: 1.4611 - val_accuracy: 0.3434 - val_loss: 1.5028 - learning_rate: 0.0010 Epoch 3/30 429/429 - 9s - 21ms/step - accuracy: 0.4676 - loss: 1.2972 - val_accuracy: 0.5363 - val_loss: 1.1662 - learning_rate: 0.0010 Epoch 4/30 429/429 - 10s - 23ms/step - accuracy: 0.5804 - loss: 1.1291 - val_accuracy: 0.6794 - val_loss: 1.0102 - learning_rate: 0.0010 Epoch 5/30 429/429 - 10s - 24ms/step - accuracy: 0.6037 - loss: 1.0897 - val_accuracy: 0.5208 - val_loss: 1.1618 - learning_rate: 0.0010 Epoch 6/30 429/429 - 10s - 23ms/step - accuracy: 0.6728 - loss: 0.9701 - val_accuracy: 0.7493 - val_loss: 0.7729 - learning_rate: 0.0010 Epoch 7/30 429/429 - 10s - 23ms/step - accuracy: 0.7211 - loss: 0.8386 - val_accuracy: 0.7366 - val_loss: 0.9209 - learning_rate: 0.0010 Epoch 8/30 429/429 - 10s - 24ms/step - accuracy: 0.7838 - loss: 0.6846 - val_accuracy: 0.7890 - val_loss: 0.6807 - learning_rate: 0.0010 Epoch 9/30 429/429 - 20s - 46ms/step - accuracy: 0.7805 - loss: 0.7205 - val_accuracy: 0.7809 - val_loss: 0.7909 - learning_rate: 0.0010 Epoch 10/30 429/429 - 10s - 23ms/step - accuracy: 0.7990 - loss: 0.6458 - val_accuracy: 0.7897 - val_loss: 0.6460 - learning_rate: 0.0010 Epoch 11/30 429/429 - 10s - 24ms/step - accuracy: 0.8646 - loss: 0.4593 - val_accuracy: 0.8098 - val_loss: 0.6309 - learning_rate: 0.0010 Epoch 12/30 429/429 - 10s - 24ms/step - accuracy: 0.8935 - loss: 0.3565 - val_accuracy: 0.8118 - val_loss: 0.6628 - learning_rate: 0.0010 Epoch 13/30 Epoch 13: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257. 429/429 - 9s - 22ms/step - accuracy: 0.9097 - loss: 0.3077 - val_accuracy: 0.8165 - val_loss: 0.6719 - learning_rate: 0.0010 Epoch 14/30 429/429 - 10s - 24ms/step - accuracy: 0.9294 - loss: 0.2452 - val_accuracy: 0.8105 - val_loss: 0.6895 - learning_rate: 5.0000e-04 Epoch 15/30 Epoch 15: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628. 429/429 - 10s - 24ms/step - accuracy: 0.9380 - loss: 0.2174 - val_accuracy: 0.8145 - val_loss: 0.6868 - learning_rate: 5.0000e-04 Epoch 16/30 429/429 - 9s - 22ms/step - accuracy: 0.9461 - loss: 0.1855 - val_accuracy: 0.8246 - val_loss: 0.7113 - learning_rate: 2.5000e-04 Epoch 16: early stopping Restoring model weights from the end of the best epoch: 11. Training finished in 174 seconds LSTM validation macro F1: 0.8116
In [ ]:
fig, axes = plt.subplots(1, 2, figsize=(11, 4.2))
axes[0].plot(history.history["loss"], color="#2b6a8f", marker="o", markersize=3.5, label="Training")
axes[0].plot(history.history["val_loss"], color="#c1440e", marker="s", markersize=3.5, label="Validation")
axes[0].set_ylabel("Loss", fontsize=10)
axes[0].set_title("Loss during training", fontsize=11)
axes[1].plot(history.history["accuracy"], color="#2b6a8f", marker="o", markersize=3.5, label="Training")
axes[1].plot(history.history["val_accuracy"], color="#c1440e", marker="s", markersize=3.5, label="Validation")
axes[1].set_ylabel("Accuracy", fontsize=10)
axes[1].set_title("Accuracy during training", fontsize=11)
for ax in axes:
ax.set_xlabel("Epoch", fontsize=10)
ax.legend(frameon=False, fontsize=9)
ax.grid(linestyle=":", linewidth=0.6, color="#b0b0b0")
ax.set_axisbelow(True)
ax.spines["top"].set_visible(False)
ax.spines["right"].set_visible(False)
plt.tight_layout()
plt.savefig(os.path.join(OUT_DIR, "fig_lstm_training.png"), dpi=300, bbox_inches="tight")
plt.show()
Section 9. Evaluation on the held-out test set¶
All configuration decisions are now final. The test partition is used once, from this point on, to produce the reported results.
In [ ]:
from sklearn.metrics import (
accuracy_score, precision_score, recall_score,
classification_report, confusion_matrix,
)
svm_pred = svm_model.predict(X_test_tf)
rf_pred = rf_model.predict(X_test_tf)
lstm_prob = lstm_model.predict(X_test_seq, batch_size=LSTM_BATCH, verbose=1)
lstm_pred = lstm_prob.argmax(axis=1)
PREDICTIONS = {
"SVM": svm_pred,
"Random Forest": rf_pred,
"LSTM": lstm_pred,
}
summary = pd.DataFrame([
{
"Model": name,
"Accuracy": accuracy_score(y_test, pred),
"Macro Precision": precision_score(y_test, pred, average="macro", zero_division=0),
"Macro Recall": recall_score(y_test, pred, average="macro", zero_division=0),
"Macro F1": f1_score(y_test, pred, average="macro", zero_division=0),
}
for name, pred in PREDICTIONS.items()
]).round(4)
print("Comparative performance on the test set")
print(summary.to_string(index=False))
summary.to_csv(os.path.join(OUT_DIR, "table_model_comparison.csv"), index=False)
47/47 ━━━━━━━━━━━━━━━━━━━━ 1s 12ms/step Comparative performance on the test set Model Accuracy Macro Precision Macro Recall Macro F1 SVM 0.8290 0.8344 0.8290 0.8293 Random Forest 0.7906 0.8019 0.7906 0.7921 LSTM 0.8128 0.8201 0.8128 0.8139
In [ ]:
for name, pred in PREDICTIONS.items():
print("\n" + "=" * 62)
print(f"Per-class results, {name}")
print("=" * 62)
print(classification_report(y_test, pred, target_names=CLASS_NAMES,
digits=4, zero_division=0))
report_df = pd.DataFrame(
classification_report(y_test, pred, target_names=CLASS_NAMES,
output_dict=True, zero_division=0)
).transpose().round(4)
report_df.to_csv(os.path.join(
OUT_DIR, f"table_per_class_{name.lower().replace(' ', '_')}.csv"))
==============================================================
Per-class results, SVM
==============================================================
precision recall f1-score support
adhd 0.8279 0.8145 0.8211 248
anxiety 0.7976 0.8105 0.8040 248
bipolar 0.8186 0.7490 0.7822 247
depression 0.7099 0.8387 0.7689 248
ptsd 0.9324 0.7814 0.8502 247
none 0.9202 0.9798 0.9490 247
accuracy 0.8290 1485
macro avg 0.8344 0.8290 0.8293 1485
weighted avg 0.8343 0.8290 0.8292 1485
==============================================================
Per-class results, Random Forest
==============================================================
precision recall f1-score support
adhd 0.7823 0.7823 0.7823 248
anxiety 0.7884 0.7661 0.7771 248
bipolar 0.8357 0.7004 0.7621 247
depression 0.6219 0.8024 0.7007 248
ptsd 0.8937 0.7490 0.8150 247
none 0.8893 0.9433 0.9155 247
accuracy 0.7906 1485
macro avg 0.8019 0.7906 0.7921 1485
weighted avg 0.8017 0.7906 0.7920 1485
==============================================================
Per-class results, LSTM
==============================================================
precision recall f1-score support
adhd 0.8672 0.8427 0.8548 248
anxiety 0.6910 0.8387 0.7577 248
bipolar 0.8512 0.7409 0.7922 247
depression 0.7203 0.7581 0.7387 248
ptsd 0.8710 0.7652 0.8147 247
none 0.9200 0.9312 0.9256 247
accuracy 0.8128 1485
macro avg 0.8201 0.8128 0.8139 1485
weighted avg 0.8200 0.8128 0.8139 1485
In [ ]:
fig, ax = plt.subplots(figsize=(8, 4.6))
metrics = ["Accuracy", "Macro Precision", "Macro Recall", "Macro F1"]
x = np.arange(len(metrics))
width = 0.26
for i, (_, row) in enumerate(summary.iterrows()):
bars = ax.bar(x + (i - 1) * width, [row[m] for m in metrics], width,
label=row["Model"], color=PALETTE[i],
edgecolor="#1c1c1c", linewidth=0.6)
ax.bar_label(bars, fmt="%.3f", padding=2, fontsize=7.5)
ax.set_xticks(x)
ax.set_xticklabels(metrics, fontsize=9.5)
ax.set_ylabel("Score", fontsize=10, labelpad=8)
ax.set_ylim(0, 1.05)
ax.set_title("Comparative performance on the held-out test set", fontsize=11.5, pad=12)
ax.legend(frameon=False, fontsize=9)
ax.grid(axis="y", linestyle=":", linewidth=0.6, color="#b0b0b0")
ax.set_axisbelow(True)
ax.spines["top"].set_visible(False)
ax.spines["right"].set_visible(False)
plt.tight_layout()
plt.savefig(os.path.join(OUT_DIR, "fig_model_comparison.png"), dpi=300, bbox_inches="tight")
plt.show()
In [ ]:
fig, axes = plt.subplots(1, 3, figsize=(16, 4.8))
for ax, (name, pred) in zip(axes, PREDICTIONS.items()):
cm = confusion_matrix(y_test, pred)
ax.imshow(cm, cmap="Blues", vmin=0, vmax=cm.max())
for i in range(N_CLASSES):
for j in range(N_CLASSES):
colour = "white" if cm[i, j] > cm.max() * 0.55 else "#1c1c1c"
ax.text(j, i, cm[i, j], ha="center", va="center", fontsize=8.5, color=colour)
ax.set_xticks(range(N_CLASSES))
ax.set_yticks(range(N_CLASSES))
ax.set_xticklabels(CLASS_NAMES, rotation=45, ha="right", fontsize=8.5)
ax.set_yticklabels(CLASS_NAMES, fontsize=8.5)
ax.set_xlabel("Predicted class", fontsize=9.5)
ax.set_ylabel("True class", fontsize=9.5)
ax.set_title(f"{name}, macro F1 = {f1_score(y_test, pred, average='macro'):.3f}",
fontsize=10.5, pad=10)
plt.tight_layout()
plt.savefig(os.path.join(OUT_DIR, "fig_confusion_matrices.png"), dpi=300, bbox_inches="tight")
plt.show()
Section 10. Interpretability¶
SHAP attributions are computed for the linear model. Because the Support Vector Machine is linear over TF-IDF features, the exact Shapley value of a feature reduces to its coefficient multiplied by its centred value, so the attribution can be computed directly and without sampling error.
The analysis asks two questions. Which terms carry the greatest weight for each class, and do those terms correspond to recognised clinical markers rather than to artefacts of the source community.
In [ ]:
%pip install shap --quiet
import shap
print("SHAP version:", shap.__version__)
SHAP version: 0.52.0
In [ ]:
# For a linear model the Shapley value of feature j for class k is the coefficient
# multiplied by the deviation of the feature from its background mean. This is the
# closed form that shap.LinearExplainer evaluates, so no sampling error arises.
feature_names = np.array(vectoriser.get_feature_names_out())
coefficients = svm_model.coef_
background_mean = np.asarray(X_train_tf.mean(axis=0)).ravel()
SAMPLE_SIZE = 500
rng = np.random.default_rng(SEED)
sample_idx = rng.choice(X_test_tf.shape[0],
size=min(SAMPLE_SIZE, X_test_tf.shape[0]),
replace=False)
X_sample = X_test_tf[sample_idx].toarray()
y_sample = y_test[sample_idx]
# shap_values[k] holds the attributions for class k over the sampled instances
deviation = X_sample - background_mean
shap_values = np.stack([deviation * coefficients[k] for k in range(N_CLASSES)])
print("SHAP attributions computed")
print(" classes:", shap_values.shape[0])
print(" instances:", shap_values.shape[1])
print(" features:", shap_values.shape[2])
# Verification against the library implementation, where the API permits it
try:
background_var = (np.asarray(X_train_tf.power(2).mean(axis=0)).ravel()
- background_mean ** 2)
explainer = shap.LinearExplainer(
(coefficients, svm_model.intercept_), (background_mean, background_var)
)
library_values = np.array(explainer.shap_values(X_sample))
if library_values.shape[0] != N_CLASSES:
library_values = np.moveaxis(library_values, -1, 0)
difference = np.abs(library_values - shap_values).max()
print(f" maximum difference from shap.LinearExplainer: {difference:.2e}")
except Exception as error:
print(" library cross-check unavailable:", type(error).__name__)
SHAP attributions computed classes: 6 instances: 500 features: 10000 maximum difference from shap.LinearExplainer: 0.00e+00
In [ ]:
TOP_N = 12
def top_terms_for_class(class_index, n=TOP_N):
"""Terms with the largest mean attribution among instances of that class."""
belongs = y_sample == class_index
values = shap_values[class_index][belongs]
mean_contribution = values.mean(axis=0)
order = np.argsort(mean_contribution)[::-1][:n]
return pd.DataFrame({
"term": feature_names[order],
"mean_shap": mean_contribution[order].round(5),
})
shap_tables = {}
for k, class_name in enumerate(CLASS_NAMES):
table = top_terms_for_class(k)
shap_tables[class_name] = table
print(f"\nMost influential terms for the {class_name} class")
print(table.to_string(index=False))
combined = pd.concat(
[t.assign(**{"class": c}) for c, t in shap_tables.items()], ignore_index=True
)
combined.to_csv(os.path.join(OUT_DIR, "table_shap_top_terms.csv"), index=False)
Most influential terms for the adhd class
term mean_shap
adhd 0.40642
adderall 0.02680
anxiety 0.01993
add 0.01384
bipolar 0.01279
medication 0.01277
ptsd 0.01167
task 0.01100
distracted 0.01099
diagnosed 0.00910
depression 0.00908
focus 0.00859
Most influential terms for the anxiety class
term mean_shap
anxiety 0.32803
anxious 0.04706
panic 0.03151
attack 0.02885
adhd 0.02317
panic attack 0.02121
ptsd 0.01919
bipolar 0.01161
proud 0.01093
fear 0.00828
depressed 0.00699
depression 0.00696
Most influential terms for the bipolar class
term mean_shap
bipolar 0.20020
manic 0.06166
med 0.05221
episode 0.04390
mania 0.03718
adhd 0.01901
lamictal 0.01411
ptsd 0.01339
anxiety 0.01334
lithium 0.01262
bipolar disorder 0.01220
psychotic 0.01111
Most influential terms for the depression class
term mean_shap
depression 0.08389
life 0.03547
anxiety 0.03426
adhd 0.02652
depressed 0.02530
ptsd 0.01605
bipolar 0.01516
fuck 0.01250
lonely 0.01152
feel 0.01025
wish 0.00960
kill 0.00951
Most influential terms for the ptsd class
term mean_shap
ptsd 0.27022
trauma 0.10397
flashback 0.02931
trigger 0.02400
therapy 0.01765
adhd 0.01382
happened 0.01265
triggered 0.01211
abuse 0.01130
bipolar 0.00993
therapist 0.00946
memory 0.00937
Most influential terms for the none class
term mean_shap
data 0.03016
dataset 0.02622
feel 0.02596
english 0.02495
anxiety 0.02286
adhd 0.02280
math 0.01898
india 0.01434
song 0.01307
datasets 0.01286
mathematics 0.01246
depression 0.01234
In [ ]:
fig, axes = plt.subplots(2, 3, figsize=(15, 8))
for ax, (class_name, table) in zip(axes.ravel(), shap_tables.items()):
ordered = table.iloc[::-1]
ax.barh(ordered["term"], ordered["mean_shap"],
color="#4f9dc4", edgecolor="#1c1c1c", linewidth=0.5)
ax.set_title(class_name, fontsize=11, pad=8)
ax.set_xlabel("Mean SHAP value", fontsize=9)
ax.tick_params(axis="y", labelsize=8.5)
ax.tick_params(axis="x", labelsize=8)
ax.grid(axis="x", linestyle=":", linewidth=0.6, color="#b0b0b0")
ax.set_axisbelow(True)
ax.spines["top"].set_visible(False)
ax.spines["right"].set_visible(False)
fig.suptitle("Terms with the greatest positive attribution for each class",
fontsize=12.5, y=1.00)
plt.tight_layout()
plt.savefig(os.path.join(OUT_DIR, "fig_shap_per_class.png"), dpi=300, bbox_inches="tight")
plt.show()
Section 11. Bias and fairness audit¶
Aggregate metrics conceal uneven performance. Three analyses are reported here. Per-class error rates show whether error concentrates in particular conditions. False negative and false positive rates are separated, since in a screening context a missed case and an unnecessary review carry different costs. Performance is then stratified by post length, because users who write briefly are not a random subset of users.
Demographic attributes are not available in this corpus and cannot be inferred responsibly, so the audit is bounded by what the data supports.
In [ ]:
def error_rates(y_true, y_pred):
"""Per-class false negative and false positive rates from the confusion matrix."""
cm = confusion_matrix(y_true, y_pred)
rows = []
for k, class_name in enumerate(CLASS_NAMES):
true_positive = cm[k, k]
false_negative = cm[k].sum() - true_positive
false_positive = cm[:, k].sum() - true_positive
true_negative = cm.sum() - true_positive - false_negative - false_positive
rows.append({
"class": class_name,
"support": cm[k].sum(),
"false_negative_rate": false_negative / (false_negative + true_positive),
"false_positive_rate": false_positive / (false_positive + true_negative),
})
return pd.DataFrame(rows).round(4)
fairness_tables = {}
for name, pred in PREDICTIONS.items():
table = error_rates(y_test, pred)
fairness_tables[name] = table
spread = table["false_negative_rate"].max() - table["false_negative_rate"].min()
print(f"\n{name}")
print(table.to_string(index=False))
print(f" spread in false negative rate across classes: {spread:.4f}")
pd.concat(
[t.assign(model=m) for m, t in fairness_tables.items()], ignore_index=True
).to_csv(os.path.join(OUT_DIR, "table_fairness_error_rates.csv"), index=False)
SVM
class support false_negative_rate false_positive_rate
adhd 248 0.1855 0.0340
anxiety 248 0.1895 0.0412
bipolar 247 0.2510 0.0331
depression 248 0.1613 0.0687
ptsd 247 0.2186 0.0113
none 247 0.0202 0.0170
spread in false negative rate across classes: 0.2308
Random Forest
class support false_negative_rate false_positive_rate
adhd 248 0.2177 0.0437
anxiety 248 0.2339 0.0412
bipolar 247 0.2996 0.0275
depression 248 0.1976 0.0978
ptsd 247 0.2510 0.0178
none 247 0.0567 0.0234
spread in false negative rate across classes: 0.2429
LSTM
class support false_negative_rate false_positive_rate
adhd 248 0.1573 0.0259
anxiety 248 0.1613 0.0752
bipolar 247 0.2591 0.0258
depression 248 0.2419 0.0590
ptsd 247 0.2348 0.0226
none 247 0.0688 0.0162
spread in false negative rate across classes: 0.1903
In [ ]:
fig, ax = plt.subplots(figsize=(9, 4.6))
x = np.arange(N_CLASSES)
width = 0.26
for i, (name, table) in enumerate(fairness_tables.items()):
bars = ax.bar(x + (i - 1) * width, table["false_negative_rate"], width,
label=name, color=PALETTE[i], edgecolor="#1c1c1c", linewidth=0.6)
ax.bar_label(bars, fmt="%.2f", padding=2, fontsize=7)
ax.set_xticks(x)
ax.set_xticklabels(CLASS_NAMES, fontsize=9.5)
ax.set_xlabel("Mental health class", fontsize=10, labelpad=8)
ax.set_ylabel("False negative rate", fontsize=10, labelpad=8)
ax.set_title("False negative rate by class, a missed case in a screening context",
fontsize=11.5, pad=12)
ax.legend(frameon=False, fontsize=9)
ax.grid(axis="y", linestyle=":", linewidth=0.6, color="#b0b0b0")
ax.set_axisbelow(True)
ax.spines["top"].set_visible(False)
ax.spines["right"].set_visible(False)
plt.tight_layout()
plt.savefig(os.path.join(OUT_DIR, "fig_fairness_false_negatives.png"), dpi=300, bbox_inches="tight")
plt.show()
In [ ]:
# Post length is the only stratification variable the corpus supports. Quartiles
# are taken from the test partition itself so that the strata are balanced.
lengths = test_df["word_count"].values
edges = np.quantile(lengths, [0, 0.25, 0.5, 0.75, 1.0])
strata_labels = [
f"Q1, up to {edges[1]:.0f} words",
f"Q2, {edges[1]:.0f} to {edges[2]:.0f}",
f"Q3, {edges[2]:.0f} to {edges[3]:.0f}",
f"Q4, above {edges[3]:.0f}",
]
strata = np.digitize(lengths, edges[1:-1], right=True)
rows = []
for name, pred in PREDICTIONS.items():
for s, label in enumerate(strata_labels):
mask = strata == s
rows.append({
"model": name,
"stratum": label,
"n": int(mask.sum()),
"macro_f1": f1_score(y_test[mask], pred[mask], average="macro", zero_division=0),
})
length_audit = pd.DataFrame(rows).round(4)
print("Performance by post length stratum")
print(length_audit.to_string(index=False))
length_audit.to_csv(os.path.join(OUT_DIR, "table_fairness_by_length.csv"), index=False)
Performance by post length stratum
model stratum n macro_f1
SVM Q1, up to 81 words 375 0.7850
SVM Q2, 81 to 138 370 0.8192
SVM Q3, 138 to 246 372 0.8266
SVM Q4, above 246 368 0.8732
Random Forest Q1, up to 81 words 375 0.7346
Random Forest Q2, 81 to 138 370 0.7883
Random Forest Q3, 138 to 246 372 0.8021
Random Forest Q4, above 246 368 0.8172
LSTM Q1, up to 81 words 375 0.7803
LSTM Q2, 81 to 138 370 0.8124
LSTM Q3, 138 to 246 372 0.8081
LSTM Q4, above 246 368 0.8389
In [ ]:
fig, ax = plt.subplots(figsize=(9, 4.6))
x = np.arange(len(strata_labels))
width = 0.26
for i, name in enumerate(PREDICTIONS):
subset = length_audit[length_audit["model"] == name]
bars = ax.bar(x + (i - 1) * width, subset["macro_f1"], width,
label=name, color=PALETTE[i], edgecolor="#1c1c1c", linewidth=0.6)
ax.bar_label(bars, fmt="%.3f", padding=2, fontsize=7)
ax.set_xticks(x)
ax.set_xticklabels(strata_labels, fontsize=8.5)
ax.set_xlabel("Post length stratum", fontsize=10, labelpad=8)
ax.set_ylabel("Macro F1", fontsize=10, labelpad=8)
ax.set_ylim(0, 1.05)
ax.set_title("Performance across post length strata", fontsize=11.5, pad=12)
ax.legend(frameon=False, fontsize=9)
ax.grid(axis="y", linestyle=":", linewidth=0.6, color="#b0b0b0")
ax.set_axisbelow(True)
ax.spines["top"].set_visible(False)
ax.spines["right"].set_visible(False)
plt.tight_layout()
plt.savefig(os.path.join(OUT_DIR, "fig_fairness_by_length.png"), dpi=300, bbox_inches="tight")
plt.show()
Section 12. Summary of saved outputs¶
In [ ]:
print("Files written to", OUT_DIR)
print()
for name in sorted(os.listdir(OUT_DIR)):
size = os.path.getsize(os.path.join(OUT_DIR, name)) / 1024
print(f" {name:<44} {size:>8.1f} KB")
print("\nHeadline results")
print(summary.to_string(index=False))
Files written to /content/drive/MyDrive/dissertation data set/outputs
fig_class_distribution.png 119.1 KB
fig_confusion_matrices.png 234.6 KB
fig_fairness_by_length.png 127.0 KB
fig_fairness_false_negatives.png 141.6 KB
fig_lstm_training.png 238.0 KB
fig_model_comparison.png 101.3 KB
fig_post_length.png 128.4 KB
fig_shap_per_class.png 418.5 KB
table_class_distribution.csv 0.2 KB
table_fairness_by_length.csv 0.4 KB
table_fairness_error_rates.csv 0.6 KB
table_model_comparison.csv 0.2 KB
table_per_class_lstm.csv 0.3 KB
table_per_class_random_forest.csv 0.3 KB
table_per_class_svm.csv 0.3 KB
table_shap_top_terms.csv 1.6 KB
Headline results
Model Accuracy Macro Precision Macro Recall Macro F1
SVM 0.8290 0.8344 0.8290 0.8293
Random Forest 0.7906 0.8019 0.7906 0.7921
LSTM 0.8128 0.8201 0.8128 0.8139