Jeremy Janella
1734 lines
62 KiB
Plaintext
1734 lines
62 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"id": "ea5511c9-c672-4545-ba1f-84f9f3e2d775",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"import json\n",
|
||
"import pickle\n",
|
||
"import random\n",
|
||
"import re\n",
|
||
"import urllib.request\n",
|
||
"import zipfile\n",
|
||
"from pathlib import Path\n",
|
||
"import numpy as np\n",
|
||
"\n",
|
||
"# Reproducibility\n",
|
||
"random.seed(1337)\n",
|
||
"np.random.seed(1337)\n",
|
||
"\n",
|
||
"import pandas as pd\n",
|
||
"import matplotlib.pyplot as plt\n",
|
||
"from sklearn.feature_extraction.text import CountVectorizer\n",
|
||
"from sklearn.model_selection import train_test_split\n",
|
||
"from sklearn.naive_bayes import MultinomialNB\n",
|
||
"from sklearn.metrics import classification_report, accuracy_score\n",
|
||
"\n",
|
||
"from htb_ai_library import (\n",
|
||
" AZURE,\n",
|
||
" HACKER_GREY,\n",
|
||
" HTB_GREEN,\n",
|
||
" MALWARE_RED,\n",
|
||
" NODE_BLACK,\n",
|
||
" NUGGET_YELLOW,\n",
|
||
" WHITE,\n",
|
||
" AQUAMARINE,\n",
|
||
" load_model,\n",
|
||
" save_model,\n",
|
||
")\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 2,
|
||
"id": "e66ce529-60c0-42ac-8536-02f0ffb88cbc",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"[*] Loading SMS Spam Dataset...\n",
|
||
"[+] Using cached dataset: data/sms_spam.csv\n",
|
||
"[+] Loaded 5574 messages\n",
|
||
" Spam: 747\n",
|
||
" Ham: 4827\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"print(\"\\n[*] Loading SMS Spam Dataset...\")\n",
|
||
"\n",
|
||
"data_dir = Path(\"data\")\n",
|
||
"data_dir.mkdir(exist_ok=True)\n",
|
||
"dataset_path = data_dir / \"sms_spam.csv\"\n",
|
||
"\n",
|
||
"if dataset_path.exists():\n",
|
||
" print(f\"[+] Using cached dataset: {dataset_path}\")\n",
|
||
" df = pd.read_csv(dataset_path)\n",
|
||
"else:\n",
|
||
" print(\"[*] Downloading from UCI repository...\")\n",
|
||
" url = \"https://archive.ics.uci.edu/ml/machine-learning-databases/00228/smsspamcollection.zip\"\n",
|
||
" zip_path = data_dir / \"sms_spam.zip\"\n",
|
||
"\n",
|
||
" urllib.request.urlretrieve(url, zip_path)\n",
|
||
"\n",
|
||
" with zipfile.ZipFile(zip_path, \"r\") as zf:\n",
|
||
" with zf.open(\"SMSSpamCollection\") as f:\n",
|
||
" lines = [line.decode(\"utf-8\").strip() for line in f]\n",
|
||
"\n",
|
||
" # Parse tab-separated format\n",
|
||
" data = []\n",
|
||
" for line in lines:\n",
|
||
" parts = line.split('\\t')\n",
|
||
" if len(parts) == 2:\n",
|
||
" data.append({\"label\": parts[0].lower(), \"message\": parts[1]})\n",
|
||
"\n",
|
||
" df = pd.DataFrame(data)\n",
|
||
" df.to_csv(dataset_path, index=False)\n",
|
||
" zip_path.unlink()\n",
|
||
" print(f\"[+] Dataset saved to {dataset_path}\")\n",
|
||
"\n",
|
||
"print(f\"[+] Loaded {len(df)} messages\")\n",
|
||
"print(f\" Spam: {sum(df['label'] == 'spam')}\")\n",
|
||
"print(f\" Ham: {sum(df['label'] == 'ham')}\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 3,
|
||
"id": "4ff06a17-1640-4b1b-9a65-c9ec5686a982",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"[*] Sample messages:\n",
|
||
"\n",
|
||
"SPAM samples:\n",
|
||
" - Free entry in 2 a wkly comp to win FA Cup final tkts 21st May 2005. Text FA to 8...\n",
|
||
" - FreeMsg Hey there darling it's been 3 week's now and no word back! I'd like some...\n",
|
||
" - WINNER!! As a valued network customer you have been selected to receivea £900 pr...\n",
|
||
"\n",
|
||
"HAM samples:\n",
|
||
" - Go until jurong point, crazy.. Available only in bugis n great world la e buffet...\n",
|
||
" - Ok lar... Joking wif u oni......\n",
|
||
" - U dun say so early hor... U c already then say......\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"print(\"\\n[*] Sample messages:\")\n",
|
||
"print(\"\\nSPAM samples:\")\n",
|
||
"for msg in df[df['label'] == 'spam']['message'].head(3):\n",
|
||
" print(f\" - {msg[:80]}...\")\n",
|
||
"print(\"\\nHAM samples:\")\n",
|
||
"for msg in df[df['label'] == 'ham']['message'].head(3):\n",
|
||
" print(f\" - {msg[:80]}...\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "3446771b-b673-4e4b-8951-4efe5b3eff7c",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Cleaning"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 4,
|
||
"id": "4e826b27-ba03-4d56-aa24-92906e8c98b8",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"import html as html_module\n",
|
||
"import unicodedata\n",
|
||
"\n",
|
||
"def minimal_clean(text):\n",
|
||
" \"\"\"\n",
|
||
" Minimal cleaning that preserves spam indicators.\n",
|
||
"\n",
|
||
" Parameters: text (str) raw SMS message\n",
|
||
" Returns: str cleaned text with entities decoded, unicode normalized, and\n",
|
||
" whitespace collapsed while keeping informative symbols.\n",
|
||
" \"\"\"\n",
|
||
" # Decode HTML entities (e.g., & -> &)\n",
|
||
" text = html_module.unescape(text)\n",
|
||
"\n",
|
||
" # Normalize unicode characters\n",
|
||
" text = unicodedata.normalize('NFKC', text)\n",
|
||
"\n",
|
||
" # Clean up excessive whitespace\n",
|
||
" text = re.sub(r'\\s+', ' ', text)\n",
|
||
" text = re.sub(r'\\n+', ' ', text)\n",
|
||
" text = re.sub(r'\\r+', ' ', text)\n",
|
||
"\n",
|
||
" return text.strip()\n",
|
||
"\n",
|
||
"def clean_text(text):\n",
|
||
" \"\"\"\n",
|
||
" Final cleaning for vectorization.\n",
|
||
"\n",
|
||
" Converts to lowercase and removes only problematic characters so that\n",
|
||
" informative symbols remain available to the vectorizer.\n",
|
||
"\n",
|
||
" Parameters:\n",
|
||
" text (str): Preprocessed message from `minimal_clean`.\n",
|
||
"\n",
|
||
" Returns:\n",
|
||
" str: Normalized, whitespace‑collapsed text ready for tokenization.\n",
|
||
" \"\"\"\n",
|
||
" text = text.lower()\n",
|
||
" # Keep numbers, currency symbols, punctuation - they're spam features!\n",
|
||
" # Only remove truly problematic characters\n",
|
||
" text = re.sub(r'[^\\w\\s£$€¥!?.,;:\\'\\\"-]', ' ', text)\n",
|
||
" text = re.sub(r'\\s+', ' ', text)\n",
|
||
" return text.strip()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "39bdf320-c877-468f-833f-ba2fbf4025ec",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Preprocessing"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 5,
|
||
"id": "56d40a5c-77e3-4edf-a111-d157f283ffec",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"[*] Applying minimal text cleaning (preserving spam indicators)...\n",
|
||
"\n",
|
||
"[*] Sample spam messages with preserved features:\n",
|
||
" - Summers finally here! Fancy a chat or flirt with sexy singles in yr area? To get MATCHED up just rep...\n",
|
||
" - This is the 2nd time we have tried 2 contact u. U have won the 750 Pound prize. 2 claim is easy, cal...\n",
|
||
" - Get ur 1st RINGTONE FREE NOW! Reply to this msg with TONE. Gr8 TOP 20 tones to your phone every week...\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"print(\"[*] Applying minimal text cleaning (preserving spam indicators)...\")\n",
|
||
"df['preprocessed'] = df['message'].apply(minimal_clean)\n",
|
||
"\n",
|
||
"# Apply final cleaning for vectorization\n",
|
||
"df['clean_message'] = df['preprocessed'].apply(clean_text)\n",
|
||
"\n",
|
||
"print(\"\\n[*] Sample spam messages with preserved features:\")\n",
|
||
"spam_samples = df[df['label'] == 'spam'].sample(3, random_state=42)\n",
|
||
"for idx, row in spam_samples.iterrows():\n",
|
||
" msg = row['preprocessed'][:100] + \"...\" if len(row['preprocessed']) > 100 else row['preprocessed']\n",
|
||
" print(f\" - {msg}\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 6,
|
||
"id": "423ce49f-60c9-4131-aea9-d190560e19cd",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"[+] Removed 419 duplicates\n",
|
||
"[+] Removed 0 empty messages\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# Remove only exact duplicates\n",
|
||
"original_size = len(df)\n",
|
||
"df = df.drop_duplicates(subset=['label', 'clean_message'])\n",
|
||
"print(f\"\\n[+] Removed {original_size - len(df)} duplicates\")\n",
|
||
"\n",
|
||
"# Remove empty messages\n",
|
||
"before_empty = len(df)\n",
|
||
"df = df[df['clean_message'].str.len() > 0]\n",
|
||
"print(f\"[+] Removed {before_empty - len(df)} empty messages\")\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "9a7e2e62-bb23-4dde-8736-f03fcb25d558",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Split"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 7,
|
||
"id": "babced9b-8363-41b5-8c60-b96131433a95",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"[+] Data split:\n",
|
||
" Training: 4124 messages\n",
|
||
" Testing: 1031 messages\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"X = df['clean_message'].values\n",
|
||
"y = df['label'].values\n",
|
||
"\n",
|
||
"X_train, X_test, y_train, y_test = train_test_split(\n",
|
||
" X, y, test_size=0.2, random_state=42, stratify=y\n",
|
||
")\n",
|
||
"\n",
|
||
"print(f\"\\n[+] Data split:\")\n",
|
||
"print(f\" Training: {len(X_train)} messages\")\n",
|
||
"print(f\" Testing: {len(X_test)} messages\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "60a9d5ae-a10a-491f-a05e-c2c8dc59dec1",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Train"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 8,
|
||
"id": "5de0e33d-9f0c-4c67-b73b-aec1d9832bb0",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"[*] Training Naive Bayes classifier...\n",
|
||
"[+] Loading saved model from models/spam_classifier.pkl\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"print(\"\\n[*] Training Naive Bayes classifier...\")\n",
|
||
"\n",
|
||
"model_dir = Path(\"models\")\n",
|
||
"model_dir.mkdir(exist_ok=True)\n",
|
||
"model_path = model_dir / \"spam_classifier.pkl\"\n",
|
||
"\n",
|
||
"if model_path.exists():\n",
|
||
" print(f\"[+] Loading saved model from {model_path}\")\n",
|
||
" with open(model_path, 'rb') as f:\n",
|
||
" saved_data = pickle.load(f)\n",
|
||
" vectorizer = saved_data['vectorizer']\n",
|
||
" classifier = saved_data['classifier']\n",
|
||
"\n",
|
||
" # Transform data using existing vocabulary\n",
|
||
" X_train_vec = vectorizer.transform(X_train)\n",
|
||
" X_test_vec = vectorizer.transform(X_test)\n",
|
||
"\n",
|
||
"else:\n",
|
||
" # Configure vectorizer to capture spam patterns\n",
|
||
" vectorizer = CountVectorizer(\n",
|
||
" max_features=3000,\n",
|
||
" token_pattern=r'\\b\\w+\\b|[£$€¥]+|\\d+|!!+|\\?\\?+|\\.\\.+',\n",
|
||
" lowercase=True,\n",
|
||
" stop_words='english'\n",
|
||
" )\n",
|
||
" X_train_vec = vectorizer.fit_transform(X_train)\n",
|
||
" X_test_vec = vectorizer.transform(X_test)\n",
|
||
" \n",
|
||
" classifier = MultinomialNB()\n",
|
||
" classifier.fit(X_train_vec, y_train)\n",
|
||
"\n",
|
||
" # Save model for reproducibility\n",
|
||
" with open(model_path, 'wb') as f:\n",
|
||
" pickle.dump({'vectorizer': vectorizer, 'classifier': classifier}, f)\n",
|
||
" print(f\"[+] Model saved to {model_path}\")\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "487125a8-5d4f-45cd-bc29-42aad90df8b5",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Eval"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 9,
|
||
"id": "d3d09aad-c687-49ab-9e42-7f1f21d40ea8",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"[+] Training accuracy: 0.9918\n",
|
||
"[+] Testing accuracy: 0.9855\n",
|
||
"\n",
|
||
"[*] Classification Report:\n",
|
||
" precision recall f1-score support\n",
|
||
"\n",
|
||
" ham 0.99 0.99 0.99 903\n",
|
||
" spam 0.95 0.93 0.94 128\n",
|
||
"\n",
|
||
" accuracy 0.99 1031\n",
|
||
" macro avg 0.97 0.96 0.97 1031\n",
|
||
"weighted avg 0.99 0.99 0.99 1031\n",
|
||
"\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# Calculate accuracy scores\n",
|
||
"train_acc = classifier.score(X_train_vec, y_train)\n",
|
||
"test_acc = classifier.score(X_test_vec, y_test)\n",
|
||
"print(f\"[+] Training accuracy: {train_acc:.4f}\")\n",
|
||
"print(f\"[+] Testing accuracy: {test_acc:.4f}\")\n",
|
||
"\n",
|
||
"# Get detailed predictions\n",
|
||
"y_pred = classifier.predict(X_test_vec)\n",
|
||
"print(\"\\n[*] Classification Report:\")\n",
|
||
"print(classification_report(y_test, y_pred))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 10,
|
||
"id": "659be282-cdfe-4d38-9f28-4a293731e81d",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"[*] Extracting GoodWords from model...\n",
|
||
"[+] Top 10 GoodWords (most 'ham-like'):\n",
|
||
" lor | goodness: 50.94 | ham_p: 0.0047 | spam_p: 0.0001\n",
|
||
" ü | goodness: 47.57 | ham_p: 0.0044 | spam_p: 0.0001\n",
|
||
" ... | goodness: 45.81 | ham_p: 0.0299 | spam_p: 0.0007\n",
|
||
" da | goodness: 45.70 | ham_p: 0.0043 | spam_p: 0.0001\n",
|
||
" later | goodness: 29.97 | ham_p: 0.0028 | spam_p: 0.0001\n",
|
||
" doing | goodness: 26.22 | ham_p: 0.0024 | spam_p: 0.0001\n",
|
||
" really | goodness: 25.47 | ham_p: 0.0024 | spam_p: 0.0001\n",
|
||
" ask | goodness: 24.72 | ham_p: 0.0023 | spam_p: 0.0001\n",
|
||
" cos | goodness: 23.60 | ham_p: 0.0022 | spam_p: 0.0001\n",
|
||
" lol | goodness: 23.60 | ham_p: 0.0022 | spam_p: 0.0001\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"print(\"\\n[*] Extracting GoodWords from model...\")\n",
|
||
"\n",
|
||
"# Get feature names and probabilities\n",
|
||
"feature_names = vectorizer.get_feature_names_out()\n",
|
||
"ham_log_probs = classifier.feature_log_prob_[0] # Ham class\n",
|
||
"spam_log_probs = classifier.feature_log_prob_[1] # Spam class\n",
|
||
"\n",
|
||
"# Calculate goodness scores\n",
|
||
"goodness_scores = []\n",
|
||
"for i, word in enumerate(feature_names):\n",
|
||
" ham_prob = np.exp(ham_log_probs[i])\n",
|
||
" spam_prob = np.exp(spam_log_probs[i])\n",
|
||
" goodness = ham_prob / (spam_prob + 1e-10)\n",
|
||
" goodness_scores.append((word, goodness, ham_prob, spam_prob))\n",
|
||
"\n",
|
||
"# Sort by goodness\n",
|
||
"goodness_scores.sort(key=lambda x: x[1], reverse=True)\n",
|
||
"top_good_words = goodness_scores[:100]\n",
|
||
"\n",
|
||
"print(f\"[+] Top 10 GoodWords (most 'ham-like'):\")\n",
|
||
"for word, score, hp, sp in top_good_words[:10]:\n",
|
||
" print(f\" {word:15} | goodness: {score:8.2f} | ham_p: {hp:.4f} | spam_p: {sp:.4f}\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "7e816857-59b4-472d-8bf7-c024f5156bd7",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Attack"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 11,
|
||
"id": "020d5ebc-9d9f-456f-89aa-a58199fb193d",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"[*] Testing GoodWords attack...\n",
|
||
"[+] Testing on 128 spam messages\n",
|
||
"[*] Testing word counts: [0, 5, 10, 15, 20, 25, 30, 35, 40]\n",
|
||
" Using words: lor, ü, ..., da, later\n",
|
||
"\n",
|
||
"Original: as a registered subscriber yr draw 4 a £100 gift voucher will b entered on receipt of a correct ans. when are the next olympics. txt ans to 80062...\n",
|
||
"Augmented: as a registered subscriber yr draw 4 a £100 gift voucher will b entered on receipt of a correct ans. when are the next olympics. txt ans to 80062 lor ü ... da later...\n",
|
||
" Words: 0 | Evasion: 7.03% (9/128)\n",
|
||
" Words: 5 | Evasion: 41.41% (53/128)\n",
|
||
" Words: 10 | Evasion: 74.22% (95/128)\n",
|
||
" Words: 15 | Evasion: 96.09% (123/128)\n",
|
||
" Words: 20 | Evasion: 100.00% (128/128)\n",
|
||
" Words: 25 | Evasion: 100.00% (128/128)\n",
|
||
" Words: 30 | Evasion: 100.00% (128/128)\n",
|
||
" Words: 35 | Evasion: 100.00% (128/128)\n",
|
||
" Words: 40 | Evasion: 100.00% (128/128)\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"print(\"\\n[*] Testing GoodWords attack...\")\n",
|
||
"\n",
|
||
"# Extract only spam messages for testing\n",
|
||
"spam_test_messages = X_test[y_test == 'spam']\n",
|
||
"print(f\"[+] Testing on {len(spam_test_messages)} spam messages\")\n",
|
||
"\n",
|
||
"# Define test points from baseline (0) to saturation (40)\n",
|
||
"word_counts = [0, 5, 10, 15, 20, 25, 30, 35, 40]\n",
|
||
"attack_results = []\n",
|
||
"\n",
|
||
"print(f\"[*] Testing word counts: {word_counts}\")\n",
|
||
"\n",
|
||
"for num_words in word_counts:\n",
|
||
" # Select the top N good words for this iteration\n",
|
||
" selected_words = [w for w, _, _, _ in top_good_words[:num_words]]\n",
|
||
"\n",
|
||
" # Show which words we're using (first iteration only for clarity)\n",
|
||
" if num_words == 5:\n",
|
||
" print(f\" Using words: {', '.join(selected_words)}\")\n",
|
||
"\n",
|
||
"def augment_message(message, words_to_add):\n",
|
||
" \"\"\"Append good words to a message\"\"\"\n",
|
||
" if len(words_to_add) > 0:\n",
|
||
" return message + \" \" + \" \".join(words_to_add)\n",
|
||
" return message\n",
|
||
"\n",
|
||
"# Test augmentation on one example using the top 5 words\n",
|
||
"sample_spam = spam_test_messages[0]\n",
|
||
"sample_augmented = augment_message(\n",
|
||
" sample_spam,\n",
|
||
" [w for w, _, _, _ in top_good_words[:5]]\n",
|
||
")\n",
|
||
"print(f\"\\nOriginal: {sample_spam}...\")\n",
|
||
"print(f\"Augmented: {sample_augmented}...\")\n",
|
||
"\n",
|
||
"for num_words in word_counts:\n",
|
||
" # Select the top N good words for this iteration\n",
|
||
" selected_words = [w for w, _, _, _ in top_good_words[:num_words]]\n",
|
||
"\n",
|
||
" # Count how many spam messages evade after augmentation\n",
|
||
" evaded = 0\n",
|
||
" for message in spam_test_messages:\n",
|
||
" # Augment the message\n",
|
||
" augmented = augment_message(message, selected_words)\n",
|
||
"\n",
|
||
" # Transform and predict\n",
|
||
" vec = vectorizer.transform([augmented])\n",
|
||
" prob = classifier.predict_proba(vec)[0]\n",
|
||
"\n",
|
||
" # Check evasion: ham probability > spam probability\n",
|
||
" if prob[0] > prob[1]:\n",
|
||
" evaded += 1\n",
|
||
"\n",
|
||
" # Record results for this configuration\n",
|
||
" evasion_rate = (evaded / len(spam_test_messages)) * 100\n",
|
||
" attack_results.append({\n",
|
||
" 'num_words': num_words,\n",
|
||
" 'evasion_rate': evasion_rate,\n",
|
||
" 'evaded': evaded,\n",
|
||
" 'total': len(spam_test_messages)\n",
|
||
" })\n",
|
||
"\n",
|
||
" print(f\" Words: {num_words:2d} | Evasion: {evasion_rate:6.2f}% ({evaded}/{len(spam_test_messages)})\")\n",
|
||
"\n",
|
||
"results_df = pd.DataFrame(attack_results)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "27ca7953-f8b5-4997-a3ce-8db22fc2a544",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Plot"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 12,
|
||
"id": "1a8ffb38-bd5f-4e94-8861-8f7826ee4fe2",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"[+] Plot saved to attachments/attack_effectiveness.png\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"plt.style.use('dark_background')\n",
|
||
"fig, ax = plt.subplots(figsize=(12, 6), facecolor=NODE_BLACK)\n",
|
||
"\n",
|
||
"ax.plot(results_df['num_words'], results_df['evasion_rate'],\n",
|
||
" marker='o', markersize=8, linewidth=2.5,\n",
|
||
" color=HTB_GREEN, markeredgecolor='white', markeredgewidth=1)\n",
|
||
"\n",
|
||
"ax.fill_between(results_df['num_words'], 0, results_df['evasion_rate'],\n",
|
||
" alpha=0.3, color=HTB_GREEN)\n",
|
||
"\n",
|
||
"# Add threshold lines\n",
|
||
"ax.axhline(y=50, color=NUGGET_YELLOW, linestyle='--', alpha=0.7, label='50% threshold')\n",
|
||
"ax.axhline(y=90, color=AZURE, linestyle='--', alpha=0.7, label='90% threshold')\n",
|
||
"\n",
|
||
"# Highlight maximum\n",
|
||
"max_idx = results_df['evasion_rate'].idxmax()\n",
|
||
"max_rate = results_df.loc[max_idx, 'evasion_rate']\n",
|
||
"max_words = results_df.loc[max_idx, 'num_words']\n",
|
||
"ax.scatter(max_words, max_rate, s=200, color=MALWARE_RED, zorder=5)\n",
|
||
"ax.annotate(f'Peak: {max_rate:.1f}%\\n@ {max_words} words',\n",
|
||
" xy=(max_words, max_rate), xytext=(max_words+5, max_rate-10),\n",
|
||
" color='white', fontsize=10,\n",
|
||
" arrowprops=dict(arrowstyle='->', color=MALWARE_RED, lw=1.5))\n",
|
||
"\n",
|
||
"ax.set_xlabel('Number of Good Words Added', fontsize=12, color=HTB_GREEN)\n",
|
||
"ax.set_ylabel('Evasion Rate (%)', fontsize=12, color=HTB_GREEN)\n",
|
||
"ax.set_title('GoodWords Attack Effectiveness', fontsize=14, color=HTB_GREEN, pad=20)\n",
|
||
"ax.grid(True, alpha=0.2)\n",
|
||
"ax.set_facecolor(NODE_BLACK)\n",
|
||
"ax.legend()\n",
|
||
"\n",
|
||
"for spine in ax.spines.values():\n",
|
||
" spine.set_color(HACKER_GREY)\n",
|
||
"ax.tick_params(colors=HACKER_GREY)\n",
|
||
"\n",
|
||
"plt.tight_layout()\n",
|
||
"output_dir = Path(\"attachments\")\n",
|
||
"output_dir.mkdir(exist_ok=True)\n",
|
||
"plt.savefig(output_dir / \"attack_effectiveness.png\", dpi=150, facecolor=NODE_BLACK)\n",
|
||
"plt.close()\n",
|
||
"print(f\"\\n[+] Plot saved to {output_dir / 'attack_effectiveness.png'}\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "raw",
|
||
"id": "d6613d7e-8bac-4be9-a7c8-9ed9cf44f870",
|
||
"metadata": {},
|
||
"source": [
|
||
"logP(ham∣message)=logP(ham)+∑w∈wordslogP(w∣ham)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "21dcba53-c378-44d0-923b-cff8c5579503",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Optimization (Best Word :0)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 13,
|
||
"id": "f0aaa9b2-3a81-48a6-bd30-60c10934186f",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"[*] Analyzing individual word impact...\n",
|
||
"[+] Testing 20 words on 50 spam messages\n",
|
||
" Word 'lor': 7.13% reduction\n",
|
||
" Word 'ü': 7.04% reduction\n",
|
||
" Word '...': 6.98% reduction\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"print(\"\\n[*] Analyzing individual word impact...\")\n",
|
||
"\n",
|
||
"# Use 50 spam messages as a representative sample\n",
|
||
"sample_spam = spam_test_messages[:50]\n",
|
||
"word_impacts = []\n",
|
||
"\n",
|
||
"print(f\"[+] Testing {len(top_good_words[:20])} words on {len(sample_spam)} spam messages\")\n",
|
||
"\n",
|
||
"for word, _, _, _ in top_good_words[:20]:\n",
|
||
" total_impact = 0\n",
|
||
"\n",
|
||
" for message in sample_spam:\n",
|
||
" # Calculate original spam probability\n",
|
||
" vec_orig = vectorizer.transform([message])\n",
|
||
" prob_orig = classifier.predict_proba(vec_orig)[0][1] # spam prob\n",
|
||
"\n",
|
||
" # Calculate probability after adding the word\n",
|
||
" vec_aug = vectorizer.transform([message + \" \" + word])\n",
|
||
" prob_aug = classifier.predict_proba(vec_aug)[0][1]\n",
|
||
"\n",
|
||
" # Measure the probability reduction\n",
|
||
" impact = prob_orig - prob_aug\n",
|
||
" total_impact += impact\n",
|
||
"\n",
|
||
" # Calculate average impact across all messages\n",
|
||
" avg_impact = (total_impact / len(sample_spam)) * 100\n",
|
||
" word_impacts.append((word, avg_impact))\n",
|
||
"\n",
|
||
" # Show progress for first few words\n",
|
||
" if len(word_impacts) <= 3:\n",
|
||
" print(f\" Word '{word}': {avg_impact:.2f}% reduction\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "248d4b21-e1dd-4b82-a0ae-d142e5623f37",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Plot"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 14,
|
||
"id": "74929764-4fcc-4b48-91f5-2e7635295190",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"[+] Plot saved to attachments/word_impact.png\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# Plot word impacts\n",
|
||
"fig, ax = plt.subplots(figsize=(10, 8), facecolor=NODE_BLACK)\n",
|
||
"\n",
|
||
"words = [w for w, _ in word_impacts]\n",
|
||
"impacts = [i for _, i in word_impacts]\n",
|
||
"colors = [HTB_GREEN if i > 15 else NUGGET_YELLOW if i > 10 else HACKER_GREY for i in impacts]\n",
|
||
"\n",
|
||
"bars = ax.barh(range(len(words)), impacts, color=colors, edgecolor='white', linewidth=0.5)\n",
|
||
"\n",
|
||
"ax.set_yticks(range(len(words)))\n",
|
||
"ax.set_yticklabels(words)\n",
|
||
"ax.set_xlabel('Average Spam Probability Reduction (%)', fontsize=12, color=HTB_GREEN)\n",
|
||
"ax.set_title('Individual Word Impact on Spam Detection', fontsize=14, color=HTB_GREEN, pad=20)\n",
|
||
"ax.grid(axis='x', alpha=0.2)\n",
|
||
"ax.set_facecolor(NODE_BLACK)\n",
|
||
"\n",
|
||
"for spine in ax.spines.values():\n",
|
||
" spine.set_color(HACKER_GREY)\n",
|
||
"ax.tick_params(colors=HACKER_GREY)\n",
|
||
"\n",
|
||
"plt.tight_layout()\n",
|
||
"plt.savefig(output_dir / \"word_impact.png\", dpi=150, facecolor=NODE_BLACK)\n",
|
||
"plt.close()\n",
|
||
"print(f\"[+] Plot saved to {output_dir / 'word_impact.png'}\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "7a888c46-129d-4182-afbc-44d1b2cd3cc1",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Probability Shift Analysis"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 15,
|
||
"id": "a816c1fa-5eba-46b2-b553-ab21cb408bc8",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"[*] Visualizing probability shifts...\n",
|
||
"[+] Plot saved to attachments/probability_shift.png\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"print(\"\\n[*] Visualizing probability shifts...\")\n",
|
||
"\n",
|
||
"# Sample messages for detailed analysis\n",
|
||
"sample_messages = spam_test_messages[:8]\n",
|
||
"test_word_counts = [0, 5, 10, 20, 30]\n",
|
||
"\n",
|
||
"fig, ax = plt.subplots(figsize=(14, 6), facecolor=NODE_BLACK)\n",
|
||
"\n",
|
||
"x = np.arange(len(sample_messages))\n",
|
||
"width = 0.15\n",
|
||
"colors_list = [MALWARE_RED, NUGGET_YELLOW, AZURE, HTB_GREEN, AQUAMARINE]\n",
|
||
"\n",
|
||
"for i, num_words in enumerate(test_word_counts):\n",
|
||
" # Select the appropriate number of good words\n",
|
||
" selected = [w for w, _, _, _ in top_good_words[:num_words]]\n",
|
||
" probs = []\n",
|
||
"\n",
|
||
" for msg in sample_messages:\n",
|
||
" # Augment message with selected words\n",
|
||
" if num_words > 0:\n",
|
||
" aug_msg = msg + \" \" + \" \".join(selected)\n",
|
||
" else:\n",
|
||
" aug_msg = msg\n",
|
||
"\n",
|
||
" # Calculate spam probability\n",
|
||
" vec = vectorizer.transform([aug_msg])\n",
|
||
" spam_prob = classifier.predict_proba(vec)[0][1]\n",
|
||
" probs.append(spam_prob)\n",
|
||
"\n",
|
||
" # Create grouped bars with distinct colors\n",
|
||
" bars = ax.bar(x + i*width, probs, width,\n",
|
||
" label=f'{num_words} words',\n",
|
||
" color=colors_list[i], alpha=0.8)\n",
|
||
"\n",
|
||
" # Mark successful evasions\n",
|
||
" for j, (bar, prob) in enumerate(zip(bars, probs)):\n",
|
||
" if prob < 0.5:\n",
|
||
" ax.text(bar.get_x() + bar.get_width()/2, prob + 0.02,\n",
|
||
" '✓', ha='center', va='bottom', color=HTB_GREEN, fontweight='bold')\n",
|
||
"\n",
|
||
"ax.axhline(y=0.5, color='white', linestyle='--', alpha=0.5, label='Decision boundary')\n",
|
||
"ax.set_xlabel('Message Index', fontsize=12, color=HTB_GREEN)\n",
|
||
"ax.set_ylabel('Spam Probability', fontsize=12, color=HTB_GREEN)\n",
|
||
"ax.set_title('Probability Shift with Increasing Good Words', fontsize=14, color=HTB_GREEN, pad=20)\n",
|
||
"ax.set_xticks(x + width * 2)\n",
|
||
"ax.set_xticklabels([f'M{i+1}' for i in range(len(sample_messages))])\n",
|
||
"ax.legend(loc='upper right')\n",
|
||
"ax.grid(axis='y', alpha=0.2)\n",
|
||
"ax.set_facecolor(NODE_BLACK)\n",
|
||
"\n",
|
||
"for spine in ax.spines.values():\n",
|
||
" spine.set_color(HACKER_GREY)\n",
|
||
"ax.tick_params(colors=HACKER_GREY)\n",
|
||
"\n",
|
||
"plt.tight_layout()\n",
|
||
"plt.savefig(output_dir / \"probability_shift.png\", dpi=150, facecolor=NODE_BLACK)\n",
|
||
"plt.close()\n",
|
||
"print(f\"[+] Plot saved to {output_dir / 'probability_shift.png'}\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "61bbc496-93de-413b-bb99-52d82e01e82e",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Begin Blackbox attack"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 16,
|
||
"id": "62174fd9-69c8-444b-825a-0e9d4c2acf48",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"[*] Simulating black-box attack scenario...\n",
|
||
"[*] Budget: 1000 queries\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"print(\"\\n[*] Simulating black-box attack scenario...\")\n",
|
||
"print(\"[*] Budget: 1000 queries\")\n",
|
||
"\n",
|
||
"# Simulate limited query access\n",
|
||
"query_budget = 1000\n",
|
||
"queries_used = 0\n",
|
||
"query_log = []"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 17,
|
||
"id": "985bf085-e771-497d-be38-91b95a6805ba",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"[*] Example: extract_ham_word_freq\n",
|
||
" Ham messages sampled: 500\n",
|
||
" Unique tokens found: 2162\n",
|
||
" you: 207\n",
|
||
" the: 125\n",
|
||
" and: 107\n",
|
||
" that: 67\n",
|
||
" for: 62\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"def extract_ham_word_freq(X_train, y_train, sample_size=500):\n",
|
||
" \"\"\"\n",
|
||
" Compute token frequencies from a sample of ham messages.\n",
|
||
"\n",
|
||
" Parameters\n",
|
||
" ----------\n",
|
||
" X_train : array-like of str\n",
|
||
" Cleaned training messages.\n",
|
||
" y_train : array-like of str\n",
|
||
" Labels aligned with X_train ('ham' or 'spam').\n",
|
||
" sample_size : int, default 500\n",
|
||
" Number of ham messages to analyze.\n",
|
||
"\n",
|
||
" Returns\n",
|
||
" -------\n",
|
||
" dict[str, int]\n",
|
||
" Mapping of word -> frequency within sampled ham messages.\n",
|
||
" \"\"\"\n",
|
||
" ham_msgs = X_train[y_train == 'ham']\n",
|
||
" limit = min(sample_size, len(ham_msgs))\n",
|
||
" freq = {}\n",
|
||
" for msg in ham_msgs[:limit]:\n",
|
||
" for w in str(msg).split():\n",
|
||
" if 2 < len(w) < 10: # keep typical conversational tokens\n",
|
||
" freq[w] = freq.get(w, 0) + 1\n",
|
||
" return freq\n",
|
||
"\n",
|
||
"wf_example = extract_ham_word_freq(X_train, y_train, sample_size=500)\n",
|
||
"print(\"[*] Example: extract_ham_word_freq\")\n",
|
||
"print(f\" Ham messages sampled: {min(500, sum(y_train == 'ham'))}\")\n",
|
||
"print(f\" Unique tokens found: {len(wf_example)}\")\n",
|
||
"top5 = sorted(wf_example.items(), key=lambda x: (-x[1], x[0]))[:5]\n",
|
||
"for w, c in top5:\n",
|
||
" print(f\" {w}: {c}\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 18,
|
||
"id": "425ebd85-4ad2-4e22-b98f-163b497dadb8",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"[*] Example: select_high_frequency_words\n",
|
||
" Selected top words: 100 (min_freq=5)\n",
|
||
" First 10: you, the, and, that, for, your, are, have, but, not\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"def select_high_frequency_words(word_freq, max_words=100, min_freq=5):\n",
|
||
" \"\"\"\n",
|
||
" Select the most frequent ham words above a minimum frequency.\n",
|
||
"\n",
|
||
" Parameters\n",
|
||
" ----------\n",
|
||
" word_freq : dict[str, int]\n",
|
||
" Token frequency table for sampled ham messages.\n",
|
||
" max_words : int, default 100\n",
|
||
" Maximum number of words to return.\n",
|
||
" min_freq : int, default 5\n",
|
||
" Minimum frequency a word must meet to be considered.\n",
|
||
"\n",
|
||
" Returns\n",
|
||
" -------\n",
|
||
" list[str]\n",
|
||
" Top words sorted by decreasing frequency then lexicographically.\n",
|
||
" \"\"\"\n",
|
||
" sorted_by_freq = sorted(word_freq.items(), key=lambda x: (-x[1], x[0]))\n",
|
||
" top = [w for w, c in sorted_by_freq if c > min_freq][:max_words]\n",
|
||
" return top\n",
|
||
"\n",
|
||
"top_words_example = select_high_frequency_words(wf_example, max_words=100, min_freq=5)\n",
|
||
"print(\"[*] Example: select_high_frequency_words\")\n",
|
||
"print(f\" Selected top words: {len(top_words_example)} (min_freq=5)\")\n",
|
||
"print(\" First 10:\", \", \".join(top_words_example[:10]))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 19,
|
||
"id": "3649a469-5d97-4205-bedc-3d9320bfade0",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"[*] Example: merge_with_curated\n",
|
||
" Merged size: 111 | Added curated: 11\n",
|
||
" Sample added terms: ask, didnt, doing, ill, later\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"def merge_with_curated(top_words, additional_candidates=None):\n",
|
||
" \"\"\"\n",
|
||
" Merge data-driven top words with curated conversational candidates.\n",
|
||
"\n",
|
||
" Parameters\n",
|
||
" ----------\n",
|
||
" top_words : list[str]\n",
|
||
" High-frequency ham words from the previous step.\n",
|
||
" additional_candidates : list[str] | None\n",
|
||
" Optional curated list to include regardless of frequency.\n",
|
||
"\n",
|
||
" Returns\n",
|
||
" -------\n",
|
||
" list[str]\n",
|
||
" Deduplicated merged list (lexicographically ordered).\n",
|
||
" \"\"\"\n",
|
||
" if additional_candidates is None:\n",
|
||
" additional_candidates = [\n",
|
||
" \"ok\", \"cos\", \"ill\", \"thats\", \"later\", \"said\", \"ask\", \"didnt\",\n",
|
||
" \"dont\", \"doing\", \"going\", \"come\", \"home\", \"tomorrow\", \"today\", \"sorry\",\n",
|
||
" \"thanks\", \"yeah\", \"yes\", \"sure\", \"see\", \"tell\", \"know\", \"think\",\n",
|
||
" ]\n",
|
||
" merged = set(top_words) | set(additional_candidates)\n",
|
||
" return sorted(merged)\n",
|
||
"\n",
|
||
"merged_example = merge_with_curated(top_words_example)\n",
|
||
"added = sorted(set(merged_example) - set(top_words_example))\n",
|
||
"print(\"[*] Example: merge_with_curated\")\n",
|
||
"print(f\" Merged size: {len(merged_example)} | Added curated: {len(added)}\")\n",
|
||
"print(\" Sample added terms:\", \", \".join(added[:5]))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 20,
|
||
"id": "be05bc00-7ec4-4c5d-bb77-9779240cbcf7",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"[*] Example: build_candidate_vocabulary\n",
|
||
" Candidates: 111\n",
|
||
" First 10: you, the, and, that, for, your, are, have, but, not\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"def build_candidate_vocabulary(\n",
|
||
" X_train,\n",
|
||
" y_train,\n",
|
||
" sample_size=500,\n",
|
||
" max_words=100,\n",
|
||
" min_freq=5,\n",
|
||
" additional_candidates=None,\n",
|
||
"):\n",
|
||
" \"\"\"\n",
|
||
" Build a candidate vocabulary for black-box discovery from ham messages.\n",
|
||
"\n",
|
||
" Parameters\n",
|
||
" ----------\n",
|
||
" X_train : array-like of str\n",
|
||
" Cleaned training messages.\n",
|
||
" y_train : array-like of str\n",
|
||
" Labels aligned with X_train ('ham' or 'spam').\n",
|
||
" sample_size : int, default 500\n",
|
||
" Number of ham messages to analyze.\n",
|
||
" max_words : int, default 100\n",
|
||
" Maximum number of top frequent ham words to keep before merging extras.\n",
|
||
" min_freq : int, default 5\n",
|
||
" Minimum frequency threshold for inclusion from the ham corpus.\n",
|
||
" additional_candidates : list[str] | None\n",
|
||
" Optional curated conversational terms to include.\n",
|
||
"\n",
|
||
" Returns\n",
|
||
" -------\n",
|
||
" list[str]\n",
|
||
" Deduplicated candidate words ordered by decreasing ham frequency,\n",
|
||
" then lexicographically for stable ties.\n",
|
||
" \"\"\"\n",
|
||
" word_freq = extract_ham_word_freq(X_train, y_train, sample_size=sample_size)\n",
|
||
" top_words = select_high_frequency_words(word_freq, max_words=max_words, min_freq=min_freq)\n",
|
||
" merged = merge_with_curated(top_words, additional_candidates=additional_candidates)\n",
|
||
"\n",
|
||
" # Stable final ordering driven by ham frequency, then lexical for ties\n",
|
||
" def sort_key(w):\n",
|
||
" return (-word_freq.get(w, 0), w)\n",
|
||
"\n",
|
||
" return sorted(merged, key=sort_key)\n",
|
||
"\n",
|
||
"cv_example = build_candidate_vocabulary(X_train, y_train)\n",
|
||
"print(\"[*] Example: build_candidate_vocabulary\")\n",
|
||
"print(f\" Candidates: {len(cv_example)}\")\n",
|
||
"print(\" First 10:\", \", \".join(cv_example[:10]))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 21,
|
||
"id": "109b6a4a-95ab-43b3-a400-5d9f5c99f194",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"[+] Testing 111 candidate words extracted from ham messages\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# Build candidate vocabulary for discovery\n",
|
||
"candidate_words = build_candidate_vocabulary(X_train, y_train)\n",
|
||
"print(f\"[+] Testing {len(candidate_words)} candidate words extracted from ham messages\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 22,
|
||
"id": "1ff0d821-d5e0-4b18-b07c-2fa5252da403",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"[*] Budget allocation:\n",
|
||
" exploration : 400 queries\n",
|
||
" exploitation: 400 queries\n",
|
||
" combination : 200 queries\n",
|
||
" Total: 1000 / 1000\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"def estimate_budget_allocation(total_budget):\n",
|
||
" \"\"\"\n",
|
||
" Estimate allocation across exploration, exploitation, and combination.\n",
|
||
"\n",
|
||
" Parameters\n",
|
||
" ----------\n",
|
||
" total_budget : int\n",
|
||
" Total query budget available for discovery.\n",
|
||
"\n",
|
||
" Returns\n",
|
||
" -------\n",
|
||
" dict\n",
|
||
" Mapping phase -> integer number of queries that sums to `total_budget`.\n",
|
||
" \"\"\"\n",
|
||
" explore = int(0.4 * total_budget)\n",
|
||
" exploit = int(0.4 * total_budget)\n",
|
||
" combine = total_budget - explore - exploit # absorb rounding\n",
|
||
" return {\n",
|
||
" 'exploration': explore,\n",
|
||
" 'exploitation': exploit,\n",
|
||
" 'combination': combine,\n",
|
||
" }\n",
|
||
"\n",
|
||
"# Quick demo for budget allocation\n",
|
||
"allocation = estimate_budget_allocation(query_budget)\n",
|
||
"print(\"\\n[*] Budget allocation:\")\n",
|
||
"for phase, budget in allocation.items():\n",
|
||
" print(f\" {phase:12}: {budget:4d} queries\")\n",
|
||
"print(f\" Total: {sum(allocation.values())} / {query_budget}\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 23,
|
||
"id": "3e399027-9815-42ee-875e-c00798d3a534",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"[*] Discovery phase: testing 111 candidates...\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# Discovery phase - test word effectiveness\n",
|
||
"word_scores = {}\n",
|
||
"test_spam_samples = spam_test_messages[:50] # More test messages\n",
|
||
"\n",
|
||
"# Test in batches to be more efficient\n",
|
||
"print(f\"[*] Discovery phase: testing {len(candidate_words)} candidates...\")\n",
|
||
"\n",
|
||
"# Randomly sample candidates and messages for better coverage\n",
|
||
"np.random.shuffle(candidate_words)\n",
|
||
"np.random.shuffle(test_spam_samples)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 24,
|
||
"id": "031afe59-0dcb-4a13-b8c2-7df1d34aaebe",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def initialize_adaptive_scorer():\n",
|
||
" \"\"\"Initialize adaptive scoring data structures\"\"\"\n",
|
||
" return {\n",
|
||
" 'word_scores': {}, # Maps word -> effectiveness score\n",
|
||
" 'word_counts': {}, # Maps word -> number of times tested\n",
|
||
" 'exploration_rate': 0.2 # 20% exploration for discovery phase\n",
|
||
" }"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 25,
|
||
"id": "c52c95ea-65e7-4a83-815e-3f0dfcab9ded",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def epsilon_greedy_select(scorer, available_words):\n",
|
||
" \"\"\"Select word using epsilon-greedy strategy\n",
|
||
"\n",
|
||
" Parameters:\n",
|
||
" scorer (dict): Adaptive scorer state\n",
|
||
" available_words (list): Candidate words to choose from\n",
|
||
"\n",
|
||
" Returns:\n",
|
||
" str: Selected word for testing\n",
|
||
" \"\"\"\n",
|
||
" import random\n",
|
||
"\n",
|
||
" if random.random() < scorer['exploration_rate']:\n",
|
||
" # Exploration: try untested or rarely tested words\n",
|
||
" untested = [w for w in available_words if w not in scorer['word_counts']]\n",
|
||
" if untested:\n",
|
||
" return random.choice(untested)\n",
|
||
" else:\n",
|
||
" # Choose least tested word\n",
|
||
" return min(available_words,\n",
|
||
" key=lambda w: scorer['word_counts'].get(w, 0))\n",
|
||
" python\n",
|
||
" else:\n",
|
||
" # Exploitation: choose best performing word\n",
|
||
" return max(available_words,\n",
|
||
" key=lambda w: scorer['word_scores'].get(w, 0))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 26,
|
||
"id": "91904977-0044-4bdc-94e8-64be9a0c889d",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def update_word_score(scorer, word, impact, alpha=0.3):\n",
|
||
" \"\"\"Update word score using exponential moving average\n",
|
||
"\n",
|
||
" Parameters:\n",
|
||
" scorer (dict): Adaptive scorer state\n",
|
||
" word (str): Word being scored\n",
|
||
" impact (float): Observed reduction in spam probability\n",
|
||
" alpha (float): Learning rate\n",
|
||
" \"\"\"\n",
|
||
" if word not in scorer['word_scores']:\n",
|
||
" scorer['word_scores'][word] = impact\n",
|
||
" scorer['word_counts'][word] = 1\n",
|
||
" else:\n",
|
||
" # Exponential moving average\n",
|
||
" old_score = scorer['word_scores'][word]\n",
|
||
" scorer['word_scores'][word] = (1 - alpha) * old_score + alpha * impact\n",
|
||
" scorer['word_counts'][word] += 1"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 27,
|
||
"id": "883fa203-95b5-4965-9a80-e5f052b91c4b",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def discover_word_combinations(message, test_words, max_size=3):\n",
|
||
" \"\"\"Discover effective word combinations through systematic search\n",
|
||
"\n",
|
||
" Parameters:\n",
|
||
" message (str): Target spam message\n",
|
||
" test_words (list): Promising words to test\n",
|
||
" max_size (int): Maximum combination size\n",
|
||
"\n",
|
||
" Returns:\n",
|
||
" dict: Mapping of word combinations to effectiveness scores\n",
|
||
" \"\"\"\n",
|
||
" from itertools import combinations\n",
|
||
"\n",
|
||
" combination_scores = {}\n",
|
||
" message_vec = vectorizer.transform([message])\n",
|
||
" message_score = classifier.predict_proba(message_vec)[0][1]\n",
|
||
"\n",
|
||
" # Test individual words first\n",
|
||
" for word in test_words[:20]:\n",
|
||
" test_message = message + \" \" + word\n",
|
||
" test_vec = vectorizer.transform([test_message])\n",
|
||
" score = classifier.predict_proba(test_vec)[0][1]\n",
|
||
" impact = message_score - score\n",
|
||
" combination_scores[(word,)] = impact\n",
|
||
"\n",
|
||
" # Test pairs for synergy\n",
|
||
" if max_size >= 2:\n",
|
||
" for word1, word2 in combinations(test_words[:15], 2):\n",
|
||
" test_message = message + \" \" + word1 + \" \" + word2\n",
|
||
" test_vec = vectorizer.transform([test_message])\n",
|
||
" score = classifier.predict_proba(test_vec)[0][1]\n",
|
||
"\n",
|
||
" # Calculate synergy\n",
|
||
" individual_impact = combination_scores.get((word1,), 0) + combination_scores.get((word2,), 0)\n",
|
||
" actual_impact = message_score - score\n",
|
||
" synergy = actual_impact - individual_impact\n",
|
||
"\n",
|
||
" if synergy > 0: # Positive synergy detected\n",
|
||
" combination_scores[(word1, word2)] = actual_impact\n",
|
||
"\n",
|
||
" # Test triplets for top pairs\n",
|
||
" if max_size >= 3:\n",
|
||
" top_pairs = sorted(\n",
|
||
" [(k, v) for k, v in combination_scores.items() if len(k) == 2],\n",
|
||
" key=lambda x: x[1], reverse=True\n",
|
||
" )[:5]\n",
|
||
"\n",
|
||
" for pair, pair_score in top_pairs:\n",
|
||
" for word in test_words[:10]:\n",
|
||
" if word not in pair:\n",
|
||
" triplet = tuple(sorted(pair + (word,)))\n",
|
||
" test_message = message + \" \" + \" \".join(triplet)\n",
|
||
" test_vec = vectorizer.transform([test_message])\n",
|
||
" score = classifier.predict_proba(test_vec)[0][1]\n",
|
||
" combination_scores[triplet] = message_score - score\n",
|
||
"\n",
|
||
" return combination_scores"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 28,
|
||
"id": "e4444202-9df3-45dd-ae52-79bced2fe939",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def three_phase_discovery(spam_messages, candidate_words, budget=1000):\n",
|
||
" \"\"\"Three-phase discovery: exploration, exploitation, combination\n",
|
||
"\n",
|
||
" Parameters:\n",
|
||
" spam_messages (list): Target spam messages\n",
|
||
" candidate_words (list): Vocabulary to test\n",
|
||
" budget (int): Total query budget\n",
|
||
"\n",
|
||
" Returns:\n",
|
||
" tuple: (discovered_words, combination_scores, queries_used)\n",
|
||
" \"\"\"\n",
|
||
" scorer = initialize_adaptive_scorer()\n",
|
||
" queries_used = 0\n",
|
||
"\n",
|
||
" # Allocate budgets using 40-40-20 split strategy\n",
|
||
" allocation = estimate_budget_allocation(budget)\n",
|
||
" exploration_budget = allocation['exploration']\n",
|
||
" exploitation_budget = allocation['exploitation']\n",
|
||
" combination_budget = allocation['combination']\n",
|
||
"\n",
|
||
" # Phase 1: Broad exploration (allocated budget)\n",
|
||
" print(f\"[*] Phase 1: Exploration (budget: {exploration_budget} queries)\")\n",
|
||
"\n",
|
||
" p1_marks = {\n",
|
||
" max(1, int(0.25 * exploration_budget)),\n",
|
||
" max(1, int(0.50 * exploration_budget)),\n",
|
||
" max(1, int(0.75 * exploration_budget)),\n",
|
||
" }\n",
|
||
" p1_reported = set()\n",
|
||
"\n",
|
||
" # Select a message and a candidate word\n",
|
||
" test_message = random.choice(spam_messages)\n",
|
||
" word = epsilon_greedy_select(scorer, candidate_words)\n",
|
||
"\n",
|
||
" # Baseline and augmented spam probabilities\n",
|
||
" vec_orig = vectorizer.transform([test_message])\n",
|
||
" prob_orig = classifier.predict_proba(vec_orig)[0][1] # spam prob\n",
|
||
"\n",
|
||
" vec_aug = vectorizer.transform([test_message + \" \" + word])\n",
|
||
" prob_aug = classifier.predict_proba(vec_aug)[0][1]\n",
|
||
"\n",
|
||
" impact = prob_orig - prob_aug\n",
|
||
"\n",
|
||
" # Update running score and consume query budget\n",
|
||
" update_word_score(scorer, word, impact)\n",
|
||
" queries_used += 2\n",
|
||
"\n",
|
||
" # Optional milestone report\n",
|
||
" if queries_used in p1_marks and queries_used not in p1_reported:\n",
|
||
" top3 = sorted(scorer['word_scores'].items(), key=lambda x: x[1], reverse=True)[:3]\n",
|
||
" print(\n",
|
||
" f\" [P1 {queries_used}/{exploration_budget}] \"\n",
|
||
" f\"tested_words={len(scorer['word_scores'])} | \"\n",
|
||
" f\"top3=\" + \", \".join(f\"{w}:{s:.3f}\" for w, s in top3)\n",
|
||
" )\n",
|
||
" p1_reported.add(queries_used)\n",
|
||
"\n",
|
||
" while queries_used < exploration_budget and len(candidate_words) > 0:\n",
|
||
" # Select inputs\n",
|
||
" test_message = random.choice(spam_messages)\n",
|
||
" word = epsilon_greedy_select(scorer, candidate_words)\n",
|
||
"\n",
|
||
" # Measure impact with two queries\n",
|
||
" vec_orig = vectorizer.transform([test_message])\n",
|
||
" prob_orig = classifier.predict_proba(vec_orig)[0][1]\n",
|
||
" vec_aug = vectorizer.transform([test_message + \" \" + word])\n",
|
||
" prob_aug = classifier.predict_proba(vec_aug)[0][1]\n",
|
||
" impact = prob_orig - prob_aug\n",
|
||
"\n",
|
||
" # Update score and account for budget\n",
|
||
" update_word_score(scorer, word, impact)\n",
|
||
" queries_used += 2\n",
|
||
"\n",
|
||
" # Milestone report\n",
|
||
" if queries_used in p1_marks and queries_used not in p1_reported:\n",
|
||
" top3 = sorted(scorer['word_scores'].items(), key=lambda x: x[1], reverse=True)[:3]\n",
|
||
" print(\n",
|
||
" f\" [P1 {queries_used}/{exploration_budget}] \"\n",
|
||
" f\"tested_words={len(scorer['word_scores'])} | \"\n",
|
||
" f\"top3=\" + \", \".join(f\"{w}:{s:.3f}\" for w, s in top3)\n",
|
||
" )\n",
|
||
" p1_reported.add(queries_used)\n",
|
||
"\n",
|
||
" print(f\"[+] Exploration complete. Queries: {queries_used}, Words tested: {len(scorer['word_scores'])}\")\n",
|
||
" top5 = sorted(scorer['word_scores'].items(), key=lambda x: x[1], reverse=True)[:5]\n",
|
||
" if top5:\n",
|
||
" print(\" Top5 after exploration:\")\n",
|
||
" for w, s in top5:\n",
|
||
" print(f\" {w:12} | score: {s:.3f}\")\n",
|
||
"\n",
|
||
" # Phase 2: Focused exploitation\n",
|
||
" scorer['exploration_rate'] = 0.1 # Reduce exploration\n",
|
||
"\n",
|
||
" # Get top words for exploitation\n",
|
||
" top_words = sorted(scorer['word_scores'].items(), key=lambda x: x[1], reverse=True)[:30]\n",
|
||
" top_word_list = [w for w, _ in top_words]\n",
|
||
"\n",
|
||
" print(f\"\\n[*] Phase 2: Exploitation (budget: {exploitation_budget} queries)\")\n",
|
||
" initial_queries = queries_used\n",
|
||
" p2_mid = initial_queries + max(1, exploitation_budget // 2)\n",
|
||
"\n",
|
||
" while queries_used < initial_queries + exploitation_budget and len(top_word_list) > 0:\n",
|
||
" test_message = random.choice(spam_messages[:20]) # Focus on fewer messages\n",
|
||
" word = random.choice(top_word_list[:15]) # Focus on best words\n",
|
||
"\n",
|
||
" vec_orig = vectorizer.transform([test_message])\n",
|
||
" prob_orig = classifier.predict_proba(vec_orig)[0][1]\n",
|
||
"\n",
|
||
" vec_aug = vectorizer.transform([test_message + \" \" + word])\n",
|
||
" prob_aug = classifier.predict_proba(vec_aug)[0][1]\n",
|
||
"\n",
|
||
" impact = prob_orig - prob_aug\n",
|
||
" update_word_score(scorer, word, impact)\n",
|
||
" queries_used += 2\n",
|
||
"\n",
|
||
" print(f\"[+] Exploitation complete. Total queries: {queries_used}\")\n",
|
||
"\n",
|
||
" # Phase 3: Combination discovery (allocated budget)\n",
|
||
" remaining_combo = combination_budget\n",
|
||
" print(f\"\\n[*] Phase 3: Combination search (budget: {remaining_combo} queries)\")\n",
|
||
"\n",
|
||
" best_combinations = {}\n",
|
||
" combos_tested = 0\n",
|
||
"\n",
|
||
" if remaining_combo > 50: # Need minimum queries for combinations\n",
|
||
" for i in range(min(3, len(spam_messages))):\n",
|
||
" if queries_used >= budget or remaining_combo <= 0:\n",
|
||
" break\n",
|
||
"\n",
|
||
" test_msg = spam_messages[i]\n",
|
||
" combos = discover_word_combinations(test_msg, top_word_list[:20], max_size=3)\n",
|
||
"\n",
|
||
" # Track best combinations across messages\n",
|
||
" for combo, score in combos.items():\n",
|
||
" if combo not in best_combinations or score > best_combinations[combo]:\n",
|
||
" best_combinations[combo] = score\n",
|
||
"\n",
|
||
" # Account for queries (~2 per combination) while respecting the budget\n",
|
||
" to_add = min(remaining_combo, len(combos) * 2)\n",
|
||
" queries_used += to_add\n",
|
||
" remaining_combo -= to_add\n",
|
||
" combos_tested += len(combos)\n",
|
||
"\n",
|
||
" # Midpoint snapshot\n",
|
||
" if combination_budget > 0 and remaining_combo <= combination_budget // 2 and best_combinations:\n",
|
||
" best = max(best_combinations.items(), key=lambda x: x[1])\n",
|
||
" print(\n",
|
||
" f\" [P3 mid ~{combination_budget - remaining_combo}/{combination_budget}] \"\n",
|
||
" f\"combos_tested={combos_tested} | best={' + '.join(best[0])}:{best[1]:.3f}\"\n",
|
||
" )\n",
|
||
"\n",
|
||
" if remaining_combo <= 0:\n",
|
||
" break\n",
|
||
"\n",
|
||
" print(f\"[+] Combination search complete. Total queries: {queries_used}\")\n",
|
||
"\n",
|
||
" # Return final results\n",
|
||
" final_words = sorted(scorer['word_scores'].items(), key=lambda x: x[1], reverse=True)\n",
|
||
" return final_words, best_combinations, queries_used\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "f247ef8e-f284-4057-8626-70fa863ae43d",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Run Three-Phase Discovery"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 29,
|
||
"id": "d3579810-3723-4bef-b819-e1296de2c0d5",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"[*] Using three-phase discovery algorithm...\n",
|
||
"[+] Built vocabulary of 111 candidate words\n",
|
||
"\n",
|
||
"[*] Budget allocation:\n",
|
||
" exploration : 400 queries\n",
|
||
" exploitation: 400 queries\n",
|
||
" combination : 200 queries\n",
|
||
"[*] Phase 1: Exploration (budget: 400 queries)\n",
|
||
" [P1 100/400] tested_words=11 | top3=remember:0.014, time:0.012, thats:0.007\n",
|
||
" [P1 200/400] tested_words=17 | top3=remember:0.004, time:0.004, ...:0.003\n",
|
||
" [P1 300/400] tested_words=30 | top3=...:0.003, thats:0.002, time:0.002\n",
|
||
"[+] Exploration complete. Queries: 400, Words tested: 40\n",
|
||
" Top5 after exploration:\n",
|
||
" come | score: 0.030\n",
|
||
" work | score: 0.027\n",
|
||
" yes | score: 0.026\n",
|
||
" remember | score: 0.002\n",
|
||
" ... | score: 0.001\n",
|
||
"\n",
|
||
"[*] Phase 2: Exploitation (budget: 400 queries)\n",
|
||
"[+] Exploitation complete. Total queries: 800\n",
|
||
"\n",
|
||
"[*] Phase 3: Combination search (budget: 200 queries)\n",
|
||
" [P3 mid ~200/200] combos_tested=170 | best=... + ask + doing:0.000\n",
|
||
"[+] Combination search complete. Total queries: 1000\n",
|
||
"\n",
|
||
"[+] Discovery complete. Total queries used: 1000/1000\n",
|
||
"[+] Top 10 discovered words:\n",
|
||
" remember | impact: 0.217\n",
|
||
" make | impact: 0.159\n",
|
||
" come | impact: 0.134\n",
|
||
" doing | impact: 0.099\n",
|
||
" time | impact: 0.095\n",
|
||
" ... | impact: 0.093\n",
|
||
" did | impact: 0.083\n",
|
||
" ask | impact: 0.047\n",
|
||
" cos | impact: 0.043\n",
|
||
" want | impact: 0.034\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"print(\"\\n[*] Using three-phase discovery algorithm...\")\n",
|
||
"\n",
|
||
"# Build candidate vocabulary\n",
|
||
"candidate_words = build_candidate_vocabulary(X_train, y_train)\n",
|
||
"print(f\"[+] Built vocabulary of {len(candidate_words)} candidate words\")\n",
|
||
"\n",
|
||
"# Show budget allocation\n",
|
||
"allocation = estimate_budget_allocation(query_budget)\n",
|
||
"print(f\"\\n[*] Budget allocation:\")\n",
|
||
"for phase, budget in allocation.items():\n",
|
||
" print(f\" {phase:12}: {budget:4d} queries\")\n",
|
||
"\n",
|
||
"# Run three-phase discovery\n",
|
||
"discovered_words, combination_scores, total_queries = three_phase_discovery(\n",
|
||
" spam_test_messages[:50],\n",
|
||
" candidate_words,\n",
|
||
" budget=query_budget\n",
|
||
")\n",
|
||
"\n",
|
||
"print(f\"\\n[+] Discovery complete. Total queries used: {total_queries}/{query_budget}\")\n",
|
||
"print(f\"[+] Top 10 discovered words:\")\n",
|
||
"for word, score in discovered_words[:10]:\n",
|
||
" print(f\" {word:10} | impact: {score:.3f}\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 30,
|
||
"id": "10362c0c-a816-4c51-b62f-165963b5ee3a",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"[+] Top 5 word combinations:\n",
|
||
" ..., ask, doing | synergy: 0.000\n",
|
||
" ..., come, doing | synergy: 0.000\n",
|
||
" ..., ask, come | synergy: 0.000\n",
|
||
" ..., come, cos | synergy: 0.000\n",
|
||
" ..., doing, work | synergy: 0.000\n",
|
||
"\n",
|
||
"[+] Black-box attack complete. Total queries: 1000/1000\n",
|
||
"\n",
|
||
"============================================================\n",
|
||
"ATTACK SUMMARY\n",
|
||
"============================================================\n",
|
||
"Model Accuracy: 98.55%\n",
|
||
"Best White-box Evasion: 100.0% @ 20 words\n",
|
||
"Most Effective Word: 'lor' (reduces spam prob by 7.1%)\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"if combination_scores:\n",
|
||
" print(f\"\\n[+] Top 5 word combinations:\")\n",
|
||
" top_combos = sorted(combination_scores.items(), key=lambda x: x[1], reverse=True)[:5]\n",
|
||
" for combo, score in top_combos:\n",
|
||
" combo_str = ', '.join(combo)\n",
|
||
" print(f\" {combo_str:30} | synergy: {score:.3f}\")\n",
|
||
"\n",
|
||
"# Update queries_used for compatibility\n",
|
||
"queries_used = total_queries\n",
|
||
"\n",
|
||
"# Test discovered words\n",
|
||
"blackbox_results = []\n",
|
||
"test_counts = [0, 5, 10, 15, 20, 25, 30] # Test with more words\n",
|
||
"\n",
|
||
"for num_words in test_counts:\n",
|
||
" if queries_used >= query_budget:\n",
|
||
" break\n",
|
||
"\n",
|
||
" selected = [w for w, _ in discovered_words[:num_words]]\n",
|
||
" evaded = 0\n",
|
||
" tested = 0\n",
|
||
"\n",
|
||
" # Test on a different subset of spam messages\n",
|
||
" eval_messages = spam_test_messages[30:50] # Different messages from discovery\n",
|
||
"\n",
|
||
" for msg in eval_messages:\n",
|
||
" if queries_used >= query_budget:\n",
|
||
" break\n",
|
||
"\n",
|
||
" aug = msg if num_words == 0 else msg + \" \" + \" \".join(selected)\n",
|
||
" vec = vectorizer.transform([aug])\n",
|
||
" prob = classifier.predict_proba(vec)[0][1] # spam prob\n",
|
||
" queries_used += 1\n",
|
||
"\n",
|
||
" if prob < 0.5: # evasion threshold\n",
|
||
" evaded += 1\n",
|
||
" tested += 1\n",
|
||
"\n",
|
||
" if tested > 0:\n",
|
||
" rate = (evaded / tested) * 100\n",
|
||
" blackbox_results.append({'num_words': num_words, 'evasion_rate': rate})\n",
|
||
" print(f\" Words: {num_words:2d} | Evasion: {rate:6.2f}% | Queries total: {queries_used}\")\n",
|
||
"\n",
|
||
"print(f\"\\n[+] Black-box attack complete. Total queries: {queries_used}/{query_budget}\")\n",
|
||
"\n",
|
||
"print(\"\\n\" + \"=\"*60)\n",
|
||
"print(\"ATTACK SUMMARY\")\n",
|
||
"print(\"=\"*60)\n",
|
||
"print(f\"Model Accuracy: {test_acc:.2%}\")\n",
|
||
"print(f\"Best White-box Evasion: {results_df['evasion_rate'].max():.1f}% @ {results_df.loc[results_df['evasion_rate'].idxmax(), 'num_words']} words\")\n",
|
||
"if blackbox_results:\n",
|
||
" bb_max = max(r['evasion_rate'] for r in blackbox_results)\n",
|
||
" print(f\"Best Black-box Evasion: {bb_max:.1f}% (with {queries_used} queries)\")\n",
|
||
"print(f\"Most Effective Word: '{word_impacts[0][0]}' (reduces spam prob by {word_impacts[0][1]:.1f}%)\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 31,
|
||
"id": "f0141b1b-f456-4865-82ab-511de60e8f41",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"[+] Results saved to attachments/results.json\n",
|
||
"[+] Attack demonstration complete!\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# Save results\n",
|
||
"results = {\n",
|
||
" 'white_box': attack_results,\n",
|
||
" 'black_box': blackbox_results,\n",
|
||
" 'top_good_words': [(w, float(s)) for w, s, _, _ in top_good_words[:20]],\n",
|
||
" 'word_impacts': word_impacts[:10]\n",
|
||
"}\n",
|
||
"\n",
|
||
"with open(output_dir / \"results.json\", 'w') as f:\n",
|
||
" json.dump(results, f, indent=2)\n",
|
||
"\n",
|
||
"print(f\"\\n[+] Results saved to {output_dir / 'results.json'}\")\n",
|
||
"print(\"[+] Attack demonstration complete!\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "02e40ece-f285-4e0e-9a6b-b679f802a376",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "Python 3 (ipykernel)",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.11.6"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|