{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "ea5511c9-c672-4545-ba1f-84f9f3e2d775", "metadata": {}, "outputs": [], "source": [ "import json\n", "import pickle\n", "import random\n", "import re\n", "import urllib.request\n", "import zipfile\n", "from pathlib import Path\n", "import numpy as np\n", "\n", "# Reproducibility\n", "random.seed(1337)\n", "np.random.seed(1337)\n", "\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "from sklearn.feature_extraction.text import CountVectorizer\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.naive_bayes import MultinomialNB\n", "from sklearn.metrics import classification_report, accuracy_score\n", "\n", "from htb_ai_library import (\n", " AZURE,\n", " HACKER_GREY,\n", " HTB_GREEN,\n", " MALWARE_RED,\n", " NODE_BLACK,\n", " NUGGET_YELLOW,\n", " WHITE,\n", " AQUAMARINE,\n", " load_model,\n", " save_model,\n", ")\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "e66ce529-60c0-42ac-8536-02f0ffb88cbc", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "[*] Loading SMS Spam Dataset...\n", "[+] Using cached dataset: data/sms_spam.csv\n", "[+] Loaded 5574 messages\n", " Spam: 747\n", " Ham: 4827\n" ] } ], "source": [ "print(\"\\n[*] Loading SMS Spam Dataset...\")\n", "\n", "data_dir = Path(\"data\")\n", "data_dir.mkdir(exist_ok=True)\n", "dataset_path = data_dir / \"sms_spam.csv\"\n", "\n", "if dataset_path.exists():\n", " print(f\"[+] Using cached dataset: {dataset_path}\")\n", " df = pd.read_csv(dataset_path)\n", "else:\n", " print(\"[*] Downloading from UCI repository...\")\n", " url = \"https://archive.ics.uci.edu/ml/machine-learning-databases/00228/smsspamcollection.zip\"\n", " zip_path = data_dir / \"sms_spam.zip\"\n", "\n", " urllib.request.urlretrieve(url, zip_path)\n", "\n", " with zipfile.ZipFile(zip_path, \"r\") as zf:\n", " with zf.open(\"SMSSpamCollection\") as f:\n", " lines = [line.decode(\"utf-8\").strip() for line in f]\n", "\n", " # Parse tab-separated format\n", " data = []\n", " for line in lines:\n", " parts = line.split('\\t')\n", " if len(parts) == 2:\n", " data.append({\"label\": parts[0].lower(), \"message\": parts[1]})\n", "\n", " df = pd.DataFrame(data)\n", " df.to_csv(dataset_path, index=False)\n", " zip_path.unlink()\n", " print(f\"[+] Dataset saved to {dataset_path}\")\n", "\n", "print(f\"[+] Loaded {len(df)} messages\")\n", "print(f\" Spam: {sum(df['label'] == 'spam')}\")\n", "print(f\" Ham: {sum(df['label'] == 'ham')}\")" ] }, { "cell_type": "code", "execution_count": 3, "id": "4ff06a17-1640-4b1b-9a65-c9ec5686a982", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "[*] Sample messages:\n", "\n", "SPAM samples:\n", " - Free entry in 2 a wkly comp to win FA Cup final tkts 21st May 2005. Text FA to 8...\n", " - FreeMsg Hey there darling it's been 3 week's now and no word back! I'd like some...\n", " - WINNER!! As a valued network customer you have been selected to receivea £900 pr...\n", "\n", "HAM samples:\n", " - Go until jurong point, crazy.. Available only in bugis n great world la e buffet...\n", " - Ok lar... Joking wif u oni......\n", " - U dun say so early hor... U c already then say......\n" ] } ], "source": [ "print(\"\\n[*] Sample messages:\")\n", "print(\"\\nSPAM samples:\")\n", "for msg in df[df['label'] == 'spam']['message'].head(3):\n", " print(f\" - {msg[:80]}...\")\n", "print(\"\\nHAM samples:\")\n", "for msg in df[df['label'] == 'ham']['message'].head(3):\n", " print(f\" - {msg[:80]}...\")" ] }, { "cell_type": "markdown", "id": "3446771b-b673-4e4b-8951-4efe5b3eff7c", "metadata": {}, "source": [ "# Cleaning" ] }, { "cell_type": "code", "execution_count": 4, "id": "4e826b27-ba03-4d56-aa24-92906e8c98b8", "metadata": {}, "outputs": [], "source": [ "import html as html_module\n", "import unicodedata\n", "\n", "def minimal_clean(text):\n", " \"\"\"\n", " Minimal cleaning that preserves spam indicators.\n", "\n", " Parameters: text (str) raw SMS message\n", " Returns: str cleaned text with entities decoded, unicode normalized, and\n", " whitespace collapsed while keeping informative symbols.\n", " \"\"\"\n", " # Decode HTML entities (e.g., & -> &)\n", " text = html_module.unescape(text)\n", "\n", " # Normalize unicode characters\n", " text = unicodedata.normalize('NFKC', text)\n", "\n", " # Clean up excessive whitespace\n", " text = re.sub(r'\\s+', ' ', text)\n", " text = re.sub(r'\\n+', ' ', text)\n", " text = re.sub(r'\\r+', ' ', text)\n", "\n", " return text.strip()\n", "\n", "def clean_text(text):\n", " \"\"\"\n", " Final cleaning for vectorization.\n", "\n", " Converts to lowercase and removes only problematic characters so that\n", " informative symbols remain available to the vectorizer.\n", "\n", " Parameters:\n", " text (str): Preprocessed message from `minimal_clean`.\n", "\n", " Returns:\n", " str: Normalized, whitespace‑collapsed text ready for tokenization.\n", " \"\"\"\n", " text = text.lower()\n", " # Keep numbers, currency symbols, punctuation - they're spam features!\n", " # Only remove truly problematic characters\n", " text = re.sub(r'[^\\w\\s£$€¥!?.,;:\\'\\\"-]', ' ', text)\n", " text = re.sub(r'\\s+', ' ', text)\n", " return text.strip()" ] }, { "cell_type": "markdown", "id": "39bdf320-c877-468f-833f-ba2fbf4025ec", "metadata": {}, "source": [ "# Preprocessing" ] }, { "cell_type": "code", "execution_count": 5, "id": "56d40a5c-77e3-4edf-a111-d157f283ffec", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[*] Applying minimal text cleaning (preserving spam indicators)...\n", "\n", "[*] Sample spam messages with preserved features:\n", " - Summers finally here! Fancy a chat or flirt with sexy singles in yr area? To get MATCHED up just rep...\n", " - This is the 2nd time we have tried 2 contact u. U have won the 750 Pound prize. 2 claim is easy, cal...\n", " - Get ur 1st RINGTONE FREE NOW! Reply to this msg with TONE. Gr8 TOP 20 tones to your phone every week...\n" ] } ], "source": [ "print(\"[*] Applying minimal text cleaning (preserving spam indicators)...\")\n", "df['preprocessed'] = df['message'].apply(minimal_clean)\n", "\n", "# Apply final cleaning for vectorization\n", "df['clean_message'] = df['preprocessed'].apply(clean_text)\n", "\n", "print(\"\\n[*] Sample spam messages with preserved features:\")\n", "spam_samples = df[df['label'] == 'spam'].sample(3, random_state=42)\n", "for idx, row in spam_samples.iterrows():\n", " msg = row['preprocessed'][:100] + \"...\" if len(row['preprocessed']) > 100 else row['preprocessed']\n", " print(f\" - {msg}\")" ] }, { "cell_type": "code", "execution_count": 6, "id": "423ce49f-60c9-4131-aea9-d190560e19cd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "[+] Removed 419 duplicates\n", "[+] Removed 0 empty messages\n" ] } ], "source": [ "# Remove only exact duplicates\n", "original_size = len(df)\n", "df = df.drop_duplicates(subset=['label', 'clean_message'])\n", "print(f\"\\n[+] Removed {original_size - len(df)} duplicates\")\n", "\n", "# Remove empty messages\n", "before_empty = len(df)\n", "df = df[df['clean_message'].str.len() > 0]\n", "print(f\"[+] Removed {before_empty - len(df)} empty messages\")\n" ] }, { "cell_type": "markdown", "id": "9a7e2e62-bb23-4dde-8736-f03fcb25d558", "metadata": {}, "source": [ "# Split" ] }, { "cell_type": "code", "execution_count": 7, "id": "babced9b-8363-41b5-8c60-b96131433a95", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "[+] Data split:\n", " Training: 4124 messages\n", " Testing: 1031 messages\n" ] } ], "source": [ "X = df['clean_message'].values\n", "y = df['label'].values\n", "\n", "X_train, X_test, y_train, y_test = train_test_split(\n", " X, y, test_size=0.2, random_state=42, stratify=y\n", ")\n", "\n", "print(f\"\\n[+] Data split:\")\n", "print(f\" Training: {len(X_train)} messages\")\n", "print(f\" Testing: {len(X_test)} messages\")" ] }, { "cell_type": "markdown", "id": "60a9d5ae-a10a-491f-a05e-c2c8dc59dec1", "metadata": {}, "source": [ "# Train" ] }, { "cell_type": "code", "execution_count": 8, "id": "5de0e33d-9f0c-4c67-b73b-aec1d9832bb0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "[*] Training Naive Bayes classifier...\n", "[+] Loading saved model from models/spam_classifier.pkl\n" ] } ], "source": [ "print(\"\\n[*] Training Naive Bayes classifier...\")\n", "\n", "model_dir = Path(\"models\")\n", "model_dir.mkdir(exist_ok=True)\n", "model_path = model_dir / \"spam_classifier.pkl\"\n", "\n", "if model_path.exists():\n", " print(f\"[+] Loading saved model from {model_path}\")\n", " with open(model_path, 'rb') as f:\n", " saved_data = pickle.load(f)\n", " vectorizer = saved_data['vectorizer']\n", " classifier = saved_data['classifier']\n", "\n", " # Transform data using existing vocabulary\n", " X_train_vec = vectorizer.transform(X_train)\n", " X_test_vec = vectorizer.transform(X_test)\n", "\n", "else:\n", " # Configure vectorizer to capture spam patterns\n", " vectorizer = CountVectorizer(\n", " max_features=3000,\n", " token_pattern=r'\\b\\w+\\b|[£$€¥]+|\\d+|!!+|\\?\\?+|\\.\\.+',\n", " lowercase=True,\n", " stop_words='english'\n", " )\n", " X_train_vec = vectorizer.fit_transform(X_train)\n", " X_test_vec = vectorizer.transform(X_test)\n", " \n", " classifier = MultinomialNB()\n", " classifier.fit(X_train_vec, y_train)\n", "\n", " # Save model for reproducibility\n", " with open(model_path, 'wb') as f:\n", " pickle.dump({'vectorizer': vectorizer, 'classifier': classifier}, f)\n", " print(f\"[+] Model saved to {model_path}\")\n" ] }, { "cell_type": "markdown", "id": "487125a8-5d4f-45cd-bc29-42aad90df8b5", "metadata": {}, "source": [ "# Eval" ] }, { "cell_type": "code", "execution_count": 9, "id": "d3d09aad-c687-49ab-9e42-7f1f21d40ea8", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[+] Training accuracy: 0.9918\n", "[+] Testing accuracy: 0.9855\n", "\n", "[*] Classification Report:\n", " precision recall f1-score support\n", "\n", " ham 0.99 0.99 0.99 903\n", " spam 0.95 0.93 0.94 128\n", "\n", " accuracy 0.99 1031\n", " macro avg 0.97 0.96 0.97 1031\n", "weighted avg 0.99 0.99 0.99 1031\n", "\n" ] } ], "source": [ "# Calculate accuracy scores\n", "train_acc = classifier.score(X_train_vec, y_train)\n", "test_acc = classifier.score(X_test_vec, y_test)\n", "print(f\"[+] Training accuracy: {train_acc:.4f}\")\n", "print(f\"[+] Testing accuracy: {test_acc:.4f}\")\n", "\n", "# Get detailed predictions\n", "y_pred = classifier.predict(X_test_vec)\n", "print(\"\\n[*] Classification Report:\")\n", "print(classification_report(y_test, y_pred))" ] }, { "cell_type": "code", "execution_count": 10, "id": "659be282-cdfe-4d38-9f28-4a293731e81d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "[*] Extracting GoodWords from model...\n", "[+] Top 10 GoodWords (most 'ham-like'):\n", " lor | goodness: 50.94 | ham_p: 0.0047 | spam_p: 0.0001\n", " ü | goodness: 47.57 | ham_p: 0.0044 | spam_p: 0.0001\n", " ... | goodness: 45.81 | ham_p: 0.0299 | spam_p: 0.0007\n", " da | goodness: 45.70 | ham_p: 0.0043 | spam_p: 0.0001\n", " later | goodness: 29.97 | ham_p: 0.0028 | spam_p: 0.0001\n", " doing | goodness: 26.22 | ham_p: 0.0024 | spam_p: 0.0001\n", " really | goodness: 25.47 | ham_p: 0.0024 | spam_p: 0.0001\n", " ask | goodness: 24.72 | ham_p: 0.0023 | spam_p: 0.0001\n", " cos | goodness: 23.60 | ham_p: 0.0022 | spam_p: 0.0001\n", " lol | goodness: 23.60 | ham_p: 0.0022 | spam_p: 0.0001\n" ] } ], "source": [ "print(\"\\n[*] Extracting GoodWords from model...\")\n", "\n", "# Get feature names and probabilities\n", "feature_names = vectorizer.get_feature_names_out()\n", "ham_log_probs = classifier.feature_log_prob_[0] # Ham class\n", "spam_log_probs = classifier.feature_log_prob_[1] # Spam class\n", "\n", "# Calculate goodness scores\n", "goodness_scores = []\n", "for i, word in enumerate(feature_names):\n", " ham_prob = np.exp(ham_log_probs[i])\n", " spam_prob = np.exp(spam_log_probs[i])\n", " goodness = ham_prob / (spam_prob + 1e-10)\n", " goodness_scores.append((word, goodness, ham_prob, spam_prob))\n", "\n", "# Sort by goodness\n", "goodness_scores.sort(key=lambda x: x[1], reverse=True)\n", "top_good_words = goodness_scores[:100]\n", "\n", "print(f\"[+] Top 10 GoodWords (most 'ham-like'):\")\n", "for word, score, hp, sp in top_good_words[:10]:\n", " print(f\" {word:15} | goodness: {score:8.2f} | ham_p: {hp:.4f} | spam_p: {sp:.4f}\")" ] }, { "cell_type": "markdown", "id": "7e816857-59b4-472d-8bf7-c024f5156bd7", "metadata": {}, "source": [ "# Attack" ] }, { "cell_type": "code", "execution_count": 11, "id": "020d5ebc-9d9f-456f-89aa-a58199fb193d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "[*] Testing GoodWords attack...\n", "[+] Testing on 128 spam messages\n", "[*] Testing word counts: [0, 5, 10, 15, 20, 25, 30, 35, 40]\n", " Using words: lor, ü, ..., da, later\n", "\n", "Original: as a registered subscriber yr draw 4 a £100 gift voucher will b entered on receipt of a correct ans. when are the next olympics. txt ans to 80062...\n", "Augmented: as a registered subscriber yr draw 4 a £100 gift voucher will b entered on receipt of a correct ans. when are the next olympics. txt ans to 80062 lor ü ... da later...\n", " Words: 0 | Evasion: 7.03% (9/128)\n", " Words: 5 | Evasion: 41.41% (53/128)\n", " Words: 10 | Evasion: 74.22% (95/128)\n", " Words: 15 | Evasion: 96.09% (123/128)\n", " Words: 20 | Evasion: 100.00% (128/128)\n", " Words: 25 | Evasion: 100.00% (128/128)\n", " Words: 30 | Evasion: 100.00% (128/128)\n", " Words: 35 | Evasion: 100.00% (128/128)\n", " Words: 40 | Evasion: 100.00% (128/128)\n" ] } ], "source": [ "print(\"\\n[*] Testing GoodWords attack...\")\n", "\n", "# Extract only spam messages for testing\n", "spam_test_messages = X_test[y_test == 'spam']\n", "print(f\"[+] Testing on {len(spam_test_messages)} spam messages\")\n", "\n", "# Define test points from baseline (0) to saturation (40)\n", "word_counts = [0, 5, 10, 15, 20, 25, 30, 35, 40]\n", "attack_results = []\n", "\n", "print(f\"[*] Testing word counts: {word_counts}\")\n", "\n", "for num_words in word_counts:\n", " # Select the top N good words for this iteration\n", " selected_words = [w for w, _, _, _ in top_good_words[:num_words]]\n", "\n", " # Show which words we're using (first iteration only for clarity)\n", " if num_words == 5:\n", " print(f\" Using words: {', '.join(selected_words)}\")\n", "\n", "def augment_message(message, words_to_add):\n", " \"\"\"Append good words to a message\"\"\"\n", " if len(words_to_add) > 0:\n", " return message + \" \" + \" \".join(words_to_add)\n", " return message\n", "\n", "# Test augmentation on one example using the top 5 words\n", "sample_spam = spam_test_messages[0]\n", "sample_augmented = augment_message(\n", " sample_spam,\n", " [w for w, _, _, _ in top_good_words[:5]]\n", ")\n", "print(f\"\\nOriginal: {sample_spam}...\")\n", "print(f\"Augmented: {sample_augmented}...\")\n", "\n", "for num_words in word_counts:\n", " # Select the top N good words for this iteration\n", " selected_words = [w for w, _, _, _ in top_good_words[:num_words]]\n", "\n", " # Count how many spam messages evade after augmentation\n", " evaded = 0\n", " for message in spam_test_messages:\n", " # Augment the message\n", " augmented = augment_message(message, selected_words)\n", "\n", " # Transform and predict\n", " vec = vectorizer.transform([augmented])\n", " prob = classifier.predict_proba(vec)[0]\n", "\n", " # Check evasion: ham probability > spam probability\n", " if prob[0] > prob[1]:\n", " evaded += 1\n", "\n", " # Record results for this configuration\n", " evasion_rate = (evaded / len(spam_test_messages)) * 100\n", " attack_results.append({\n", " 'num_words': num_words,\n", " 'evasion_rate': evasion_rate,\n", " 'evaded': evaded,\n", " 'total': len(spam_test_messages)\n", " })\n", "\n", " print(f\" Words: {num_words:2d} | Evasion: {evasion_rate:6.2f}% ({evaded}/{len(spam_test_messages)})\")\n", "\n", "results_df = pd.DataFrame(attack_results)" ] }, { "cell_type": "markdown", "id": "27ca7953-f8b5-4997-a3ce-8db22fc2a544", "metadata": {}, "source": [ "# Plot" ] }, { "cell_type": "code", "execution_count": 12, "id": "1a8ffb38-bd5f-4e94-8861-8f7826ee4fe2", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "[+] Plot saved to attachments/attack_effectiveness.png\n" ] } ], "source": [ "plt.style.use('dark_background')\n", "fig, ax = plt.subplots(figsize=(12, 6), facecolor=NODE_BLACK)\n", "\n", "ax.plot(results_df['num_words'], results_df['evasion_rate'],\n", " marker='o', markersize=8, linewidth=2.5,\n", " color=HTB_GREEN, markeredgecolor='white', markeredgewidth=1)\n", "\n", "ax.fill_between(results_df['num_words'], 0, results_df['evasion_rate'],\n", " alpha=0.3, color=HTB_GREEN)\n", "\n", "# Add threshold lines\n", "ax.axhline(y=50, color=NUGGET_YELLOW, linestyle='--', alpha=0.7, label='50% threshold')\n", "ax.axhline(y=90, color=AZURE, linestyle='--', alpha=0.7, label='90% threshold')\n", "\n", "# Highlight maximum\n", "max_idx = results_df['evasion_rate'].idxmax()\n", "max_rate = results_df.loc[max_idx, 'evasion_rate']\n", "max_words = results_df.loc[max_idx, 'num_words']\n", "ax.scatter(max_words, max_rate, s=200, color=MALWARE_RED, zorder=5)\n", "ax.annotate(f'Peak: {max_rate:.1f}%\\n@ {max_words} words',\n", " xy=(max_words, max_rate), xytext=(max_words+5, max_rate-10),\n", " color='white', fontsize=10,\n", " arrowprops=dict(arrowstyle='->', color=MALWARE_RED, lw=1.5))\n", "\n", "ax.set_xlabel('Number of Good Words Added', fontsize=12, color=HTB_GREEN)\n", "ax.set_ylabel('Evasion Rate (%)', fontsize=12, color=HTB_GREEN)\n", "ax.set_title('GoodWords Attack Effectiveness', fontsize=14, color=HTB_GREEN, pad=20)\n", "ax.grid(True, alpha=0.2)\n", "ax.set_facecolor(NODE_BLACK)\n", "ax.legend()\n", "\n", "for spine in ax.spines.values():\n", " spine.set_color(HACKER_GREY)\n", "ax.tick_params(colors=HACKER_GREY)\n", "\n", "plt.tight_layout()\n", "output_dir = Path(\"attachments\")\n", "output_dir.mkdir(exist_ok=True)\n", "plt.savefig(output_dir / \"attack_effectiveness.png\", dpi=150, facecolor=NODE_BLACK)\n", "plt.close()\n", "print(f\"\\n[+] Plot saved to {output_dir / 'attack_effectiveness.png'}\")" ] }, { "cell_type": "raw", "id": "d6613d7e-8bac-4be9-a7c8-9ed9cf44f870", "metadata": {}, "source": [ "log⁡P(ham∣message)=log⁡P(ham)+∑w∈wordslog⁡P(w∣ham)" ] }, { "cell_type": "markdown", "id": "21dcba53-c378-44d0-923b-cff8c5579503", "metadata": {}, "source": [ "# Optimization (Best Word :0)" ] }, { "cell_type": "code", "execution_count": 13, "id": "f0aaa9b2-3a81-48a6-bd30-60c10934186f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "[*] Analyzing individual word impact...\n", "[+] Testing 20 words on 50 spam messages\n", " Word 'lor': 7.13% reduction\n", " Word 'ü': 7.04% reduction\n", " Word '...': 6.98% reduction\n" ] } ], "source": [ "print(\"\\n[*] Analyzing individual word impact...\")\n", "\n", "# Use 50 spam messages as a representative sample\n", "sample_spam = spam_test_messages[:50]\n", "word_impacts = []\n", "\n", "print(f\"[+] Testing {len(top_good_words[:20])} words on {len(sample_spam)} spam messages\")\n", "\n", "for word, _, _, _ in top_good_words[:20]:\n", " total_impact = 0\n", "\n", " for message in sample_spam:\n", " # Calculate original spam probability\n", " vec_orig = vectorizer.transform([message])\n", " prob_orig = classifier.predict_proba(vec_orig)[0][1] # spam prob\n", "\n", " # Calculate probability after adding the word\n", " vec_aug = vectorizer.transform([message + \" \" + word])\n", " prob_aug = classifier.predict_proba(vec_aug)[0][1]\n", "\n", " # Measure the probability reduction\n", " impact = prob_orig - prob_aug\n", " total_impact += impact\n", "\n", " # Calculate average impact across all messages\n", " avg_impact = (total_impact / len(sample_spam)) * 100\n", " word_impacts.append((word, avg_impact))\n", "\n", " # Show progress for first few words\n", " if len(word_impacts) <= 3:\n", " print(f\" Word '{word}': {avg_impact:.2f}% reduction\")" ] }, { "cell_type": "markdown", "id": "248d4b21-e1dd-4b82-a0ae-d142e5623f37", "metadata": {}, "source": [ "# Plot" ] }, { "cell_type": "code", "execution_count": 14, "id": "74929764-4fcc-4b48-91f5-2e7635295190", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[+] Plot saved to attachments/word_impact.png\n" ] } ], "source": [ "# Plot word impacts\n", "fig, ax = plt.subplots(figsize=(10, 8), facecolor=NODE_BLACK)\n", "\n", "words = [w for w, _ in word_impacts]\n", "impacts = [i for _, i in word_impacts]\n", "colors = [HTB_GREEN if i > 15 else NUGGET_YELLOW if i > 10 else HACKER_GREY for i in impacts]\n", "\n", "bars = ax.barh(range(len(words)), impacts, color=colors, edgecolor='white', linewidth=0.5)\n", "\n", "ax.set_yticks(range(len(words)))\n", "ax.set_yticklabels(words)\n", "ax.set_xlabel('Average Spam Probability Reduction (%)', fontsize=12, color=HTB_GREEN)\n", "ax.set_title('Individual Word Impact on Spam Detection', fontsize=14, color=HTB_GREEN, pad=20)\n", "ax.grid(axis='x', alpha=0.2)\n", "ax.set_facecolor(NODE_BLACK)\n", "\n", "for spine in ax.spines.values():\n", " spine.set_color(HACKER_GREY)\n", "ax.tick_params(colors=HACKER_GREY)\n", "\n", "plt.tight_layout()\n", "plt.savefig(output_dir / \"word_impact.png\", dpi=150, facecolor=NODE_BLACK)\n", "plt.close()\n", "print(f\"[+] Plot saved to {output_dir / 'word_impact.png'}\")" ] }, { "cell_type": "markdown", "id": "7a888c46-129d-4182-afbc-44d1b2cd3cc1", "metadata": {}, "source": [ "# Probability Shift Analysis" ] }, { "cell_type": "code", "execution_count": 15, "id": "a816c1fa-5eba-46b2-b553-ab21cb408bc8", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "[*] Visualizing probability shifts...\n", "[+] Plot saved to attachments/probability_shift.png\n" ] } ], "source": [ "print(\"\\n[*] Visualizing probability shifts...\")\n", "\n", "# Sample messages for detailed analysis\n", "sample_messages = spam_test_messages[:8]\n", "test_word_counts = [0, 5, 10, 20, 30]\n", "\n", "fig, ax = plt.subplots(figsize=(14, 6), facecolor=NODE_BLACK)\n", "\n", "x = np.arange(len(sample_messages))\n", "width = 0.15\n", "colors_list = [MALWARE_RED, NUGGET_YELLOW, AZURE, HTB_GREEN, AQUAMARINE]\n", "\n", "for i, num_words in enumerate(test_word_counts):\n", " # Select the appropriate number of good words\n", " selected = [w for w, _, _, _ in top_good_words[:num_words]]\n", " probs = []\n", "\n", " for msg in sample_messages:\n", " # Augment message with selected words\n", " if num_words > 0:\n", " aug_msg = msg + \" \" + \" \".join(selected)\n", " else:\n", " aug_msg = msg\n", "\n", " # Calculate spam probability\n", " vec = vectorizer.transform([aug_msg])\n", " spam_prob = classifier.predict_proba(vec)[0][1]\n", " probs.append(spam_prob)\n", "\n", " # Create grouped bars with distinct colors\n", " bars = ax.bar(x + i*width, probs, width,\n", " label=f'{num_words} words',\n", " color=colors_list[i], alpha=0.8)\n", "\n", " # Mark successful evasions\n", " for j, (bar, prob) in enumerate(zip(bars, probs)):\n", " if prob < 0.5:\n", " ax.text(bar.get_x() + bar.get_width()/2, prob + 0.02,\n", " '✓', ha='center', va='bottom', color=HTB_GREEN, fontweight='bold')\n", "\n", "ax.axhline(y=0.5, color='white', linestyle='--', alpha=0.5, label='Decision boundary')\n", "ax.set_xlabel('Message Index', fontsize=12, color=HTB_GREEN)\n", "ax.set_ylabel('Spam Probability', fontsize=12, color=HTB_GREEN)\n", "ax.set_title('Probability Shift with Increasing Good Words', fontsize=14, color=HTB_GREEN, pad=20)\n", "ax.set_xticks(x + width * 2)\n", "ax.set_xticklabels([f'M{i+1}' for i in range(len(sample_messages))])\n", "ax.legend(loc='upper right')\n", "ax.grid(axis='y', alpha=0.2)\n", "ax.set_facecolor(NODE_BLACK)\n", "\n", "for spine in ax.spines.values():\n", " spine.set_color(HACKER_GREY)\n", "ax.tick_params(colors=HACKER_GREY)\n", "\n", "plt.tight_layout()\n", "plt.savefig(output_dir / \"probability_shift.png\", dpi=150, facecolor=NODE_BLACK)\n", "plt.close()\n", "print(f\"[+] Plot saved to {output_dir / 'probability_shift.png'}\")" ] }, { "cell_type": "markdown", "id": "61bbc496-93de-413b-bb99-52d82e01e82e", "metadata": {}, "source": [ "# Begin Blackbox attack" ] }, { "cell_type": "code", "execution_count": 16, "id": "62174fd9-69c8-444b-825a-0e9d4c2acf48", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "[*] Simulating black-box attack scenario...\n", "[*] Budget: 1000 queries\n" ] } ], "source": [ "print(\"\\n[*] Simulating black-box attack scenario...\")\n", "print(\"[*] Budget: 1000 queries\")\n", "\n", "# Simulate limited query access\n", "query_budget = 1000\n", "queries_used = 0\n", "query_log = []" ] }, { "cell_type": "code", "execution_count": 17, "id": "985bf085-e771-497d-be38-91b95a6805ba", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[*] Example: extract_ham_word_freq\n", " Ham messages sampled: 500\n", " Unique tokens found: 2162\n", " you: 207\n", " the: 125\n", " and: 107\n", " that: 67\n", " for: 62\n" ] } ], "source": [ "def extract_ham_word_freq(X_train, y_train, sample_size=500):\n", " \"\"\"\n", " Compute token frequencies from a sample of ham messages.\n", "\n", " Parameters\n", " ----------\n", " X_train : array-like of str\n", " Cleaned training messages.\n", " y_train : array-like of str\n", " Labels aligned with X_train ('ham' or 'spam').\n", " sample_size : int, default 500\n", " Number of ham messages to analyze.\n", "\n", " Returns\n", " -------\n", " dict[str, int]\n", " Mapping of word -> frequency within sampled ham messages.\n", " \"\"\"\n", " ham_msgs = X_train[y_train == 'ham']\n", " limit = min(sample_size, len(ham_msgs))\n", " freq = {}\n", " for msg in ham_msgs[:limit]:\n", " for w in str(msg).split():\n", " if 2 < len(w) < 10: # keep typical conversational tokens\n", " freq[w] = freq.get(w, 0) + 1\n", " return freq\n", "\n", "wf_example = extract_ham_word_freq(X_train, y_train, sample_size=500)\n", "print(\"[*] Example: extract_ham_word_freq\")\n", "print(f\" Ham messages sampled: {min(500, sum(y_train == 'ham'))}\")\n", "print(f\" Unique tokens found: {len(wf_example)}\")\n", "top5 = sorted(wf_example.items(), key=lambda x: (-x[1], x[0]))[:5]\n", "for w, c in top5:\n", " print(f\" {w}: {c}\")" ] }, { "cell_type": "code", "execution_count": 18, "id": "425ebd85-4ad2-4e22-b98f-163b497dadb8", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[*] Example: select_high_frequency_words\n", " Selected top words: 100 (min_freq=5)\n", " First 10: you, the, and, that, for, your, are, have, but, not\n" ] } ], "source": [ "def select_high_frequency_words(word_freq, max_words=100, min_freq=5):\n", " \"\"\"\n", " Select the most frequent ham words above a minimum frequency.\n", "\n", " Parameters\n", " ----------\n", " word_freq : dict[str, int]\n", " Token frequency table for sampled ham messages.\n", " max_words : int, default 100\n", " Maximum number of words to return.\n", " min_freq : int, default 5\n", " Minimum frequency a word must meet to be considered.\n", "\n", " Returns\n", " -------\n", " list[str]\n", " Top words sorted by decreasing frequency then lexicographically.\n", " \"\"\"\n", " sorted_by_freq = sorted(word_freq.items(), key=lambda x: (-x[1], x[0]))\n", " top = [w for w, c in sorted_by_freq if c > min_freq][:max_words]\n", " return top\n", "\n", "top_words_example = select_high_frequency_words(wf_example, max_words=100, min_freq=5)\n", "print(\"[*] Example: select_high_frequency_words\")\n", "print(f\" Selected top words: {len(top_words_example)} (min_freq=5)\")\n", "print(\" First 10:\", \", \".join(top_words_example[:10]))" ] }, { "cell_type": "code", "execution_count": 19, "id": "3649a469-5d97-4205-bedc-3d9320bfade0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[*] Example: merge_with_curated\n", " Merged size: 111 | Added curated: 11\n", " Sample added terms: ask, didnt, doing, ill, later\n" ] } ], "source": [ "def merge_with_curated(top_words, additional_candidates=None):\n", " \"\"\"\n", " Merge data-driven top words with curated conversational candidates.\n", "\n", " Parameters\n", " ----------\n", " top_words : list[str]\n", " High-frequency ham words from the previous step.\n", " additional_candidates : list[str] | None\n", " Optional curated list to include regardless of frequency.\n", "\n", " Returns\n", " -------\n", " list[str]\n", " Deduplicated merged list (lexicographically ordered).\n", " \"\"\"\n", " if additional_candidates is None:\n", " additional_candidates = [\n", " \"ok\", \"cos\", \"ill\", \"thats\", \"later\", \"said\", \"ask\", \"didnt\",\n", " \"dont\", \"doing\", \"going\", \"come\", \"home\", \"tomorrow\", \"today\", \"sorry\",\n", " \"thanks\", \"yeah\", \"yes\", \"sure\", \"see\", \"tell\", \"know\", \"think\",\n", " ]\n", " merged = set(top_words) | set(additional_candidates)\n", " return sorted(merged)\n", "\n", "merged_example = merge_with_curated(top_words_example)\n", "added = sorted(set(merged_example) - set(top_words_example))\n", "print(\"[*] Example: merge_with_curated\")\n", "print(f\" Merged size: {len(merged_example)} | Added curated: {len(added)}\")\n", "print(\" Sample added terms:\", \", \".join(added[:5]))" ] }, { "cell_type": "code", "execution_count": 20, "id": "be05bc00-7ec4-4c5d-bb77-9779240cbcf7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[*] Example: build_candidate_vocabulary\n", " Candidates: 111\n", " First 10: you, the, and, that, for, your, are, have, but, not\n" ] } ], "source": [ "def build_candidate_vocabulary(\n", " X_train,\n", " y_train,\n", " sample_size=500,\n", " max_words=100,\n", " min_freq=5,\n", " additional_candidates=None,\n", "):\n", " \"\"\"\n", " Build a candidate vocabulary for black-box discovery from ham messages.\n", "\n", " Parameters\n", " ----------\n", " X_train : array-like of str\n", " Cleaned training messages.\n", " y_train : array-like of str\n", " Labels aligned with X_train ('ham' or 'spam').\n", " sample_size : int, default 500\n", " Number of ham messages to analyze.\n", " max_words : int, default 100\n", " Maximum number of top frequent ham words to keep before merging extras.\n", " min_freq : int, default 5\n", " Minimum frequency threshold for inclusion from the ham corpus.\n", " additional_candidates : list[str] | None\n", " Optional curated conversational terms to include.\n", "\n", " Returns\n", " -------\n", " list[str]\n", " Deduplicated candidate words ordered by decreasing ham frequency,\n", " then lexicographically for stable ties.\n", " \"\"\"\n", " word_freq = extract_ham_word_freq(X_train, y_train, sample_size=sample_size)\n", " top_words = select_high_frequency_words(word_freq, max_words=max_words, min_freq=min_freq)\n", " merged = merge_with_curated(top_words, additional_candidates=additional_candidates)\n", "\n", " # Stable final ordering driven by ham frequency, then lexical for ties\n", " def sort_key(w):\n", " return (-word_freq.get(w, 0), w)\n", "\n", " return sorted(merged, key=sort_key)\n", "\n", "cv_example = build_candidate_vocabulary(X_train, y_train)\n", "print(\"[*] Example: build_candidate_vocabulary\")\n", "print(f\" Candidates: {len(cv_example)}\")\n", "print(\" First 10:\", \", \".join(cv_example[:10]))" ] }, { "cell_type": "code", "execution_count": 21, "id": "109b6a4a-95ab-43b3-a400-5d9f5c99f194", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[+] Testing 111 candidate words extracted from ham messages\n" ] } ], "source": [ "# Build candidate vocabulary for discovery\n", "candidate_words = build_candidate_vocabulary(X_train, y_train)\n", "print(f\"[+] Testing {len(candidate_words)} candidate words extracted from ham messages\")" ] }, { "cell_type": "code", "execution_count": 22, "id": "1ff0d821-d5e0-4b18-b07c-2fa5252da403", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "[*] Budget allocation:\n", " exploration : 400 queries\n", " exploitation: 400 queries\n", " combination : 200 queries\n", " Total: 1000 / 1000\n" ] } ], "source": [ "def estimate_budget_allocation(total_budget):\n", " \"\"\"\n", " Estimate allocation across exploration, exploitation, and combination.\n", "\n", " Parameters\n", " ----------\n", " total_budget : int\n", " Total query budget available for discovery.\n", "\n", " Returns\n", " -------\n", " dict\n", " Mapping phase -> integer number of queries that sums to `total_budget`.\n", " \"\"\"\n", " explore = int(0.4 * total_budget)\n", " exploit = int(0.4 * total_budget)\n", " combine = total_budget - explore - exploit # absorb rounding\n", " return {\n", " 'exploration': explore,\n", " 'exploitation': exploit,\n", " 'combination': combine,\n", " }\n", "\n", "# Quick demo for budget allocation\n", "allocation = estimate_budget_allocation(query_budget)\n", "print(\"\\n[*] Budget allocation:\")\n", "for phase, budget in allocation.items():\n", " print(f\" {phase:12}: {budget:4d} queries\")\n", "print(f\" Total: {sum(allocation.values())} / {query_budget}\")" ] }, { "cell_type": "code", "execution_count": 23, "id": "3e399027-9815-42ee-875e-c00798d3a534", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[*] Discovery phase: testing 111 candidates...\n" ] } ], "source": [ "# Discovery phase - test word effectiveness\n", "word_scores = {}\n", "test_spam_samples = spam_test_messages[:50] # More test messages\n", "\n", "# Test in batches to be more efficient\n", "print(f\"[*] Discovery phase: testing {len(candidate_words)} candidates...\")\n", "\n", "# Randomly sample candidates and messages for better coverage\n", "np.random.shuffle(candidate_words)\n", "np.random.shuffle(test_spam_samples)" ] }, { "cell_type": "code", "execution_count": 24, "id": "031afe59-0dcb-4a13-b8c2-7df1d34aaebe", "metadata": {}, "outputs": [], "source": [ "def initialize_adaptive_scorer():\n", " \"\"\"Initialize adaptive scoring data structures\"\"\"\n", " return {\n", " 'word_scores': {}, # Maps word -> effectiveness score\n", " 'word_counts': {}, # Maps word -> number of times tested\n", " 'exploration_rate': 0.2 # 20% exploration for discovery phase\n", " }" ] }, { "cell_type": "code", "execution_count": 25, "id": "c52c95ea-65e7-4a83-815e-3f0dfcab9ded", "metadata": {}, "outputs": [], "source": [ "def epsilon_greedy_select(scorer, available_words):\n", " \"\"\"Select word using epsilon-greedy strategy\n", "\n", " Parameters:\n", " scorer (dict): Adaptive scorer state\n", " available_words (list): Candidate words to choose from\n", "\n", " Returns:\n", " str: Selected word for testing\n", " \"\"\"\n", " import random\n", "\n", " if random.random() < scorer['exploration_rate']:\n", " # Exploration: try untested or rarely tested words\n", " untested = [w for w in available_words if w not in scorer['word_counts']]\n", " if untested:\n", " return random.choice(untested)\n", " else:\n", " # Choose least tested word\n", " return min(available_words,\n", " key=lambda w: scorer['word_counts'].get(w, 0))\n", " python\n", " else:\n", " # Exploitation: choose best performing word\n", " return max(available_words,\n", " key=lambda w: scorer['word_scores'].get(w, 0))" ] }, { "cell_type": "code", "execution_count": 26, "id": "91904977-0044-4bdc-94e8-64be9a0c889d", "metadata": {}, "outputs": [], "source": [ "def update_word_score(scorer, word, impact, alpha=0.3):\n", " \"\"\"Update word score using exponential moving average\n", "\n", " Parameters:\n", " scorer (dict): Adaptive scorer state\n", " word (str): Word being scored\n", " impact (float): Observed reduction in spam probability\n", " alpha (float): Learning rate\n", " \"\"\"\n", " if word not in scorer['word_scores']:\n", " scorer['word_scores'][word] = impact\n", " scorer['word_counts'][word] = 1\n", " else:\n", " # Exponential moving average\n", " old_score = scorer['word_scores'][word]\n", " scorer['word_scores'][word] = (1 - alpha) * old_score + alpha * impact\n", " scorer['word_counts'][word] += 1" ] }, { "cell_type": "code", "execution_count": 27, "id": "883fa203-95b5-4965-9a80-e5f052b91c4b", "metadata": {}, "outputs": [], "source": [ "def discover_word_combinations(message, test_words, max_size=3):\n", " \"\"\"Discover effective word combinations through systematic search\n", "\n", " Parameters:\n", " message (str): Target spam message\n", " test_words (list): Promising words to test\n", " max_size (int): Maximum combination size\n", "\n", " Returns:\n", " dict: Mapping of word combinations to effectiveness scores\n", " \"\"\"\n", " from itertools import combinations\n", "\n", " combination_scores = {}\n", " message_vec = vectorizer.transform([message])\n", " message_score = classifier.predict_proba(message_vec)[0][1]\n", "\n", " # Test individual words first\n", " for word in test_words[:20]:\n", " test_message = message + \" \" + word\n", " test_vec = vectorizer.transform([test_message])\n", " score = classifier.predict_proba(test_vec)[0][1]\n", " impact = message_score - score\n", " combination_scores[(word,)] = impact\n", "\n", " # Test pairs for synergy\n", " if max_size >= 2:\n", " for word1, word2 in combinations(test_words[:15], 2):\n", " test_message = message + \" \" + word1 + \" \" + word2\n", " test_vec = vectorizer.transform([test_message])\n", " score = classifier.predict_proba(test_vec)[0][1]\n", "\n", " # Calculate synergy\n", " individual_impact = combination_scores.get((word1,), 0) + combination_scores.get((word2,), 0)\n", " actual_impact = message_score - score\n", " synergy = actual_impact - individual_impact\n", "\n", " if synergy > 0: # Positive synergy detected\n", " combination_scores[(word1, word2)] = actual_impact\n", "\n", " # Test triplets for top pairs\n", " if max_size >= 3:\n", " top_pairs = sorted(\n", " [(k, v) for k, v in combination_scores.items() if len(k) == 2],\n", " key=lambda x: x[1], reverse=True\n", " )[:5]\n", "\n", " for pair, pair_score in top_pairs:\n", " for word in test_words[:10]:\n", " if word not in pair:\n", " triplet = tuple(sorted(pair + (word,)))\n", " test_message = message + \" \" + \" \".join(triplet)\n", " test_vec = vectorizer.transform([test_message])\n", " score = classifier.predict_proba(test_vec)[0][1]\n", " combination_scores[triplet] = message_score - score\n", "\n", " return combination_scores" ] }, { "cell_type": "code", "execution_count": 28, "id": "e4444202-9df3-45dd-ae52-79bced2fe939", "metadata": {}, "outputs": [], "source": [ "def three_phase_discovery(spam_messages, candidate_words, budget=1000):\n", " \"\"\"Three-phase discovery: exploration, exploitation, combination\n", "\n", " Parameters:\n", " spam_messages (list): Target spam messages\n", " candidate_words (list): Vocabulary to test\n", " budget (int): Total query budget\n", "\n", " Returns:\n", " tuple: (discovered_words, combination_scores, queries_used)\n", " \"\"\"\n", " scorer = initialize_adaptive_scorer()\n", " queries_used = 0\n", "\n", " # Allocate budgets using 40-40-20 split strategy\n", " allocation = estimate_budget_allocation(budget)\n", " exploration_budget = allocation['exploration']\n", " exploitation_budget = allocation['exploitation']\n", " combination_budget = allocation['combination']\n", "\n", " # Phase 1: Broad exploration (allocated budget)\n", " print(f\"[*] Phase 1: Exploration (budget: {exploration_budget} queries)\")\n", "\n", " p1_marks = {\n", " max(1, int(0.25 * exploration_budget)),\n", " max(1, int(0.50 * exploration_budget)),\n", " max(1, int(0.75 * exploration_budget)),\n", " }\n", " p1_reported = set()\n", "\n", " # Select a message and a candidate word\n", " test_message = random.choice(spam_messages)\n", " word = epsilon_greedy_select(scorer, candidate_words)\n", "\n", " # Baseline and augmented spam probabilities\n", " vec_orig = vectorizer.transform([test_message])\n", " prob_orig = classifier.predict_proba(vec_orig)[0][1] # spam prob\n", "\n", " vec_aug = vectorizer.transform([test_message + \" \" + word])\n", " prob_aug = classifier.predict_proba(vec_aug)[0][1]\n", "\n", " impact = prob_orig - prob_aug\n", "\n", " # Update running score and consume query budget\n", " update_word_score(scorer, word, impact)\n", " queries_used += 2\n", "\n", " # Optional milestone report\n", " if queries_used in p1_marks and queries_used not in p1_reported:\n", " top3 = sorted(scorer['word_scores'].items(), key=lambda x: x[1], reverse=True)[:3]\n", " print(\n", " f\" [P1 {queries_used}/{exploration_budget}] \"\n", " f\"tested_words={len(scorer['word_scores'])} | \"\n", " f\"top3=\" + \", \".join(f\"{w}:{s:.3f}\" for w, s in top3)\n", " )\n", " p1_reported.add(queries_used)\n", "\n", " while queries_used < exploration_budget and len(candidate_words) > 0:\n", " # Select inputs\n", " test_message = random.choice(spam_messages)\n", " word = epsilon_greedy_select(scorer, candidate_words)\n", "\n", " # Measure impact with two queries\n", " vec_orig = vectorizer.transform([test_message])\n", " prob_orig = classifier.predict_proba(vec_orig)[0][1]\n", " vec_aug = vectorizer.transform([test_message + \" \" + word])\n", " prob_aug = classifier.predict_proba(vec_aug)[0][1]\n", " impact = prob_orig - prob_aug\n", "\n", " # Update score and account for budget\n", " update_word_score(scorer, word, impact)\n", " queries_used += 2\n", "\n", " # Milestone report\n", " if queries_used in p1_marks and queries_used not in p1_reported:\n", " top3 = sorted(scorer['word_scores'].items(), key=lambda x: x[1], reverse=True)[:3]\n", " print(\n", " f\" [P1 {queries_used}/{exploration_budget}] \"\n", " f\"tested_words={len(scorer['word_scores'])} | \"\n", " f\"top3=\" + \", \".join(f\"{w}:{s:.3f}\" for w, s in top3)\n", " )\n", " p1_reported.add(queries_used)\n", "\n", " print(f\"[+] Exploration complete. Queries: {queries_used}, Words tested: {len(scorer['word_scores'])}\")\n", " top5 = sorted(scorer['word_scores'].items(), key=lambda x: x[1], reverse=True)[:5]\n", " if top5:\n", " print(\" Top5 after exploration:\")\n", " for w, s in top5:\n", " print(f\" {w:12} | score: {s:.3f}\")\n", "\n", " # Phase 2: Focused exploitation\n", " scorer['exploration_rate'] = 0.1 # Reduce exploration\n", "\n", " # Get top words for exploitation\n", " top_words = sorted(scorer['word_scores'].items(), key=lambda x: x[1], reverse=True)[:30]\n", " top_word_list = [w for w, _ in top_words]\n", "\n", " print(f\"\\n[*] Phase 2: Exploitation (budget: {exploitation_budget} queries)\")\n", " initial_queries = queries_used\n", " p2_mid = initial_queries + max(1, exploitation_budget // 2)\n", "\n", " while queries_used < initial_queries + exploitation_budget and len(top_word_list) > 0:\n", " test_message = random.choice(spam_messages[:20]) # Focus on fewer messages\n", " word = random.choice(top_word_list[:15]) # Focus on best words\n", "\n", " vec_orig = vectorizer.transform([test_message])\n", " prob_orig = classifier.predict_proba(vec_orig)[0][1]\n", "\n", " vec_aug = vectorizer.transform([test_message + \" \" + word])\n", " prob_aug = classifier.predict_proba(vec_aug)[0][1]\n", "\n", " impact = prob_orig - prob_aug\n", " update_word_score(scorer, word, impact)\n", " queries_used += 2\n", "\n", " print(f\"[+] Exploitation complete. Total queries: {queries_used}\")\n", "\n", " # Phase 3: Combination discovery (allocated budget)\n", " remaining_combo = combination_budget\n", " print(f\"\\n[*] Phase 3: Combination search (budget: {remaining_combo} queries)\")\n", "\n", " best_combinations = {}\n", " combos_tested = 0\n", "\n", " if remaining_combo > 50: # Need minimum queries for combinations\n", " for i in range(min(3, len(spam_messages))):\n", " if queries_used >= budget or remaining_combo <= 0:\n", " break\n", "\n", " test_msg = spam_messages[i]\n", " combos = discover_word_combinations(test_msg, top_word_list[:20], max_size=3)\n", "\n", " # Track best combinations across messages\n", " for combo, score in combos.items():\n", " if combo not in best_combinations or score > best_combinations[combo]:\n", " best_combinations[combo] = score\n", "\n", " # Account for queries (~2 per combination) while respecting the budget\n", " to_add = min(remaining_combo, len(combos) * 2)\n", " queries_used += to_add\n", " remaining_combo -= to_add\n", " combos_tested += len(combos)\n", "\n", " # Midpoint snapshot\n", " if combination_budget > 0 and remaining_combo <= combination_budget // 2 and best_combinations:\n", " best = max(best_combinations.items(), key=lambda x: x[1])\n", " print(\n", " f\" [P3 mid ~{combination_budget - remaining_combo}/{combination_budget}] \"\n", " f\"combos_tested={combos_tested} | best={' + '.join(best[0])}:{best[1]:.3f}\"\n", " )\n", "\n", " if remaining_combo <= 0:\n", " break\n", "\n", " print(f\"[+] Combination search complete. Total queries: {queries_used}\")\n", "\n", " # Return final results\n", " final_words = sorted(scorer['word_scores'].items(), key=lambda x: x[1], reverse=True)\n", " return final_words, best_combinations, queries_used\n" ] }, { "cell_type": "markdown", "id": "f247ef8e-f284-4057-8626-70fa863ae43d", "metadata": {}, "source": [ "# Run Three-Phase Discovery" ] }, { "cell_type": "code", "execution_count": 29, "id": "d3579810-3723-4bef-b819-e1296de2c0d5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "[*] Using three-phase discovery algorithm...\n", "[+] Built vocabulary of 111 candidate words\n", "\n", "[*] Budget allocation:\n", " exploration : 400 queries\n", " exploitation: 400 queries\n", " combination : 200 queries\n", "[*] Phase 1: Exploration (budget: 400 queries)\n", " [P1 100/400] tested_words=11 | top3=remember:0.014, time:0.012, thats:0.007\n", " [P1 200/400] tested_words=17 | top3=remember:0.004, time:0.004, ...:0.003\n", " [P1 300/400] tested_words=30 | top3=...:0.003, thats:0.002, time:0.002\n", "[+] Exploration complete. Queries: 400, Words tested: 40\n", " Top5 after exploration:\n", " come | score: 0.030\n", " work | score: 0.027\n", " yes | score: 0.026\n", " remember | score: 0.002\n", " ... | score: 0.001\n", "\n", "[*] Phase 2: Exploitation (budget: 400 queries)\n", "[+] Exploitation complete. Total queries: 800\n", "\n", "[*] Phase 3: Combination search (budget: 200 queries)\n", " [P3 mid ~200/200] combos_tested=170 | best=... + ask + doing:0.000\n", "[+] Combination search complete. Total queries: 1000\n", "\n", "[+] Discovery complete. Total queries used: 1000/1000\n", "[+] Top 10 discovered words:\n", " remember | impact: 0.217\n", " make | impact: 0.159\n", " come | impact: 0.134\n", " doing | impact: 0.099\n", " time | impact: 0.095\n", " ... | impact: 0.093\n", " did | impact: 0.083\n", " ask | impact: 0.047\n", " cos | impact: 0.043\n", " want | impact: 0.034\n" ] } ], "source": [ "print(\"\\n[*] Using three-phase discovery algorithm...\")\n", "\n", "# Build candidate vocabulary\n", "candidate_words = build_candidate_vocabulary(X_train, y_train)\n", "print(f\"[+] Built vocabulary of {len(candidate_words)} candidate words\")\n", "\n", "# Show budget allocation\n", "allocation = estimate_budget_allocation(query_budget)\n", "print(f\"\\n[*] Budget allocation:\")\n", "for phase, budget in allocation.items():\n", " print(f\" {phase:12}: {budget:4d} queries\")\n", "\n", "# Run three-phase discovery\n", "discovered_words, combination_scores, total_queries = three_phase_discovery(\n", " spam_test_messages[:50],\n", " candidate_words,\n", " budget=query_budget\n", ")\n", "\n", "print(f\"\\n[+] Discovery complete. Total queries used: {total_queries}/{query_budget}\")\n", "print(f\"[+] Top 10 discovered words:\")\n", "for word, score in discovered_words[:10]:\n", " print(f\" {word:10} | impact: {score:.3f}\")" ] }, { "cell_type": "code", "execution_count": 30, "id": "10362c0c-a816-4c51-b62f-165963b5ee3a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "[+] Top 5 word combinations:\n", " ..., ask, doing | synergy: 0.000\n", " ..., come, doing | synergy: 0.000\n", " ..., ask, come | synergy: 0.000\n", " ..., come, cos | synergy: 0.000\n", " ..., doing, work | synergy: 0.000\n", "\n", "[+] Black-box attack complete. Total queries: 1000/1000\n", "\n", "============================================================\n", "ATTACK SUMMARY\n", "============================================================\n", "Model Accuracy: 98.55%\n", "Best White-box Evasion: 100.0% @ 20 words\n", "Most Effective Word: 'lor' (reduces spam prob by 7.1%)\n" ] } ], "source": [ "if combination_scores:\n", " print(f\"\\n[+] Top 5 word combinations:\")\n", " top_combos = sorted(combination_scores.items(), key=lambda x: x[1], reverse=True)[:5]\n", " for combo, score in top_combos:\n", " combo_str = ', '.join(combo)\n", " print(f\" {combo_str:30} | synergy: {score:.3f}\")\n", "\n", "# Update queries_used for compatibility\n", "queries_used = total_queries\n", "\n", "# Test discovered words\n", "blackbox_results = []\n", "test_counts = [0, 5, 10, 15, 20, 25, 30] # Test with more words\n", "\n", "for num_words in test_counts:\n", " if queries_used >= query_budget:\n", " break\n", "\n", " selected = [w for w, _ in discovered_words[:num_words]]\n", " evaded = 0\n", " tested = 0\n", "\n", " # Test on a different subset of spam messages\n", " eval_messages = spam_test_messages[30:50] # Different messages from discovery\n", "\n", " for msg in eval_messages:\n", " if queries_used >= query_budget:\n", " break\n", "\n", " aug = msg if num_words == 0 else msg + \" \" + \" \".join(selected)\n", " vec = vectorizer.transform([aug])\n", " prob = classifier.predict_proba(vec)[0][1] # spam prob\n", " queries_used += 1\n", "\n", " if prob < 0.5: # evasion threshold\n", " evaded += 1\n", " tested += 1\n", "\n", " if tested > 0:\n", " rate = (evaded / tested) * 100\n", " blackbox_results.append({'num_words': num_words, 'evasion_rate': rate})\n", " print(f\" Words: {num_words:2d} | Evasion: {rate:6.2f}% | Queries total: {queries_used}\")\n", "\n", "print(f\"\\n[+] Black-box attack complete. Total queries: {queries_used}/{query_budget}\")\n", "\n", "print(\"\\n\" + \"=\"*60)\n", "print(\"ATTACK SUMMARY\")\n", "print(\"=\"*60)\n", "print(f\"Model Accuracy: {test_acc:.2%}\")\n", "print(f\"Best White-box Evasion: {results_df['evasion_rate'].max():.1f}% @ {results_df.loc[results_df['evasion_rate'].idxmax(), 'num_words']} words\")\n", "if blackbox_results:\n", " bb_max = max(r['evasion_rate'] for r in blackbox_results)\n", " print(f\"Best Black-box Evasion: {bb_max:.1f}% (with {queries_used} queries)\")\n", "print(f\"Most Effective Word: '{word_impacts[0][0]}' (reduces spam prob by {word_impacts[0][1]:.1f}%)\")" ] }, { "cell_type": "code", "execution_count": 31, "id": "f0141b1b-f456-4865-82ab-511de60e8f41", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "[+] Results saved to attachments/results.json\n", "[+] Attack demonstration complete!\n" ] } ], "source": [ "# Save results\n", "results = {\n", " 'white_box': attack_results,\n", " 'black_box': blackbox_results,\n", " 'top_good_words': [(w, float(s)) for w, s, _, _ in top_good_words[:20]],\n", " 'word_impacts': word_impacts[:10]\n", "}\n", "\n", "with open(output_dir / \"results.json\", 'w') as f:\n", " json.dump(results, f, indent=2)\n", "\n", "print(f\"\\n[+] Results saved to {output_dir / 'results.json'}\")\n", "print(\"[+] Attack demonstration complete!\")" ] }, { "cell_type": "code", "execution_count": null, "id": "02e40ece-f285-4e0e-9a6b-b679f802a376", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.6" } }, "nbformat": 4, "nbformat_minor": 5 }