{ "cells": [ { "cell_type": "markdown", "id": "36ec729e-fdba-4cd3-9a54-a255dff0b1c6", "metadata": { "editable": true, "slideshow": { "slide_type": "" }, "tags": [] }, "source": [ "# Setup " ] }, { "cell_type": "code", "execution_count": 1, "id": "f04bdbd1-41c5-4c6e-b067-899d258c5863", "metadata": {}, "outputs": [], "source": [ "import os\n", "import json\n", "import numpy as np\n", "import torch\n", "import torch.nn.functional as F\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.preprocessing import StandardScaler\n", "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, classification_report" ] }, { "cell_type": "code", "execution_count": 2, "id": "607dd542-f8c4-463a-942f-833529b36283", "metadata": {}, "outputs": [], "source": [ "from htb_ai_library import (\n", " set_reproducibility, use_htb_style,\n", " MLP, AttackModel,\n", " load_adult_census,\n", " train_fixed_epochs, train_with_early_stopping, evaluate_model,\n", " get_model_predictions, prepare_attack_data, create_dataloader,\n", " plot_training_history, plot_overfitting_gap, plot_confidence_distributions,\n", " plot_shadow_confidence_distributions, plot_attack_roc_curve, plot_precision_recall_curve,\n", " plot_attack_accuracy_comparison, analyze_attack_decision_boundary, plot_decision_boundary,\n", ")" ] }, { "cell_type": "code", "execution_count": 3, "id": "c3dd5b87-6ce5-429c-aee4-7bb74906e42f", "metadata": {}, "outputs": [], "source": [ "RANDOM_SEED = 1337\n", "DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "set_reproducibility(RANDOM_SEED)\n", "use_htb_style()" ] }, { "cell_type": "code", "execution_count": 4, "id": "91df6714-97d6-42f4-be6d-cb13c129ce54", "metadata": {}, "outputs": [], "source": [ "OUTPUT_DIR = \"output\"\n", "MODEL_DIR = f\"{OUTPUT_DIR}/models\"\n", "FIGS_DIR = \"figs\"\n", "FIG_PREFIX = \"Introduction_\"\n", "os.makedirs(MODEL_DIR, exist_ok=True)\n", "os.makedirs(FIGS_DIR, exist_ok=True)\n", "\n", "DATASET_CONFIG = {\n", " \"num_classes\": 2,\n", "}" ] }, { "cell_type": "code", "execution_count": 5, "id": "b886c546-96d9-4e6d-a624-e04ba13135cd", "metadata": {}, "outputs": [], "source": [ "TARGET_MODEL_CONFIG = {\n", " \"hidden_layers\": [256, 128],\n", " \"dropout\": 0.0, # No dropout to maximize overfitting\n", " \"epochs\": 100,\n", " \"batch_size\": 32,\n", " \"learning_rate\": 0.001,\n", "}" ] }, { "cell_type": "code", "execution_count": 6, "id": "3a8e92bd-28b2-4fee-98d3-52e046d6460a", "metadata": {}, "outputs": [], "source": [ "SHADOW_MODEL_CONFIG = {\n", " \"num_shadow_models\": 5,\n", " \"hidden_layers\": [128, 64],\n", " \"dropout\": 0.3,\n", " \"epochs\": 100,\n", " \"batch_size\": 64,\n", " \"learning_rate\": 0.001,\n", " \"early_stopping_patience\": 10,\n", " \"shadow_data_size\": 0.5,\n", "}" ] }, { "cell_type": "code", "execution_count": 7, "id": "9d74286a-4dd8-433d-83b2-cedea64f286b", "metadata": {}, "outputs": [], "source": [ "ATTACK_MODEL_CONFIG = {\n", " \"hidden_layers\": [64, 32],\n", " \"dropout\": 0.2,\n", " \"epochs\": 100,\n", " \"batch_size\": 128,\n", " \"learning_rate\": 0.001,\n", " \"early_stopping_patience\": 15,\n", "}" ] }, { "cell_type": "code", "execution_count": 8, "id": "7c71ab82-08fa-4d58-8dcf-d788348118d7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loading Adult Census dataset...\n", "Dataset loaded: 14 features\n", " Target training (members): 24421 samples\n", " Shadow training: 12210 samples\n", " Attack evaluation (non-members): 12211 samples\n" ] } ], "source": [ "print(\"Loading Adult Census dataset...\")\n", "X_target, y_target, X_shadow, y_shadow, X_attack_eval, y_attack_eval, num_features = load_adult_census(\n", " random_state=RANDOM_SEED\n", ")\n", "\n", "print(f\"Dataset loaded: {num_features} features\")\n", "print(f\" Target training (members): {len(X_target)} samples\")\n", "print(f\" Shadow training: {len(X_shadow)} samples\")\n", "print(f\" Attack evaluation (non-members): {len(X_attack_eval)} samples\")" ] }, { "cell_type": "markdown", "id": "955d799e-85d0-4a64-b80c-6b6de8a78e9f", "metadata": {}, "source": [ "# Training the Target Model\n", "\n", "The target model is the victim we will attack. We train it to deliberately overfit, starting with data preparation:" ] }, { "cell_type": "code", "execution_count": 9, "id": "62eee3d5-7cf0-4879-8903-8863920a1ee6", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "============================================================\n", "Training Target Model\n", "============================================================\n", "Architecture: 14 -> [256, 128] -> 2\n", "Training for 100 epochs (no early stopping)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Training: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:47<00:00, 2.08it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Target Model Performance:\n", " Training Accuracy: 0.9373\n", " Test Accuracy: 0.8247\n", " Overfitting Gap: 0.1125\n" ] } ], "source": [ "print(\"\\n\" + \"=\" * 60)\n", "print(\"Training Target Model\")\n", "print(\"=\" * 60)\n", "\n", "scaler = StandardScaler()\n", "X_target_norm = scaler.fit_transform(X_target)\n", "X_attack_eval_norm = scaler.transform(X_attack_eval)\n", "\n", "train_loader = create_dataloader(X_target_norm, y_target, TARGET_MODEL_CONFIG['batch_size'])\n", "test_loader = create_dataloader(X_attack_eval_norm, y_attack_eval,\n", " TARGET_MODEL_CONFIG['batch_size'], shuffle=False)\n", "\n", "target_model = MLP(\n", " input_size=num_features,\n", " hidden_layers=TARGET_MODEL_CONFIG['hidden_layers'],\n", " num_classes=DATASET_CONFIG['num_classes'],\n", " dropout=TARGET_MODEL_CONFIG['dropout']\n", ")\n", "\n", "print(f\"Architecture: {num_features} -> {TARGET_MODEL_CONFIG['hidden_layers']} -> 2\")\n", "print(f\"Training for {TARGET_MODEL_CONFIG['epochs']} epochs (no early stopping)\")\n", "\n", "history = train_fixed_epochs(\n", " target_model, train_loader, test_loader,\n", " device=DEVICE,\n", " epochs=TARGET_MODEL_CONFIG['epochs'],\n", " learning_rate=TARGET_MODEL_CONFIG['learning_rate']\n", ")\n", "\n", "train_acc, _, _ = evaluate_model(target_model, train_loader, DEVICE)\n", "test_acc, _, _ = evaluate_model(target_model, test_loader, DEVICE)\n", "\n", "print(f\"\\nTarget Model Performance:\")\n", "print(f\" Training Accuracy: {train_acc:.4f}\")\n", "print(f\" Test Accuracy: {test_acc:.4f}\")\n", "print(f\" Overfitting Gap: {train_acc - test_acc:.4f}\")\n", "\n", "plot_overfitting_gap(train_acc, test_acc,\n", " save_path=os.path.join(FIGS_DIR, f\"{FIG_PREFIX}overfitting_gap.png\"))\n", "\n", "def predict_proba(self, x):\n", " logits = self.forward(x)\n", " return F.softmax(logits, dim=1)" ] }, { "cell_type": "markdown", "id": "e0792f6a-cac3-44a7-9ea4-3f4e3d8d8f84", "metadata": {}, "source": [ "# Traing Shadow Models" ] }, { "cell_type": "code", "execution_count": 10, "id": "dc440eff-452b-4712-9e4b-844a7e0aa543", "metadata": { "editable": true, "slideshow": { "slide_type": "" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "============================================================\n", "Training Shadow Models\n", "============================================================\n", "\n", "Created 5 shadow model data splits\n", "Samples per shadow model: ~6105 in, ~6105 out\n", "\n", "Training Shadow Model 1/5\n", " Shadow 1 - Train Acc: 0.8572, Out Acc: 0.8488\n", "\n", "Training Shadow Model 2/5\n", " Shadow 2 - Train Acc: 0.8624, Out Acc: 0.8491\n", "\n", "Training Shadow Model 3/5\n", " Shadow 3 - Train Acc: 0.8598, Out Acc: 0.8531\n", "\n", "Training Shadow Model 4/5\n", " Shadow 4 - Train Acc: 0.8678, Out Acc: 0.8460\n", "\n", "Training Shadow Model 5/5\n", " Shadow 5 - Train Acc: 0.8690, Out Acc: 0.8450\n", "\n", "Total attack training samples: 61050\n", " Members: 30525\n", " Non-members: 30525\n" ] } ], "source": [ "print(\"\\n\" + \"=\" * 60)\n", "print(\"Training Shadow Models\")\n", "print(\"=\" * 60)\n", "\n", "shadow_splits = []\n", "for i in range(SHADOW_MODEL_CONFIG['num_shadow_models']):\n", " seed = RANDOM_SEED + i\n", " X_train_s, X_out_s, y_train_s, y_out_s = train_test_split(\n", " X_shadow, y_shadow, train_size=SHADOW_MODEL_CONFIG['shadow_data_size'],\n", " random_state=seed, stratify=y_shadow\n", " )\n", " shadow_splits.append((X_train_s, X_out_s, y_train_s, y_out_s))\n", "\n", "print(f\"\\nCreated {len(shadow_splits)} shadow model data splits\")\n", "print(f\"Samples per shadow model: ~{len(shadow_splits[0][0])} in, ~{len(shadow_splits[0][1])} out\")\n", "\n", "all_attack_X = []\n", "all_attack_y = []\n", "all_preds_in = []\n", "all_preds_out = []\n", "\n", "for i, (X_train_s, X_out_s, y_train_s, y_out_s) in enumerate(shadow_splits):\n", " print(f\"\\nTraining Shadow Model {i+1}/{SHADOW_MODEL_CONFIG['num_shadow_models']}\")\n", "\n", " # Normalize using target scaler for transferability\n", " X_train_s_norm = scaler.transform(X_train_s)\n", " X_out_s_norm = scaler.transform(X_out_s)\n", "\n", " # Create validation split for early stopping\n", " X_tr_s, X_val_s, y_tr_s, y_val_s = train_test_split(\n", " X_train_s_norm, y_train_s, test_size=0.2,\n", " random_state=RANDOM_SEED + i, stratify=y_train_s\n", " )\n", " train_loader_s = create_dataloader(X_tr_s, y_tr_s, SHADOW_MODEL_CONFIG['batch_size'])\n", " val_loader_s = create_dataloader(X_val_s, y_val_s, SHADOW_MODEL_CONFIG['batch_size'], shuffle=False)\n", "\n", " # Initialize and train shadow model\n", " shadow_model = MLP(\n", " input_size=num_features,\n", " hidden_layers=SHADOW_MODEL_CONFIG['hidden_layers'],\n", " num_classes=DATASET_CONFIG['num_classes'],\n", " dropout=SHADOW_MODEL_CONFIG['dropout']\n", " )\n", " train_with_early_stopping(\n", " shadow_model, train_loader_s, val_loader_s,\n", " device=DEVICE,\n", " epochs=SHADOW_MODEL_CONFIG['epochs'],\n", " learning_rate=SHADOW_MODEL_CONFIG['learning_rate'],\n", " patience=SHADOW_MODEL_CONFIG['early_stopping_patience'],\n", " verbose=False\n", " )\n", "\n", " # Collect predictions on members and non-members\n", " preds_in = get_model_predictions(shadow_model, X_train_s_norm, DEVICE)\n", " preds_out = get_model_predictions(shadow_model, X_out_s_norm, DEVICE)\n", "\n", " # Transform to attack features and accumulate\n", " attack_X_s, attack_y_s = prepare_attack_data(preds_in, preds_out, y_train_s, y_out_s)\n", " all_attack_X.append(attack_X_s)\n", " all_attack_y.append(attack_y_s)\n", " all_preds_in.append(preds_in)\n", " all_preds_out.append(preds_out)\n", "\n", " # Verify overfitting gap exists\n", " full_train_loader_s = create_dataloader(X_train_s_norm, y_train_s,\n", " SHADOW_MODEL_CONFIG['batch_size'], shuffle=False)\n", " full_out_loader_s = create_dataloader(X_out_s_norm, y_out_s,\n", " SHADOW_MODEL_CONFIG['batch_size'], shuffle=False)\n", " train_acc_s, _, _ = evaluate_model(shadow_model, full_train_loader_s, DEVICE)\n", " out_acc_s, _, _ = evaluate_model(shadow_model, full_out_loader_s, DEVICE)\n", " print(f\" Shadow {i+1} - Train Acc: {train_acc_s:.4f}, Out Acc: {out_acc_s:.4f}\")\n", "\n", "attack_X = np.concatenate(all_attack_X, axis=0)\n", "attack_y = np.concatenate(all_attack_y, axis=0)\n", "\n", "print(f\"\\nTotal attack training samples: {len(attack_X)}\")\n", "print(f\" Members: {np.sum(attack_y == 1)}\")\n", "print(f\" Non-members: {np.sum(attack_y == 0)}\")" ] }, { "cell_type": "code", "execution_count": 11, "id": "40179133-a5a9-404c-be03-2e156e8422d4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Attack Data Statistics:\n", " Member confidence - Mean: 0.8553, Std: 0.1597\n", " Non-member confidence - Mean: 0.8553, Std: 0.1600\n", " Confidence gap: -0.0000\n" ] } ], "source": [ "member_confidences = attack_X[attack_y == 1, :2].max(axis=1)\n", "non_member_confidences = attack_X[attack_y == 0, :2].max(axis=1)\n", "\n", "print(f\"\\nAttack Data Statistics:\")\n", "print(f\" Member confidence - Mean: {member_confidences.mean():.4f}, Std: {member_confidences.std():.4f}\")\n", "print(f\" Non-member confidence - Mean: {non_member_confidences.mean():.4f}, Std: {non_member_confidences.std():.4f}\")\n", "print(f\" Confidence gap: {member_confidences.mean() - non_member_confidences.mean():.4f}\")" ] }, { "cell_type": "markdown", "id": "9ab6b609-2648-4ae0-8a0d-807c782c1887", "metadata": {}, "source": [ "# Building the Attack Classifier" ] }, { "cell_type": "code", "execution_count": 12, "id": "e3bd9859-2567-4807-a8d7-f77cd4bb101f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "============================================================\n", "Training Attack Model\n", "============================================================\n", "\n", "Attack data split:\n", " Training + Validation: 48840 samples\n", " Test: 12210 samples\n" ] } ], "source": [ "print(\"\\n\" + \"=\" * 60)\n", "print(\"Training Attack Model\")\n", "print(\"=\" * 60)\n", "\n", "X_attack_train, X_attack_test, y_attack_train, y_attack_test = train_test_split(\n", " attack_X, attack_y, test_size=0.2, random_state=RANDOM_SEED, stratify=attack_y\n", ")\n", "\n", "print(f\"\\nAttack data split:\")\n", "print(f\" Training + Validation: {len(X_attack_train)} samples\")\n", "print(f\" Test: {len(X_attack_test)} samples\")" ] }, { "cell_type": "code", "execution_count": 13, "id": "9df5f8d5-ad89-431b-bec3-a8722f08d727", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " Training: 39072 samples\n", " Validation: 9768 samples\n" ] } ], "source": [ "X_attack_tr, X_attack_val, y_attack_tr, y_attack_val = train_test_split(\n", " X_attack_train, y_attack_train, test_size=0.2, random_state=RANDOM_SEED, stratify=y_attack_train\n", ")\n", "\n", "print(f\" Training: {len(X_attack_tr)} samples\")\n", "print(f\" Validation: {len(X_attack_val)} samples\")" ] }, { "cell_type": "code", "execution_count": 14, "id": "1b62b351-1697-4f77-8671-52cf37ac0348", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "DataLoaders created with batch size 128\n" ] } ], "source": [ "attack_train_loader = create_dataloader(X_attack_tr, y_attack_tr, ATTACK_MODEL_CONFIG['batch_size'])\n", "attack_val_loader = create_dataloader(X_attack_val, y_attack_val, ATTACK_MODEL_CONFIG['batch_size'], shuffle=False)\n", "attack_test_loader = create_dataloader(X_attack_test, y_attack_test, ATTACK_MODEL_CONFIG['batch_size'], shuffle=False)\n", "\n", "print(f\"\\nDataLoaders created with batch size {ATTACK_MODEL_CONFIG['batch_size']}\")" ] }, { "cell_type": "code", "execution_count": 15, "id": "61b56511-d196-484b-8ce9-569342585af3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Attack model architecture: 4 -> [64, 32] -> 2\n", "Dropout: 0.2\n", "\n", "Training attack model...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Training: 34%|██████████████████████████████████████████████████████████████▌ | 34/100 [00:08<00:17, 3.80it/s, train_loss=0.6926, val_loss=0.6923, val_acc=0.5081]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Early stopping at epoch 35\n" ] } ], "source": [ "attack_input_size = attack_X.shape[1]\n", "attack_model = AttackModel(\n", " input_size=attack_input_size,\n", " hidden_layers=ATTACK_MODEL_CONFIG['hidden_layers'],\n", " dropout=ATTACK_MODEL_CONFIG['dropout']\n", ")\n", "\n", "print(f\"\\nAttack model architecture: {attack_input_size} -> {ATTACK_MODEL_CONFIG['hidden_layers']} -> 2\")\n", "print(f\"Dropout: {ATTACK_MODEL_CONFIG['dropout']}\")\n", "\n", "print(\"\\nTraining attack model...\")\n", "\n", "history_attack = train_with_early_stopping(\n", " attack_model, attack_train_loader, attack_val_loader,\n", " device=DEVICE,\n", " epochs=ATTACK_MODEL_CONFIG['epochs'],\n", " learning_rate=ATTACK_MODEL_CONFIG['learning_rate'],\n", " patience=ATTACK_MODEL_CONFIG['early_stopping_patience']\n", ")\n", "\n", "plot_training_history(\n", " history_attack,\n", " \"Attack Model Training\",\n", " save_path=os.path.join(FIGS_DIR, f\"{FIG_PREFIX}attack_training.png\")\n", ")" ] }, { "cell_type": "code", "execution_count": 16, "id": "5fd7747d-3b7b-4856-847a-dccf698567aa", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Attack Model Test Performance:\n", " Accuracy: 0.5033\n", " Samples: 12210\n", "\n", "Detailed Classification Report:\n", " precision recall f1-score support\n", "\n", " Non-Member 0.5292 0.0593 0.1066 6105\n", " Member 0.5017 0.9473 0.6560 6105\n", "\n", " accuracy 0.5033 12210\n", " macro avg 0.5155 0.5033 0.3813 12210\n", "weighted avg 0.5155 0.5033 0.3813 12210\n", "\n", "\n", "Attack model saved to output/models/attack_model.pt\n" ] } ], "source": [ "attack_test_acc, attack_test_predictions, attack_test_probs = evaluate_model(attack_model, attack_test_loader, DEVICE)\n", "\n", "print(f\"\\nAttack Model Test Performance:\")\n", "print(f\" Accuracy: {attack_test_acc:.4f}\")\n", "print(f\" Samples: {len(attack_test_predictions)}\")\n", "\n", "print(\"\\nDetailed Classification Report:\")\n", "print(classification_report(\n", " y_attack_test,\n", " attack_test_predictions,\n", " target_names=['Non-Member', 'Member'],\n", " digits=4\n", "))\n", "\n", "# Save the attack model\n", "attack_model_path = os.path.join(MODEL_DIR, \"attack_model.pt\")\n", "torch.save(attack_model.state_dict(), attack_model_path)\n", "print(f\"\\nAttack model saved to {attack_model_path}\")" ] }, { "cell_type": "code", "execution_count": 17, "id": "8b97af19-12db-47d7-ac45-18d315636fc2", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Decision Boundary Analysis:\n", " Class 0: Membership threshold at confidence ~0.535\n", " Class 1: Membership threshold at confidence ~0.500\n" ] } ], "source": [ "boundary_analysis = analyze_attack_decision_boundary(attack_model, DEVICE)\n", "\n", "print(\"\\nDecision Boundary Analysis:\")\n", "for cls, data in boundary_analysis.items():\n", " threshold_idx = np.argmin(np.abs(data['membership_probs'] - 0.5))\n", " threshold_conf = data['confidences'][threshold_idx]\n", " print(f\" Class {cls}: Membership threshold at confidence ~{threshold_conf:.3f}\")" ] }, { "cell_type": "code", "execution_count": 18, "id": "e2400dfe-c68b-4226-aea9-927f675a539a", "metadata": {}, "outputs": [], "source": [ "plot_decision_boundary(\n", " boundary_analysis,\n", " save_path=os.path.join(FIGS_DIR, f\"{FIG_PREFIX}decision_boundary.png\")\n", ")" ] }, { "cell_type": "markdown", "id": "7eb34d7a-5a50-4daa-8dcc-596462be8ade", "metadata": {}, "source": [ "# Executing and Evaluating the Attack" ] }, { "cell_type": "code", "execution_count": 19, "id": "54fc818c-3c91-4013-a9a7-8d5f44b77db0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "============================================================\n", "Executing Membership Inference Attack\n", "============================================================\n", "\n", "Target model predictions collected:\n", " Members: 24421 samples\n", " Non-members: 12211 samples\n", "\n", "Attack input prepared:\n", " Member features: (24421, 4)\n", " Non-member features: (12211, 4)\n", "\n", "Total attack evaluation samples: 36632\n", " Members: 24421\n", " Non-members: 12211\n", "\n", "Attack predictions generated\n", " Mean membership probability: 0.5000\n", "\n", "Membership Inference Attack Results:\n", " Attack Accuracy: 0.6892\n", " Attack Precision: 0.6898\n", " Attack Recall: 0.9702\n", " Attack F1 Score: 0.8063\n" ] } ], "source": [ "print(\"\\n\" + \"=\" * 60)\n", "print(\"Executing Membership Inference Attack\")\n", "print(\"=\" * 60)\n", "\n", "preds_members = get_model_predictions(target_model, X_target_norm, DEVICE)\n", "preds_non_members = get_model_predictions(target_model, X_attack_eval_norm, DEVICE)\n", "\n", "print(f\"\\nTarget model predictions collected:\")\n", "print(f\" Members: {len(preds_members)} samples\")\n", "print(f\" Non-members: {len(preds_non_members)} samples\")\n", "\n", "attack_X_members, attack_y_members = prepare_attack_data(\n", " preds_members, np.zeros((0, preds_members.shape[1])),\n", " y_target, np.array([], dtype=np.int64)\n", ")\n", "\n", "attack_X_non_members, attack_y_non_members = prepare_attack_data(\n", " np.zeros((0, preds_non_members.shape[1])), preds_non_members,\n", " np.array([], dtype=np.int64), y_attack_eval\n", ")\n", "\n", "print(f\"\\nAttack input prepared:\")\n", "print(f\" Member features: {attack_X_members.shape}\")\n", "print(f\" Non-member features: {attack_X_non_members.shape}\")\n", "\n", "attack_X_eval = np.concatenate([attack_X_members, attack_X_non_members], axis=0)\n", "attack_y_eval = np.concatenate([attack_y_members, attack_y_non_members], axis=0)\n", "\n", "print(f\"\\nTotal attack evaluation samples: {len(attack_X_eval)}\")\n", "print(f\" Members: {np.sum(attack_y_eval == 1)}\")\n", "print(f\" Non-members: {np.sum(attack_y_eval == 0)}\")\n", "\n", "attack_eval_loader = create_dataloader(attack_X_eval, attack_y_eval, ATTACK_MODEL_CONFIG['batch_size'], shuffle=False)\n", "\n", "_, attack_predictions, attack_probs = evaluate_model(attack_model, attack_eval_loader, DEVICE)\n", "\n", "membership_probs = attack_probs[:, 1]\n", "\n", "print(f\"\\nAttack predictions generated\")\n", "print(f\" Mean membership probability: {membership_probs.mean():.4f}\")\n", "\n", "attack_accuracy = accuracy_score(attack_y_eval, attack_predictions)\n", "attack_precision = precision_score(attack_y_eval, attack_predictions)\n", "attack_recall = recall_score(attack_y_eval, attack_predictions)\n", "attack_f1 = f1_score(attack_y_eval, attack_predictions)\n", "\n", "print(f\"\\nMembership Inference Attack Results:\")\n", "print(f\" Attack Accuracy: {attack_accuracy:.4f}\")\n", "print(f\" Attack Precision: {attack_precision:.4f}\")\n", "print(f\" Attack Recall: {attack_recall:.4f}\")\n", "print(f\" Attack F1 Score: {attack_f1:.4f}\")" ] }, { "cell_type": "code", "execution_count": 20, "id": "4aeaf5c4-87f3-4d42-a503-2b1768a28747", "metadata": { "editable": true, "slideshow": { "slide_type": "" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Results stored for visualization\n", "\n", "============================================================\n", "Generating Visualizations\n", "============================================================\n", "Attack AUC: 0.5731\n", "\n", "Results saved to figs/Introduction_attack_results.json\n" ] } ], "source": [ "results = {\n", " 'attack_accuracy': attack_accuracy,\n", " 'attack_precision': attack_precision,\n", " 'attack_recall': attack_recall,\n", " 'attack_f1': attack_f1,\n", " 'attack_y_true': attack_y_eval,\n", " 'attack_y_pred': attack_predictions,\n", " 'attack_probs': membership_probs,\n", " 'confidence_members': np.max(preds_members, axis=1),\n", " 'confidence_non_members': np.max(preds_non_members, axis=1),\n", "}\n", "\n", "print(\"\\nResults stored for visualization\")\n", "\n", "print(\"\\n\" + \"=\" * 60)\n", "print(\"Generating Visualizations\")\n", "print(\"=\" * 60)\n", "\n", "auc_score = plot_attack_roc_curve(\n", " results['attack_y_true'],\n", " results['attack_probs'],\n", " save_path=os.path.join(FIGS_DIR, f\"{FIG_PREFIX}attack_roc.png\")\n", ")\n", "results['attack_auc'] = auc_score\n", "\n", "print(f\"Attack AUC: {auc_score:.4f}\")\n", "\n", "plot_precision_recall_curve(\n", " results['attack_y_true'],\n", " results['attack_probs'],\n", " save_path=os.path.join(FIGS_DIR, f\"{FIG_PREFIX}attack_pr.png\")\n", ")\n", "\n", "plot_attack_accuracy_comparison(\n", " results,\n", " save_path=os.path.join(FIGS_DIR, f\"{FIG_PREFIX}attack_metrics.png\")\n", ")\n", "\n", "output = {\n", " 'target_model': {\n", " 'train_accuracy': float(train_acc),\n", " 'test_accuracy': float(test_acc),\n", " 'overfitting_gap': float(train_acc - test_acc),\n", " },\n", " 'attack_results': {\n", " 'accuracy': float(results['attack_accuracy']),\n", " 'precision': float(results['attack_precision']),\n", " 'recall': float(results['attack_recall']),\n", " 'f1_score': float(results['attack_f1']),\n", " 'auc': float(results['attack_auc']),\n", " 'advantage': float(results['attack_accuracy'] - 0.5),\n", " },\n", " 'configuration': {\n", " 'random_seed': RANDOM_SEED,\n", " 'num_shadow_models': SHADOW_MODEL_CONFIG['num_shadow_models'],\n", " 'target_architecture': TARGET_MODEL_CONFIG['hidden_layers'],\n", " 'attack_architecture': ATTACK_MODEL_CONFIG['hidden_layers'],\n", " }\n", "}\n", "\n", "results_path = os.path.join(FIGS_DIR, f\"{FIG_PREFIX}attack_results.json\")\n", "with open(results_path, 'w') as f:\n", " json.dump(output, f, indent=2)\n", "\n", "print(f\"\\nResults saved to {results_path}\")" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.6" } }, "nbformat": 4, "nbformat_minor": 5 }