added material
This commit is contained in:
Jeremy Janella
@@ -0,0 +1,276 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "15680588-0a0e-4b6d-87db-0fdaa37f03c4",
|
||||
"metadata": {
|
||||
"editable": true,
|
||||
"slideshow": {
|
||||
"slide_type": ""
|
||||
},
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import json\n",
|
||||
"import torch\n",
|
||||
"import torch.optim as optim\n",
|
||||
"from safetensors.torch import save_file\n",
|
||||
"\n",
|
||||
"from htb_ai_library import (\n",
|
||||
" set_reproducibility, use_htb_style,\n",
|
||||
" CIFAR10CNN,\n",
|
||||
" get_cifar10_loaders,\n",
|
||||
" train_baseline_sgd,\n",
|
||||
" train_dp_sgd,\n",
|
||||
" evaluate_accuracy,\n",
|
||||
" compute_mia_advantage,\n",
|
||||
" plot_accuracy_comparison,\n",
|
||||
" plot_privacy_utility_tradeoff,\n",
|
||||
")\n",
|
||||
"RANDOM_SEED = 1337\n",
|
||||
"BATCH_SIZE = 256\n",
|
||||
"BASELINE_EPOCHS = 20\n",
|
||||
"BASELINE_LR = 0.1\n",
|
||||
"DP_EPOCHS = 20\n",
|
||||
"DP_LR = 0.1\n",
|
||||
"MAX_GRAD_NORM = 1.0\n",
|
||||
"DELTA = 1e-5\n",
|
||||
"\n",
|
||||
"set_reproducibility(RANDOM_SEED)\n",
|
||||
"use_htb_style()\n",
|
||||
"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
|
||||
"\n",
|
||||
"os.makedirs(\"figs\", exist_ok=True)\n",
|
||||
"os.makedirs(\"output\", exist_ok=True)\n",
|
||||
"os.makedirs(\"models\", exist_ok=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "61bc8506-b6e5-401e-9d8b-def2eaa5ff94",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"================================================================================\n",
|
||||
" DP-SGD PRIVACY MITIGATION DEMONSTRATION\n",
|
||||
"================================================================================\n",
|
||||
"\n",
|
||||
"Device: cpu\n",
|
||||
"Random seed: 1337\n",
|
||||
"\n",
|
||||
"Loading CIFAR-10 dataset...\n",
|
||||
"Files already downloaded and verified\n",
|
||||
"Files already downloaded and verified\n",
|
||||
"Training samples: 50,000\n",
|
||||
"Test samples: 10,000\n",
|
||||
"Batch size: 256\n",
|
||||
"\n",
|
||||
"================================================================================\n",
|
||||
" TRAINING: BASELINE MODEL (No Privacy Protection)\n",
|
||||
"================================================================================\n",
|
||||
"Epoch [1/20] Loss: nan | Acc: 37.60%\n",
|
||||
"Epoch [2/20] Loss: nan | Acc: 10.00%\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "KeyboardInterrupt",
|
||||
"evalue": "",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
|
||||
"\u001b[31mKeyboardInterrupt\u001b[39m Traceback (most recent call last)",
|
||||
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 19\u001b[39m\n\u001b[32m 16\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33m\"\u001b[39m\u001b[33m=\u001b[39m\u001b[33m\"\u001b[39m * \u001b[32m80\u001b[39m)\n\u001b[32m 18\u001b[39m baseline_model = CIFAR10CNN().to(device)\n\u001b[32m---> \u001b[39m\u001b[32m19\u001b[39m baseline_model = \u001b[43mtrain_baseline_sgd\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbaseline_model\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain_loader\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[43m=\u001b[49m\u001b[43mBASELINE_EPOCHS\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlearning_rate\u001b[49m\u001b[43m=\u001b[49m\u001b[43mBASELINE_LR\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/htb_ai_library/training/loops.py:244\u001b[39m, in \u001b[36mtrain_baseline_sgd\u001b[39m\u001b[34m(model, train_loader, device, epochs, learning_rate, momentum)\u001b[39m\n\u001b[32m 242\u001b[39m output = model(data)\n\u001b[32m 243\u001b[39m loss = criterion(output, target)\n\u001b[32m--> \u001b[39m\u001b[32m244\u001b[39m \u001b[43mloss\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 245\u001b[39m optimizer.step()\n\u001b[32m 246\u001b[39m running_loss += loss.item()\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/torch/_tensor.py:581\u001b[39m, in \u001b[36mTensor.backward\u001b[39m\u001b[34m(self, gradient, retain_graph, create_graph, inputs)\u001b[39m\n\u001b[32m 571\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[32m 572\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[32m 573\u001b[39m Tensor.backward,\n\u001b[32m 574\u001b[39m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[32m (...)\u001b[39m\u001b[32m 579\u001b[39m inputs=inputs,\n\u001b[32m 580\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m581\u001b[39m \u001b[43mtorch\u001b[49m\u001b[43m.\u001b[49m\u001b[43mautograd\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 582\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m=\u001b[49m\u001b[43minputs\u001b[49m\n\u001b[32m 583\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/torch/autograd/__init__.py:347\u001b[39m, in \u001b[36mbackward\u001b[39m\u001b[34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[39m\n\u001b[32m 342\u001b[39m retain_graph = create_graph\n\u001b[32m 344\u001b[39m \u001b[38;5;66;03m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[32m 345\u001b[39m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[32m 346\u001b[39m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m347\u001b[39m \u001b[43m_engine_run_backward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 348\u001b[39m \u001b[43m \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 349\u001b[39m \u001b[43m \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 350\u001b[39m \u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 351\u001b[39m \u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 352\u001b[39m \u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 353\u001b[39m \u001b[43m \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 354\u001b[39m \u001b[43m \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 355\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/torch/autograd/graph.py:825\u001b[39m, in \u001b[36m_engine_run_backward\u001b[39m\u001b[34m(t_outputs, *args, **kwargs)\u001b[39m\n\u001b[32m 823\u001b[39m unregister_hooks = _register_logging_hooks_on_whole_graph(t_outputs)\n\u001b[32m 824\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m825\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mVariable\u001b[49m\u001b[43m.\u001b[49m\u001b[43m_execution_engine\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrun_backward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[32m 826\u001b[39m \u001b[43m \u001b[49m\u001b[43mt_outputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\n\u001b[32m 827\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# Calls into the C++ engine to run the backward pass\u001b[39;00m\n\u001b[32m 828\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m 829\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m attach_logging_hooks:\n",
|
||||
"\u001b[31mKeyboardInterrupt\u001b[39m: "
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(\"=\" * 80)\n",
|
||||
"print(\" DP-SGD PRIVACY MITIGATION DEMONSTRATION\")\n",
|
||||
"print(\"=\" * 80)\n",
|
||||
"print(f\"\\nDevice: {device}\")\n",
|
||||
"print(f\"Random seed: {RANDOM_SEED}\")\n",
|
||||
"\n",
|
||||
"print(\"\\nLoading CIFAR-10 dataset...\")\n",
|
||||
"train_dataset, test_dataset, train_loader, test_loader = get_cifar10_loaders(batch_size=BATCH_SIZE, download=True)\n",
|
||||
"\n",
|
||||
"print(f\"Training samples: {len(train_dataset):,}\")\n",
|
||||
"print(f\"Test samples: {len(test_dataset):,}\")\n",
|
||||
"print(f\"Batch size: {BATCH_SIZE}\")\n",
|
||||
"\n",
|
||||
"print(\"\\n\" + \"=\" * 80)\n",
|
||||
"print(\" TRAINING: BASELINE MODEL (No Privacy Protection)\")\n",
|
||||
"print(\"=\" * 80)\n",
|
||||
"\n",
|
||||
"baseline_model = CIFAR10CNN().to(device)\n",
|
||||
"baseline_model = train_baseline_sgd(baseline_model, train_loader, device, epochs=BASELINE_EPOCHS, learning_rate=BASELINE_LR)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "5a235f8e-0bef-495b-8c98-5ec97dbe545c",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"Baseline Model Performance:\n",
|
||||
" Training accuracy: 10.00%\n",
|
||||
" Test accuracy: 10.00%\n",
|
||||
" Overfitting gap: 0.00%\n",
|
||||
"\n",
|
||||
"Saved baseline model to output/baseline_model.pth\n",
|
||||
"\n",
|
||||
"================================================================================\n",
|
||||
" MEMBERSHIP INFERENCE MEASUREMENT: Baseline Model\n",
|
||||
"================================================================================\n",
|
||||
"\n",
|
||||
"MIA Results (Baseline):\n",
|
||||
" Attack accuracy: 0.5000\n",
|
||||
" Attack advantage: 0.0000\n",
|
||||
" Random baseline: 0.5000\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"train_acc_baseline = evaluate_accuracy(baseline_model, train_loader, device)\n",
|
||||
"test_acc_baseline = evaluate_accuracy(baseline_model, test_loader, device)\n",
|
||||
"\n",
|
||||
"print(\"\\nBaseline Model Performance:\")\n",
|
||||
"print(f\" Training accuracy: {train_acc_baseline:.2f}%\")\n",
|
||||
"print(f\" Test accuracy: {test_acc_baseline:.2f}%\")\n",
|
||||
"print(f\" Overfitting gap: {train_acc_baseline - test_acc_baseline:.2f}%\")\n",
|
||||
"\n",
|
||||
"torch.save(baseline_model.state_dict(), \"output/baseline_model.pth\")\n",
|
||||
"print(\"\\nSaved baseline model to output/baseline_model.pth\")\n",
|
||||
"\n",
|
||||
"print(\"\\n\" + \"=\" * 80)\n",
|
||||
"print(\" MEMBERSHIP INFERENCE MEASUREMENT: Baseline Model\")\n",
|
||||
"print(\"=\" * 80)\n",
|
||||
"\n",
|
||||
"mia_acc_baseline, mia_adv_baseline = compute_mia_advantage(\n",
|
||||
" baseline_model, train_loader, test_loader, device\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(\"\\nMIA Results (Baseline):\")\n",
|
||||
"print(f\" Attack accuracy: {mia_acc_baseline:.4f}\")\n",
|
||||
"print(f\" Attack advantage: {mia_adv_baseline:.4f}\")\n",
|
||||
"print(\" Random baseline: 0.5000\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "589e1168-8c86-405b-bbbf-5fbcdce4851a",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"================================================================================\n",
|
||||
" TRAINING: DP-SGD MODEL (Target ε=10)\n",
|
||||
"================================================================================\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/jeremy/.conda/envs/ai/lib/python3.11/site-packages/opacus/privacy_engine.py:96: UserWarning: Secure RNG turned off. This is perfectly fine for experimentation as it allows for much faster training performance, but remember to turn it on and retrain one last time before production with ``secure_mode`` turned on.\n",
|
||||
" warnings.warn(\n",
|
||||
"/home/jeremy/.conda/envs/ai/lib/python3.11/site-packages/opacus/accountants/analysis/rdp.py:332: UserWarning: Optimal order is the largest alpha. Please consider expanding the range of alphas to get a tighter privacy bound.\n",
|
||||
" warnings.warn(\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"Configuration:\n",
|
||||
" Target epsilon: 10.0\n",
|
||||
" Delta: 1e-05\n",
|
||||
" Max gradient norm: 1.0\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from opacus import PrivacyEngine\n",
|
||||
"from opacus.validators import ModuleValidator\n",
|
||||
"\n",
|
||||
"print(\"\\n\" + \"=\" * 80)\n",
|
||||
"print(\" TRAINING: DP-SGD MODEL (Target ε=10)\")\n",
|
||||
"print(\"=\" * 80)\n",
|
||||
"\n",
|
||||
"TARGET_EPSILON_10 = 10.0\n",
|
||||
"\n",
|
||||
"_, _, train_loader_dp, test_loader_dp = get_cifar10_loaders(batch_size=BATCH_SIZE, download=False)\n",
|
||||
"\n",
|
||||
"dp_model_10 = CIFAR10CNN().to(device)\n",
|
||||
"dp_model_10 = ModuleValidator.fix(dp_model_10)\n",
|
||||
"optimizer_dp = optim.SGD(dp_model_10.parameters(), lr=DP_LR, momentum=0.9)\n",
|
||||
"\n",
|
||||
"privacy_engine = PrivacyEngine(accountant=\"rdp\")\n",
|
||||
"dp_model_10, optimizer_dp, train_loader_dp = privacy_engine.make_private_with_epsilon(\n",
|
||||
" module=dp_model_10,\n",
|
||||
" optimizer=optimizer_dp,\n",
|
||||
" data_loader=train_loader_dp,\n",
|
||||
" target_epsilon=TARGET_EPSILON_10,\n",
|
||||
" target_delta=DELTA,\n",
|
||||
" epochs=DP_EPOCHS,\n",
|
||||
" max_grad_norm=MAX_GRAD_NORM,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(f\"\\nConfiguration:\")\n",
|
||||
"print(f\" Target epsilon: {TARGET_EPSILON_10}\")\n",
|
||||
"print(f\" Delta: {DELTA}\")\n",
|
||||
"print(f\" Max gradient norm: {MAX_GRAD_NORM}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "23b1fc5c-ce8f-4b2c-a24d-02c11b69ede2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Reference in New Issue
Block a user