jjanella/AI-Red-Teaming-CSCD94
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added material

Browse Source
This commit is contained in:
Jeremy Janella
2026-05-09 23:21:13 -04:00
commit 75faa5c410
369 changed files with 3468754 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
applications-of-ai-in-infosec/Log dataset.ipynb
+1263
View File
File diff suppressed because it is too large Load Diff
applications-of-ai-in-infosec/Network Anomaly Detection/KDD+.txt
+148517
View File
File diff suppressed because it is too large Load Diff
applications-of-ai-in-infosec/Network Anomaly Detection/Network Anomoly Detector.ipynb
+253
View File
@@ -0,0 +1,253 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "2e0973be-fd6d-4467-b006-f30e455eae87",
"metadata": {},
"source": [
"# Download & load the dataset"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "17023a3e-0ed8-4b1b-a7c0-b63268c8ca33",
"metadata": {},
"outputs": [],
"source": [
"import requests, zipfile, io\n",
"\n",
"# URL for the NSL-KDD dataset\n",
"url = \"https://academy.hackthebox.com/storage/modules/292/KDD_dataset.zip\"\n",
"\n",
"# Download the zip file and extract its contents\n",
"response = requests.get(url)\n",
"z = zipfile.ZipFile(io.BytesIO(response.content))\n",
"z.extractall('.') # Extracts to the current directory\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "ae73c6f7-aa06-4383-bfcb-bf33af11a36e",
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"from sklearn.ensemble import RandomForestClassifier\n",
"from sklearn.model_selection import train_test_split\n",
"from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix, classification_report\n",
"import seaborn as sns\n",
"import matplotlib.pyplot as plt"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "4a38157b-ab82-4739-b1b8-30646064f943",
"metadata": {},
"outputs": [],
"source": [
"# Set the file path to the dataset\n",
"file_path = r'KDD+.txt'\n",
"\n",
"# Define the column names corresponding to the NSL-KDD dataset\n",
"columns = [\n",
" 'duration', 'protocol_type', 'service', 'flag', 'src_bytes', 'dst_bytes', \n",
" 'land', 'wrong_fragment', 'urgent', 'hot', 'num_failed_logins', 'logged_in', \n",
" 'num_compromised', 'root_shell', 'su_attempted', 'num_root', 'num_file_creations', \n",
" 'num_shells', 'num_access_files', 'num_outbound_cmds', 'is_host_login', 'is_guest_login', \n",
" 'count', 'srv_count', 'serror_rate', 'srv_serror_rate', 'rerror_rate', 'srv_rerror_rate', \n",
" 'same_srv_rate', 'diff_srv_rate', 'srv_diff_host_rate', 'dst_host_count', 'dst_host_srv_count', \n",
" 'dst_host_same_srv_rate', 'dst_host_diff_srv_rate', 'dst_host_same_src_port_rate', \n",
" 'dst_host_srv_diff_host_rate', 'dst_host_serror_rate', 'dst_host_srv_serror_rate', \n",
" 'dst_host_rerror_rate', 'dst_host_srv_rerror_rate', 'attack', 'level'\n",
"]"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "d4d7b05e-894d-4f54-886a-f3469f473675",
"metadata": {},
"outputs": [],
"source": [
"df = pd.read_csv(file_path, names=columns)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "fe6882f3-1e5c-4195-88a4-4d58d2ac0d37",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" duration protocol_type service flag src_bytes dst_bytes land \\\n",
"0 0 tcp ftp_data SF 491 0 0 \n",
"1 0 udp other SF 146 0 0 \n",
"2 0 tcp private S0 0 0 0 \n",
"3 0 tcp http SF 232 8153 0 \n",
"4 0 tcp http SF 199 420 0 \n",
"\n",
" wrong_fragment urgent hot ... dst_host_same_srv_rate \\\n",
"0 0 0 0 ... 0.17 \n",
"1 0 0 0 ... 0.00 \n",
"2 0 0 0 ... 0.10 \n",
"3 0 0 0 ... 1.00 \n",
"4 0 0 0 ... 1.00 \n",
"\n",
" dst_host_diff_srv_rate dst_host_same_src_port_rate \\\n",
"0 0.03 0.17 \n",
"1 0.60 0.88 \n",
"2 0.05 0.00 \n",
"3 0.00 0.03 \n",
"4 0.00 0.00 \n",
"\n",
" dst_host_srv_diff_host_rate dst_host_serror_rate \\\n",
"0 0.00 0.00 \n",
"1 0.00 0.00 \n",
"2 0.00 1.00 \n",
"3 0.04 0.03 \n",
"4 0.00 0.00 \n",
"\n",
" dst_host_srv_serror_rate dst_host_rerror_rate dst_host_srv_rerror_rate \\\n",
"0 0.00 0.05 0.00 \n",
"1 0.00 0.00 0.00 \n",
"2 1.00 0.00 0.00 \n",
"3 0.01 0.00 0.01 \n",
"4 0.00 0.00 0.00 \n",
"\n",
" attack level \n",
"0 normal 20 \n",
"1 normal 15 \n",
"2 neptune 19 \n",
"3 normal 21 \n",
"4 normal 21 \n",
"\n",
"[5 rows x 43 columns]\n"
]
}
],
"source": [
"print(df.head())"
]
},
{
"cell_type": "markdown",
"id": "44b7cf45-5cfe-4a7d-9e15-b59cc369028e",
"metadata": {},
"source": [
"# Preprocessing and Splitting the Dataset"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "084802d8-ff3e-4d34-89f7-3d2e238c413a",
"metadata": {},
"outputs": [],
"source": [
"# Binary classification target\n",
"# Maps normal traffic to 0 and any type of attack to 1\n",
"df['attack_flag'] = df['attack'].apply(lambda a: 0 if a == 'normal' else 1)"
]
},
{
"cell_type": "markdown",
"id": "c275f531-ac9f-4661-a07c-cc07efc354f4",
"metadata": {},
"source": [
"# Multi-class classification target"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "1b18dd99-adb4-44ca-8b86-4807428eea87",
"metadata": {},
"outputs": [],
"source": [
"# Multi-class classification target categories\n",
"dos_attacks = ['apache2', 'back', 'land', 'neptune', 'mailbomb', 'pod', \n",
" 'processtable', 'smurf', 'teardrop', 'udpstorm', 'worm']\n",
"probe_attacks = ['ipsweep', 'mscan', 'nmap', 'portsweep', 'saint', 'satan']\n",
"privilege_attacks = ['buffer_overflow', 'loadmdoule', 'perl', 'ps', \n",
" 'rootkit', 'sqlattack', 'xterm']\n",
"access_attacks = ['ftp_write', 'guess_passwd', 'http_tunnel', 'imap', \n",
" 'multihop', 'named', 'phf', 'sendmail', 'snmpgetattack', \n",
" 'snmpguess', 'spy', 'warezclient', 'warezmaster', \n",
" 'xclock', 'xsnoop']\n",
"\n",
"def map_attack(attack):\n",
" if attack in dos_attacks:\n",
" return 1\n",
" elif attack in probe_attacks:\n",
" return 2\n",
" elif attack in privilege_attacks:\n",
" return 3\n",
" elif attack in access_attacks:\n",
" return 4\n",
" else:\n",
" return 0\n",
"\n",
"# Assign multi-class category to each row\n",
"df['attack_map'] = df['attack'].apply(map_attack)\n"
]
},
{
"cell_type": "markdown",
"id": "be5f32df-4218-416c-832f-e190f97a8e47",
"metadata": {},
"source": [
"# Encoding Categorical Variables"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "777d9ee2-40bb-400e-9264-47e67b7834c0",
"metadata": {},
"outputs": [],
"source": [
"# Numeric features that capture various statistical properties of the traffic\n",
"numeric_features = [\n",
" 'duration', 'src_bytes', 'dst_bytes', 'wrong_fragment', 'urgent', 'hot', \n",
" 'num_failed_logins', 'num_compromised', 'root_shell', 'su_attempted', \n",
" 'num_root', 'num_file_creations', 'num_shells', 'num_access_files', \n",
" 'num_outbound_cmds', 'count', 'srv_count', 'serror_rate', \n",
" 'srv_serror_rate', 'rerror_rate', 'srv_rerror_rate', 'same_srv_rate', \n",
" 'diff_srv_rate', 'srv_diff_host_rate', 'dst_host_count', 'dst_host_srv_count', \n",
" 'dst_host_same_srv_rate', 'dst_host_diff_srv_rate', \n",
" 'dst_host_same_src_port_rate', 'dst_host_srv_diff_host_rate', \n",
" 'dst_host_serror_rate', 'dst_host_srv_serror_rate', 'dst_host_rerror_rate', \n",
" 'dst_host_srv_rerror_rate'\n",
"]\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
applications-of-ai-in-infosec/Spam Classification/Spam Classification.ipynb
+902
View File
@@ -0,0 +1,902 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "738bf5f9-3757-4b17-a799-7ba335c4b269",
"metadata": {},
"source": [
"# Create the dataset"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "e9a6abb6-aeba-4b0c-bbfb-5f92a4f18e21",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Download successful\n"
]
}
],
"source": [
"import requests\n",
"import zipfile\n",
"import io\n",
"\n",
"# URL of the dataset\n",
"url = \"https://archive.ics.uci.edu/static/public/228/sms+spam+collection.zip\"\n",
"# Download the dataset\n",
"response = requests.get(url)\n",
"if response.status_code == 200:\n",
" print(\"Download successful\")\n",
"else:\n",
" print(\"Failed to download the dataset\")"
]
},
{
"cell_type": "markdown",
"id": "7e432811-930f-4fe5-913d-baf25f0dc1cf",
"metadata": {},
"source": [
"# Extract"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "0213809b-9b15-4aaa-8b23-54fcf0ce47a5",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Extraction successful\n"
]
}
],
"source": [
"# Extract the dataset\n",
"with zipfile.ZipFile(io.BytesIO(response.content)) as z:\n",
" z.extractall(\"sms_spam_collection\")\n",
" print(\"Extraction successful\")\n"
]
},
{
"cell_type": "markdown",
"id": "15103f06-8dcf-466b-8a2a-9d8e5db68bda",
"metadata": {},
"source": [
"# View files"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "db9de74d-fb53-4518-9765-61babfa84001",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Extracted files: ['SMSSpamCollection', 'readme']\n"
]
}
],
"source": [
"import os\n",
"\n",
"# List the extracted files\n",
"extracted_files = os.listdir(\"sms_spam_collection\")\n",
"print(\"Extracted files:\", extracted_files)\n"
]
},
{
"cell_type": "markdown",
"id": "19dc4d0e-407e-478a-813b-e0fe466a4912",
"metadata": {},
"source": [
"# Load Dataset"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "2fcb7819-c5ae-4932-a04d-3f7e89c23840",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"\n",
"# Load the dataset\n",
"df = pd.read_csv(\n",
" \"sms_spam_collection/SMSSpamCollection\",\n",
" sep=\"\\t\",\n",
" header=None,\n",
" names=[\"label\", \"message\"],\n",
")\n"
]
},
{
"cell_type": "markdown",
"id": "a8d0056c-f7ba-416a-b973-abbfe9c9112e",
"metadata": {},
"source": [
"# Display Dataset"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "74516d20-fc54-4555-ad45-1c54061f3721",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"-------------------- HEAD --------------------\n",
" label message\n",
"0 ham Go until jurong point, crazy.. Available only ...\n",
"1 ham Ok lar... Joking wif u oni...\n",
"2 spam Free entry in 2 a wkly comp to win FA Cup fina...\n",
"3 ham U dun say so early hor... U c already then say...\n",
"4 ham Nah I don't think he goes to usf, he lives aro...\n",
"-------------------- DESCRIBE --------------------\n",
" label message\n",
"count 5572 5572\n",
"unique 2 5169\n",
"top ham Sorry, I'll call later\n",
"freq 4825 30\n",
"-------------------- INFO --------------------\n",
"<class 'pandas.core.frame.DataFrame'>\n",
"RangeIndex: 5572 entries, 0 to 5571\n",
"Data columns (total 2 columns):\n",
" # Column Non-Null Count Dtype \n",
"--- ------ -------------- ----- \n",
" 0 label 5572 non-null object\n",
" 1 message 5572 non-null object\n",
"dtypes: object(2)\n",
"memory usage: 87.2+ KB\n",
"None\n"
]
}
],
"source": [
"# Display basic information about the dataset\n",
"print(\"-------------------- HEAD --------------------\")\n",
"print(df.head())\n",
"print(\"-------------------- DESCRIBE --------------------\")\n",
"print(df.describe())\n",
"print(\"-------------------- INFO --------------------\")\n",
"print(df.info())\n"
]
},
{
"cell_type": "markdown",
"id": "8ad80cb6-0aaa-4468-b17f-057db8a5d984",
"metadata": {},
"source": [
"# Find missing Values"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "d097fca1-daf8-418d-90d3-62f8831c9e86",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Missing values:\n",
" label 0\n",
"message 0\n",
"dtype: int64\n"
]
}
],
"source": [
"# Check for missing values\n",
"print(\"Missing values:\\n\", df.isnull().sum())\n"
]
},
{
"cell_type": "markdown",
"id": "fe212f08-ad18-4358-a89a-4247f3322163",
"metadata": {},
"source": [
"# Find duplicate values"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "c469894c-8e88-4893-90de-180842bbc084",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Duplicate entries: 403\n"
]
}
],
"source": [
"# Check for duplicates\n",
"print(\"Duplicate entries:\", df.duplicated().sum())\n",
"\n",
"# Remove duplicates if any\n",
"df = df.drop_duplicates()\n"
]
},
{
"cell_type": "markdown",
"id": "2494b426-1ce5-4f6e-92a8-9e7ffef7a651",
"metadata": {},
"source": [
"# Preprocessing the Spam Dataset\n",
"nltk tokenizes and cleans data\n",
"- lowercases data\n",
"- removes punctuation\n",
"- tokenizes text\n",
"- removes stopwords\n",
"- does stemming"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "831d49a0-0d39-44a3-894b-f9beebe82e06",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"[nltk_data] Downloading package punkt to /home/jeremy/nltk_data...\n",
"[nltk_data] Unzipping tokenizers/punkt.zip.\n",
"[nltk_data] Downloading package punkt_tab to /home/jeremy/nltk_data...\n",
"[nltk_data] Unzipping tokenizers/punkt_tab.zip.\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"=== BEFORE ANY PREPROCESSING ===\n",
" label message\n",
"0 ham Go until jurong point, crazy.. Available only ...\n",
"1 ham Ok lar... Joking wif u oni...\n",
"2 spam Free entry in 2 a wkly comp to win FA Cup fina...\n",
"3 ham U dun say so early hor... U c already then say...\n",
"4 ham Nah I don't think he goes to usf, he lives aro...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[nltk_data] Downloading package stopwords to /home/jeremy/nltk_data...\n",
"[nltk_data] Unzipping corpora/stopwords.zip.\n"
]
}
],
"source": [
"import nltk\n",
"\n",
"# Download the necessary NLTK data files\n",
"nltk.download(\"punkt\")\n",
"nltk.download(\"punkt_tab\")\n",
"nltk.download(\"stopwords\")\n",
"\n",
"print(\"=== BEFORE ANY PREPROCESSING ===\") \n",
"print(df.head(5))\n"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "0e149fe9-a3e8-42f9-8962-eb0fa159e983",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"=== AFTER LOWERCASING ===\n",
"0 go until jurong point, crazy.. available only ...\n",
"1 ok lar... joking wif u oni...\n",
"2 free entry in 2 a wkly comp to win fa cup fina...\n",
"3 u dun say so early hor... u c already then say...\n",
"4 nah i don't think he goes to usf, he lives aro...\n",
"Name: message, dtype: object\n"
]
}
],
"source": [
"# Convert all message text to lowercase\n",
"df[\"message\"] = df[\"message\"].str.lower()\n",
"print(\"\\n=== AFTER LOWERCASING ===\")\n",
"print(df[\"message\"].head(5))\n"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "883576e9-52fd-4b1f-aadb-b8f75ec4ac0d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"=== AFTER REMOVING PUNCTUATION & NUMBERS (except $ and !) ===\n",
"0 go until jurong point crazy available only in ...\n",
"1 ok lar joking wif u oni\n",
"2 free entry in a wkly comp to win fa cup final...\n",
"3 u dun say so early hor u c already then say\n",
"4 nah i dont think he goes to usf he lives aroun...\n",
"Name: message, dtype: object\n"
]
}
],
"source": [
"import re\n",
"\n",
"# Remove non-essential punctuation and numbers, keep useful symbols like $ and !\n",
"df[\"message\"] = df[\"message\"].apply(lambda x: re.sub(r\"[^a-z\\s$!]\", \"\", x))\n",
"print(\"\\n=== AFTER REMOVING PUNCTUATION & NUMBERS (except $ and !) ===\")\n",
"print(df[\"message\"].head(5))\n"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "ea8b673e-f1ed-499d-84c8-050144332a75",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"=== AFTER TOKENIZATION ===\n",
"0 [go, until, jurong, point, crazy, available, o...\n",
"1 [ok, lar, joking, wif, u, oni]\n",
"2 [free, entry, in, a, wkly, comp, to, win, fa, ...\n",
"3 [u, dun, say, so, early, hor, u, c, already, t...\n",
"4 [nah, i, dont, think, he, goes, to, usf, he, l...\n",
"Name: message, dtype: object\n"
]
}
],
"source": [
"from nltk.tokenize import word_tokenize\n",
"\n",
"# Split each message into individual tokens\n",
"df[\"message\"] = df[\"message\"].apply(word_tokenize)\n",
"print(\"\\n=== AFTER TOKENIZATION ===\")\n",
"print(df[\"message\"].head(5))\n"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "b6efb48a-f61a-4613-86b9-9ca8b38cc144",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"=== AFTER REMOVING STOP WORDS ===\n",
"0 [go, jurong, point, crazy, available, bugis, n...\n",
"1 [ok, lar, joking, wif, u, oni]\n",
"2 [free, entry, wkly, comp, win, fa, cup, final,...\n",
"3 [u, dun, say, early, hor, u, c, already, say]\n",
"4 [nah, dont, think, goes, usf, lives, around, t...\n",
"Name: message, dtype: object\n"
]
}
],
"source": [
"from nltk.corpus import stopwords\n",
"\n",
"# Define a set of English stop words and remove them from the tokens\n",
"stop_words = set(stopwords.words(\"english\"))\n",
"df[\"message\"] = df[\"message\"].apply(lambda x: [word for word in x if word not in stop_words])\n",
"print(\"\\n=== AFTER REMOVING STOP WORDS ===\")\n",
"print(df[\"message\"].head(5))\n"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "e556ed99-3b3e-4b77-9451-11500de66153",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"=== AFTER STEMMING ===\n",
"0 [go, jurong, point, crazi, avail, bugi, n, gre...\n",
"1 [ok, lar, joke, wif, u, oni]\n",
"2 [free, entri, wkli, comp, win, fa, cup, final,...\n",
"3 [u, dun, say, earli, hor, u, c, alreadi, say]\n",
"4 [nah, dont, think, goe, usf, live, around, tho...\n",
"Name: message, dtype: object\n"
]
}
],
"source": [
"from nltk.stem import PorterStemmer\n",
"\n",
"# Stem each token to reduce words to their base form\n",
"stemmer = PorterStemmer()\n",
"df[\"message\"] = df[\"message\"].apply(lambda x: [stemmer.stem(word) for word in x])\n",
"print(\"\\n=== AFTER STEMMING ===\")\n",
"print(df[\"message\"].head(5))\n"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "f1c4ecf0-79a7-4f58-b73a-b17c304ee15f",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"=== AFTER JOINING TOKENS BACK INTO STRINGS ===\n",
"0 go jurong point crazi avail bugi n great world...\n",
"1 ok lar joke wif u oni\n",
"2 free entri wkli comp win fa cup final tkt st m...\n",
"3 u dun say earli hor u c alreadi say\n",
"4 nah dont think goe usf live around though\n",
"Name: message, dtype: object\n"
]
}
],
"source": [
"# Rejoin tokens into a single string for feature extraction\n",
"df[\"message\"] = df[\"message\"].apply(lambda x: \" \".join(x))\n",
"print(\"\\n=== AFTER JOINING TOKENS BACK INTO STRINGS ===\")\n",
"print(df[\"message\"].head(5))\n"
]
},
{
"cell_type": "markdown",
"id": "417c3d1b-3c15-416a-a808-50c266d9bd8f",
"metadata": {},
"source": [
"# CountVectorizer Bag of Words vectorization\n",
"- Minimum occurance: 1\n",
"- Maxiumum appearce: 90%\n",
"- ngram range: 1-2\n",
"- Vectorizes (into vectors :0)\n",
"\n",
"ngrams are counted that fit this description, into vector"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "83210ebb-fbf2-42b4-b2b6-9b6fdcd2a39c",
"metadata": {},
"outputs": [],
"source": [
"from sklearn.feature_extraction.text import CountVectorizer\n",
"\n",
"# Initialize CountVectorizer with bigrams, min_df, and max_df to focus on relevant terms\n",
"vectorizer = CountVectorizer(min_df=1, max_df=0.9, ngram_range=(1, 2))\n",
"\n",
"# Fit and transform the message column\n",
"X = vectorizer.fit_transform(df[\"message\"])\n",
"\n",
"# Labels (target variable)\n",
"y = df[\"label\"].apply(lambda x: 1 if x == \"spam\" else 0) # Converting labels to 1 and 0\n"
]
},
{
"cell_type": "markdown",
"id": "380eae4b-e569-4bc9-b9a2-aee0214a76dd",
"metadata": {},
"source": [
"# Train the model\n",
"\n",
"Multinomial naive Bayes to find probabilities of spam and feature relationships based on the input vectors\n",
"\n",
"Pipeline datastructure chains together vectorization and modeling steps"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "6205bdc0-21f8-4d37-b682-c5d41a43610f",
"metadata": {},
"outputs": [],
"source": [
"from sklearn.model_selection import train_test_split, GridSearchCV\n",
"from sklearn.naive_bayes import MultinomialNB\n",
"from sklearn.pipeline import Pipeline\n",
"\n",
"# Build the pipeline by combining vectorization and classification\n",
"pipeline = Pipeline([\n",
" (\"vectorizer\", vectorizer),\n",
" (\"classifier\", MultinomialNB())\n",
"])"
]
},
{
"cell_type": "markdown",
"id": "ed9b3536-d2cd-4d66-8a4c-71c25f37711f",
"metadata": {},
"source": [
"GridSearchCV systematically searches through specified hyperparameter values to identify the configuration that produces the best performance\n",
"\n",
"MultinomialNB focuses on the alpha parameter, a smoothing factor that adjusts how the model handles unseen words and prevents probabilities from being zero\n",
"\n",
"The configuration is chosen by maximizing the f1 score"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "2399098c-171f-400f-af2a-d47dc4af5dea",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Best model parameters: {'classifier__alpha': 0.25}\n"
]
}
],
"source": [
"# Define the parameter grid for hyperparameter tuning\n",
"param_grid = {\n",
" \"classifier__alpha\": [0.01, 0.1, 0.15, 0.2, 0.25, 0.5, 0.75, 1.0]\n",
"}\n",
"\n",
"# Perform the grid search with 5-fold cross-validation and the F1-score as metric\n",
"grid_search = GridSearchCV(\n",
" pipeline,\n",
" param_grid,\n",
" cv=5,\n",
" scoring=\"f1\"\n",
")\n",
"\n",
"# Fit the grid search on the full dataset\n",
"grid_search.fit(df[\"message\"], y)\n",
"\n",
"# Extract the best model identified by the grid search\n",
"best_model = grid_search.best_estimator_\n",
"print(\"Best model parameters:\", grid_search.best_params_)\n"
]
},
{
"cell_type": "markdown",
"id": "ef8dfd83-8f29-4089-b096-580e4e8bb778",
"metadata": {},
"source": [
"# Testing the model\n",
"- create a new message\n",
"- preprocess the message"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "fc0b16c6-4197-4554-86c2-54ae781ae4a8",
"metadata": {},
"outputs": [],
"source": [
"# Example SMS messages for evaluation\n",
"new_messages = [\n",
" \"Congratulations! You've won a $1000 Walmart gift card. Go to http://bit.ly/1234 to claim now.\",\n",
" \"Hey, are we still meeting up for lunch today?\",\n",
" \"Urgent! Your account has been compromised. Verify your details here: www.fakebank.com/verify\",\n",
" \"Reminder: Your appointment is scheduled for tomorrow at 10am.\",\n",
" \"FREE entry in a weekly competition to win an iPad. Just text WIN to 80085 now!\",\n",
"]\n"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "3f720aa9-1ea7-4eb2-8afa-20895fb1f18f",
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import re\n",
"\n",
"# Preprocess function that mirrors the training-time preprocessing\n",
"def preprocess_message(message):\n",
" message = message.lower()\n",
" message = re.sub(r\"[^a-z\\s$!]\", \"\", message)\n",
" tokens = word_tokenize(message)\n",
" tokens = [word for word in tokens if word not in stop_words]\n",
" tokens = [stemmer.stem(word) for word in tokens]\n",
" return \" \".join(tokens)\n"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "f81b6d54-3217-43f4-b0f7-174669631cef",
"metadata": {},
"outputs": [],
"source": [
"# Preprocess and vectorize messages\n",
"processed_messages = [preprocess_message(msg) for msg in new_messages]\n"
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "2c2f8a99-274f-44fc-b66c-bf4e5f8d3985",
"metadata": {},
"outputs": [],
"source": [
"# Transform preprocessed messages into feature vectors\n",
"X_new = best_model.named_steps[\"vectorizer\"].transform(processed_messages)\n"
]
},
{
"cell_type": "markdown",
"id": "d6f0d757-ee92-4985-b96c-01d1afbc7a62",
"metadata": {},
"source": [
"# Mak predictions\n",
"- Feed the model the new message\n",
"- View the output"
]
},
{
"cell_type": "code",
"execution_count": 28,
"id": "d900f2af-f17a-417b-b6b1-092f4125b4ef",
"metadata": {},
"outputs": [],
"source": [
"# Transform preprocessed messages into feature vectors\n",
"X_new = best_model.named_steps[\"vectorizer\"].transform(processed_messages)\n"
]
},
{
"cell_type": "code",
"execution_count": 29,
"id": "7c5bc013-4695-47f1-a12c-08abc33aa54d",
"metadata": {},
"outputs": [],
"source": [
"# Predict with the trained classifier\n",
"predictions = best_model.named_steps[\"classifier\"].predict(X_new)\n",
"prediction_probabilities = best_model.named_steps[\"classifier\"].predict_proba(X_new)\n"
]
},
{
"cell_type": "code",
"execution_count": 30,
"id": "79186e20-428d-4d92-8eeb-bb932788ef29",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Message: Congratulations! You've won a $1000 Walmart gift card. Go to http://bit.ly/1234 to claim now.\n",
"Prediction: Spam\n",
"Spam Probability: 1.00\n",
"Not-Spam Probability: 0.00\n",
"--------------------------------------------------\n",
"Message: Hey, are we still meeting up for lunch today?\n",
"Prediction: Not-Spam\n",
"Spam Probability: 0.00\n",
"Not-Spam Probability: 1.00\n",
"--------------------------------------------------\n",
"Message: Urgent! Your account has been compromised. Verify your details here: www.fakebank.com/verify\n",
"Prediction: Spam\n",
"Spam Probability: 0.96\n",
"Not-Spam Probability: 0.04\n",
"--------------------------------------------------\n",
"Message: Reminder: Your appointment is scheduled for tomorrow at 10am.\n",
"Prediction: Not-Spam\n",
"Spam Probability: 0.00\n",
"Not-Spam Probability: 1.00\n",
"--------------------------------------------------\n",
"Message: FREE entry in a weekly competition to win an iPad. Just text WIN to 80085 now!\n",
"Prediction: Spam\n",
"Spam Probability: 1.00\n",
"Not-Spam Probability: 0.00\n",
"--------------------------------------------------\n"
]
}
],
"source": [
"# Display predictions and probabilities for each evaluated message\n",
"for i, msg in enumerate(new_messages):\n",
" prediction = \"Spam\" if predictions[i] == 1 else \"Not-Spam\"\n",
" spam_probability = prediction_probabilities[i][1] # Probability of being spam\n",
" ham_probability = prediction_probabilities[i][0] # Probability of being not spam\n",
" \n",
" print(f\"Message: {msg}\")\n",
" print(f\"Prediction: {prediction}\")\n",
" print(f\"Spam Probability: {spam_probability:.2f}\")\n",
" print(f\"Not-Spam Probability: {ham_probability:.2f}\")\n",
" print(\"-\" * 50)\n"
]
},
{
"cell_type": "markdown",
"id": "8c42c4be-d64e-4533-959c-b3b14e57b6b8",
"metadata": {},
"source": [
"# Saving the model with joblib"
]
},
{
"cell_type": "code",
"execution_count": 31,
"id": "74bf38e4-020d-400c-b581-b34fafb0737d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Model saved to spam_detection_model.joblib\n"
]
}
],
"source": [
"import joblib\n",
"\n",
"# Save the trained model to a file for future use\n",
"model_filename = 'spam_detection_model.joblib'\n",
"joblib.dump(best_model, model_filename)\n",
"\n",
"print(f\"Model saved to {model_filename}\")\n"
]
},
{
"cell_type": "code",
"execution_count": 32,
"id": "358bc489-7758-4ac9-8b2e-cbb8888dfb53",
"metadata": {},
"outputs": [],
"source": [
"# Load the saved model\n",
"loaded_model = joblib.load(model_filename)\n",
"\n",
"# Preprocess new messages before prediction\n",
"new_data_processed = [preprocess_message(msg) for msg in new_messages]\n",
"\n",
"# Make predictions on the preprocessed data\n",
"predictions = loaded_model.predict(new_data_processed)\n"
]
},
{
"cell_type": "markdown",
"id": "dad19530-da3c-4dbd-80de-8a5d840db7f2",
"metadata": {},
"source": [
"# HTB test the model\n",
"first run\n",
"\n",
"`sudo nix-shell -p openvpn --run \"openvpn --config /tmp/academy-regular.ovpn\"`"
]
},
{
"cell_type": "code",
"execution_count": 34,
"id": "b7be2e9a-d731-4351-a130-d7deb19af6c3",
"metadata": {},
"outputs": [
{
"ename": "ConnectionError",
"evalue": "HTTPConnectionPool(host='localhost', port=8000): Max retries exceeded with url: /api/upload (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fcadf6b8c10>: Failed to establish a new connection: [Errno 111] Connection refused'))",
"output_type": "error",
"traceback": [
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
"\u001b[31mConnectionRefusedError\u001b[39m Traceback (most recent call last)",
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/connection.py:198\u001b[39m, in \u001b[36mHTTPConnection._new_conn\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 197\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m198\u001b[39m sock = \u001b[43mconnection\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcreate_connection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 199\u001b[39m \u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_dns_host\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mport\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 200\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 201\u001b[39m \u001b[43m \u001b[49m\u001b[43msource_address\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msource_address\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 202\u001b[39m \u001b[43m \u001b[49m\u001b[43msocket_options\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msocket_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 203\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 204\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m socket.gaierror \u001b[38;5;28;01mas\u001b[39;00m e:\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/util/connection.py:85\u001b[39m, in \u001b[36mcreate_connection\u001b[39m\u001b[34m(address, timeout, source_address, socket_options)\u001b[39m\n\u001b[32m 84\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m85\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[32m 86\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m 87\u001b[39m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/util/connection.py:73\u001b[39m, in \u001b[36mcreate_connection\u001b[39m\u001b[34m(address, timeout, source_address, socket_options)\u001b[39m\n\u001b[32m 72\u001b[39m sock.bind(source_address)\n\u001b[32m---> \u001b[39m\u001b[32m73\u001b[39m \u001b[43msock\u001b[49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43msa\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 74\u001b[39m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n",
"\u001b[31mConnectionRefusedError\u001b[39m: [Errno 111] Connection refused",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[31mNewConnectionError\u001b[39m Traceback (most recent call last)",
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/connectionpool.py:787\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 786\u001b[39m \u001b[38;5;66;03m# Make the request on the HTTPConnection object\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m787\u001b[39m response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 788\u001b[39m \u001b[43m \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 789\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 790\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 791\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 792\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 793\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 794\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 795\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 796\u001b[39m \u001b[43m \u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 797\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 798\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 799\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 800\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 802\u001b[39m \u001b[38;5;66;03m# Everything went great!\u001b[39;00m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/connectionpool.py:493\u001b[39m, in \u001b[36mHTTPConnectionPool._make_request\u001b[39m\u001b[34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 492\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m493\u001b[39m \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 494\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 495\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 496\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 497\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 498\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 499\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 500\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 501\u001b[39m \u001b[43m \u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[43m=\u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 502\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 504\u001b[39m \u001b[38;5;66;03m# We are swallowing BrokenPipeError (errno.EPIPE) since the server is\u001b[39;00m\n\u001b[32m 505\u001b[39m \u001b[38;5;66;03m# legitimately able to close the connection after sending a valid response.\u001b[39;00m\n\u001b[32m 506\u001b[39m \u001b[38;5;66;03m# With this behaviour, the received response is still readable.\u001b[39;00m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/connection.py:494\u001b[39m, in \u001b[36mHTTPConnection.request\u001b[39m\u001b[34m(self, method, url, body, headers, chunked, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 493\u001b[39m \u001b[38;5;28mself\u001b[39m.putheader(header, value)\n\u001b[32m--> \u001b[39m\u001b[32m494\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mendheaders\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 496\u001b[39m \u001b[38;5;66;03m# If we're given a body we start sending that in chunks.\u001b[39;00m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/http/client.py:1281\u001b[39m, in \u001b[36mHTTPConnection.endheaders\u001b[39m\u001b[34m(self, message_body, encode_chunked)\u001b[39m\n\u001b[32m 1280\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m CannotSendHeader()\n\u001b[32m-> \u001b[39m\u001b[32m1281\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_send_output\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/http/client.py:1041\u001b[39m, in \u001b[36mHTTPConnection._send_output\u001b[39m\u001b[34m(self, message_body, encode_chunked)\u001b[39m\n\u001b[32m 1040\u001b[39m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m._buffer[:]\n\u001b[32m-> \u001b[39m\u001b[32m1041\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1043\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m message_body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 1044\u001b[39m \n\u001b[32m 1045\u001b[39m \u001b[38;5;66;03m# create a consistent interface to message_body\u001b[39;00m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/http/client.py:979\u001b[39m, in \u001b[36mHTTPConnection.send\u001b[39m\u001b[34m(self, data)\u001b[39m\n\u001b[32m 978\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.auto_open:\n\u001b[32m--> \u001b[39m\u001b[32m979\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 980\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/connection.py:325\u001b[39m, in \u001b[36mHTTPConnection.connect\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 324\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mconnect\u001b[39m(\u001b[38;5;28mself\u001b[39m) -> \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m325\u001b[39m \u001b[38;5;28mself\u001b[39m.sock = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_new_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 326\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._tunnel_host:\n\u001b[32m 327\u001b[39m \u001b[38;5;66;03m# If we're tunneling it means we're connected to our proxy.\u001b[39;00m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/connection.py:213\u001b[39m, in \u001b[36mHTTPConnection._new_conn\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 212\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m--> \u001b[39m\u001b[32m213\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m NewConnectionError(\n\u001b[32m 214\u001b[39m \u001b[38;5;28mself\u001b[39m, \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mFailed to establish a new connection: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m 215\u001b[39m ) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01me\u001b[39;00m\n\u001b[32m 217\u001b[39m sys.audit(\u001b[33m\"\u001b[39m\u001b[33mhttp.client.connect\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28mself\u001b[39m, \u001b[38;5;28mself\u001b[39m.host, \u001b[38;5;28mself\u001b[39m.port)\n",
"\u001b[31mNewConnectionError\u001b[39m: <urllib3.connection.HTTPConnection object at 0x7fcadf6b8c10>: Failed to establish a new connection: [Errno 111] Connection refused",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[31mMaxRetryError\u001b[39m Traceback (most recent call last)",
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/requests/adapters.py:644\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 643\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m644\u001b[39m resp = \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 645\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 646\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 647\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 648\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 649\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 650\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 651\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 652\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 653\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 654\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 655\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 656\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 658\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/connectionpool.py:841\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 839\u001b[39m new_e = ProtocolError(\u001b[33m\"\u001b[39m\u001b[33mConnection aborted.\u001b[39m\u001b[33m\"\u001b[39m, new_e)\n\u001b[32m--> \u001b[39m\u001b[32m841\u001b[39m retries = \u001b[43mretries\u001b[49m\u001b[43m.\u001b[49m\u001b[43mincrement\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 842\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnew_e\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_pool\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[43m=\u001b[49m\u001b[43msys\u001b[49m\u001b[43m.\u001b[49m\u001b[43mexc_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[32;43m2\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[32m 843\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 844\u001b[39m retries.sleep()\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/util/retry.py:519\u001b[39m, in \u001b[36mRetry.increment\u001b[39m\u001b[34m(self, method, url, response, error, _pool, _stacktrace)\u001b[39m\n\u001b[32m 518\u001b[39m reason = error \u001b[38;5;129;01mor\u001b[39;00m ResponseError(cause)\n\u001b[32m--> \u001b[39m\u001b[32m519\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m MaxRetryError(_pool, url, reason) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mreason\u001b[39;00m \u001b[38;5;66;03m# type: ignore[arg-type]\u001b[39;00m\n\u001b[32m 521\u001b[39m log.debug(\u001b[33m\"\u001b[39m\u001b[33mIncremented Retry for (url=\u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m): \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m\"\u001b[39m, url, new_retry)\n",
"\u001b[31mMaxRetryError\u001b[39m: HTTPConnectionPool(host='localhost', port=8000): Max retries exceeded with url: /api/upload (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fcadf6b8c10>: Failed to establish a new connection: [Errno 111] Connection refused'))",
"\nDuring handling of the above exception, another exception occurred:\n",
"\u001b[31mConnectionError\u001b[39m Traceback (most recent call last)",
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[34]\u001b[39m\u001b[32m, line 13\u001b[39m\n\u001b[32m 11\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(model_file_path, \u001b[33m\"\u001b[39m\u001b[33mrb\u001b[39m\u001b[33m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m model_file:\n\u001b[32m 12\u001b[39m files = {\u001b[33m\"\u001b[39m\u001b[33mmodel\u001b[39m\u001b[33m\"\u001b[39m: model_file}\n\u001b[32m---> \u001b[39m\u001b[32m13\u001b[39m response = \u001b[43mrequests\u001b[49m\u001b[43m.\u001b[49m\u001b[43mpost\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfiles\u001b[49m\u001b[43m=\u001b[49m\u001b[43mfiles\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 15\u001b[39m \u001b[38;5;66;03m# Pretty print the response from the server\u001b[39;00m\n\u001b[32m 16\u001b[39m \u001b[38;5;28mprint\u001b[39m(json.dumps(response.json(), indent=\u001b[32m4\u001b[39m))\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/requests/api.py:115\u001b[39m, in \u001b[36mpost\u001b[39m\u001b[34m(url, data, json, **kwargs)\u001b[39m\n\u001b[32m 103\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mpost\u001b[39m(url, data=\u001b[38;5;28;01mNone\u001b[39;00m, json=\u001b[38;5;28;01mNone\u001b[39;00m, **kwargs):\n\u001b[32m 104\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33mr\u001b[39m\u001b[33;03m\"\"\"Sends a POST request.\u001b[39;00m\n\u001b[32m 105\u001b[39m \n\u001b[32m 106\u001b[39m \u001b[33;03m :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 112\u001b[39m \u001b[33;03m :rtype: requests.Response\u001b[39;00m\n\u001b[32m 113\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m115\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mpost\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[43m=\u001b[49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/requests/api.py:59\u001b[39m, in \u001b[36mrequest\u001b[39m\u001b[34m(method, url, **kwargs)\u001b[39m\n\u001b[32m 55\u001b[39m \u001b[38;5;66;03m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[32m 56\u001b[39m \u001b[38;5;66;03m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[32m 57\u001b[39m \u001b[38;5;66;03m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[32m 58\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m sessions.Session() \u001b[38;5;28;01mas\u001b[39;00m session:\n\u001b[32m---> \u001b[39m\u001b[32m59\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msession\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/requests/sessions.py:589\u001b[39m, in \u001b[36mSession.request\u001b[39m\u001b[34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[39m\n\u001b[32m 584\u001b[39m send_kwargs = {\n\u001b[32m 585\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mtimeout\u001b[39m\u001b[33m\"\u001b[39m: timeout,\n\u001b[32m 586\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mallow_redirects\u001b[39m\u001b[33m\"\u001b[39m: allow_redirects,\n\u001b[32m 587\u001b[39m }\n\u001b[32m 588\u001b[39m send_kwargs.update(settings)\n\u001b[32m--> \u001b[39m\u001b[32m589\u001b[39m resp = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 591\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/requests/sessions.py:703\u001b[39m, in \u001b[36mSession.send\u001b[39m\u001b[34m(self, request, **kwargs)\u001b[39m\n\u001b[32m 700\u001b[39m start = preferred_clock()\n\u001b[32m 702\u001b[39m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m703\u001b[39m r = \u001b[43madapter\u001b[49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 705\u001b[39m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[32m 706\u001b[39m elapsed = preferred_clock() - start\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/requests/adapters.py:677\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 673\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e.reason, _SSLError):\n\u001b[32m 674\u001b[39m \u001b[38;5;66;03m# This branch is for urllib3 v1.22 and later.\u001b[39;00m\n\u001b[32m 675\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m SSLError(e, request=request)\n\u001b[32m--> \u001b[39m\u001b[32m677\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request=request)\n\u001b[32m 679\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m ClosedPoolError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 680\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request=request)\n",
"\u001b[31mConnectionError\u001b[39m: HTTPConnectionPool(host='localhost', port=8000): Max retries exceeded with url: /api/upload (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fcadf6b8c10>: Failed to establish a new connection: [Errno 111] Connection refused'))"
]
}
],
"source": [
"import requests\n",
"import json\n",
"\n",
"# Define the URL of the API endpoint\n",
"url = \"http://localhost:8000/api/upload\"\n",
"\n",
"# Path to the model file you want to upload\n",
"model_file_path = \"spam_detection_model.joblib\"\n",
"\n",
"# Open the file in binary mode and send the POST request\n",
"with open(model_file_path, \"rb\") as model_file:\n",
" files = {\"model\": model_file}\n",
" response = requests.post(url, files=files)\n",
"\n",
"# Pretty print the response from the server\n",
"print(json.dumps(response.json(), indent=4))\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d314b75e-595d-4c8a-9cb2-f81582a50bc0",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
applications-of-ai-in-infosec/Spam Classification/sms_spam_collection/SMSSpamCollection
+5574
View File
File diff suppressed because it is too large Load Diff
applications-of-ai-in-infosec/Spam Classification/sms_spam_collection/readme
+75
View File
@@ -0,0 +1,75 @@
SMS Spam Collection v.1
-------------------------
1. DESCRIPTION
--------------
The SMS Spam Collection v.1 (hereafter the corpus) is a set of SMS tagged messages that have been collected for SMS Spam research. It contains one set of SMS messages in English of 5,574 messages, tagged acording being ham (legitimate) or spam.
1.1. Compilation
----------------
This corpus has been collected from free or free for research sources at the Web:
- A collection of between 425 SMS spam messages extracted manually from the Grumbletext Web site. This is a UK forum in which cell phone users make public claims about SMS spam messages, most of them without reporting the very spam message received. The identification of the text of spam messages in the claims is a very hard and time-consuming task, and it involved carefully scanning hundreds of web pages. The Grumbletext Web site is: http://www.grumbletext.co.uk/
- A list of 450 SMS ham messages collected from Caroline Tag's PhD Theses available at http://etheses.bham.ac.uk/253/1/Tagg09PhD.pdf
- A subset of 3,375 SMS ham messages of the NUS SMS Corpus (NSC), which is a corpus of about 10,000 legitimate messages collected for research at the Department of Computer Science at the National University of Singapore. The messages largely originate from Singaporeans and mostly from students attending the University. These messages were collected from volunteers who were made aware that their contributions were going to be made publicly available. The NUS SMS Corpus is avalaible at: http://www.comp.nus.edu.sg/~rpnlpir/downloads/corpora/smsCorpus/
- The amount of 1,002 SMS ham messages and 322 spam messages extracted from the SMS Spam Corpus v.0.1 Big created by José María Gómez Hidalgo and public available at: http://www.esp.uem.es/jmgomez/smsspamcorpus/
1.2. Statistics
---------------
There is one collection:
- The SMS Spam Collection v.1 (text file: smsspamcollection) has a total of 4,827 SMS legitimate messages (86.6%) and a total of 747 (13.4%) spam messages.
1.3. Format
-----------
The files contain one message per line. Each line is composed by two columns: one with label (ham or spam) and other with the raw text. Here are some examples:
ham What you doing?how are you?
ham Ok lar... Joking wif u oni...
ham dun say so early hor... U c already then say...
ham MY NO. IN LUTON 0125698789 RING ME IF UR AROUND! H*
ham Siva is in hostel aha:-.
ham Cos i was out shopping wif darren jus now n i called him 2 ask wat present he wan lor. Then he started guessing who i was wif n he finally guessed darren lor.
spam FreeMsg: Txt: CALL to No: 86888 & claim your reward of 3 hours talk time to use from your phone now! ubscribe6GBP/ mnth inc 3hrs 16 stop?txtStop
spam Sunshine Quiz! Win a super Sony DVD recorder if you canname the capital of Australia? Text MQUIZ to 82277. B
spam URGENT! Your Mobile No 07808726822 was awarded a L2,000 Bonus Caller Prize on 02/09/03! This is our 2nd attempt to contact YOU! Call 0871-872-9758 BOX95QU
Note: messages are not chronologically sorted.
2. USAGE
--------
We offer a comprehensive study of this corpus in the following paper that is under review. This work presents a number of statistics, studies and baseline results for several machine learning methods.
[1] Almeida, T.A., Gómez Hidalgo, J.M., Yamakami, A. Contributions to the study of SMS Spam Filtering: New Collection and Results. Proceedings of the 2011 ACM Symposium on Document Engineering (ACM DOCENG'11), Mountain View, CA, USA, 2011. (Under review)
3. ABOUT
--------
The corpus has been collected by Tiago Agostinho de Almeida (http://www.dt.fee.unicamp.br/~tiago) and José María Gómez Hidalgo (http://www.esp.uem.es/jmgomez).
We would like to thank Dr. Min-Yen Kan (http://www.comp.nus.edu.sg/~kanmy/) and his team for making the NUS SMS Corpus available. See: http://www.comp.nus.edu.sg/~rpnlpir/downloads/corpora/smsCorpus/. He is currently collecting a bigger SMS corpus at: http://wing.comp.nus.edu.sg:8080/SMSCorpus/
4. LICENSE/DISCLAIMER
---------------------
We would appreciate if:
- In case you find this corpus useful, please make a reference to previous paper and the web page: http://www.dt.fee.unicamp.br/~tiago/smsspamcollection/ in your papers, research, etc.
- Send us a message to tiago@dt.fee.unicamp.br in case you make use of the corpus.
The SMS Spam Collection v.1 is provided for free and with no limitations excepting:
1. Tiago Agostinho de Almeida and José María Gómez Hidalgo hold the copyrigth (c) for the SMS Spam Collection v.1.
2. No Warranty/Use At Your Risk. THE CORPUS IS MADE AT NO CHARGE. ACCORDINGLY, THE CORPUS IS PROVIDED `AS IS,' WITHOUT WARRANTY OF ANY KIND, INCLUDING WITHOUT LIMITATION THE WARRANTIES THAT THEY ARE MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. YOU ARE SOLELY RESPONSIBLE FOR YOUR USE, DISTRIBUTION, MODIFICATION, REPRODUCTION AND PUBLICATION OF THE CORPUS AND ANY DERIVATIVE WORKS THEREOF BY YOU AND ANY OF YOUR SUBLICENSEES (COLLECTIVELY, `YOUR CORPUS USE'). THE ENTIRE RISK AS TO YOUR CORPUS USE IS BORNE BY YOU. YOU AGREE TO INDEMNIFY AND HOLD THE COPYRIGHT HOLDERS, AND THEIR AFFILIATES HARMLESS FROM ANY CLAIMS ARISING FROM OR RELATING TO YOUR CORPUS USE.
3. Limitation of Liability. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR THEIR AFFILIATES, OR THE CORPUS CONTRIBUTING EDITORS, BE LIABLE FOR ANY INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF ADVISED OF THE POSSIBILITY THEREOF, AND REGARDLESS OF WHETHER ANY CLAIM IS BASED UPON ANY CONTRACT, TORT OR OTHER LEGAL OR EQUITABLE THEORY, RELATING OR ARISING FROM THE CORPUS, YOUR CORPUS USE OR THIS LICENSE AGREEMENT.
applications-of-ai-in-infosec/Spam Classification/spam_detection_model.joblib
BIN
View File
Binary file not shown.
applications-of-ai-in-infosec/demo_dataset.csv
+101
View File
@@ -0,0 +1,101 @@
log_id,source_ip,destination_port,protocol,bytes_transferred,threat_level
10,10.0.0.100,STRING_PORT,FTP,4096,?
12,172.16.254.100,110,POP3,NEGATIVE,1
27,172.16.254.200,110,POP3,NON_NUMERIC,1
1,192.168.1.100,80,HTTP,1024,0
2,192.168.1.81,53,TLS,9765,0
3,192.168.1.147,80,SSH,1296,1
4,192.168.1.94,22,POP3,7185,0
5,192.168.1.55,110,DNS,5088,2
6,192.168.1.171,443,HTTPS,5989,2
7,192.168.1.196,22,SMTP,3166,2
8,192.168.1.67,443,HTTP,8279,1
9,192.168.1.66,22,SSH,6124,0
11,192.168.1.55,80,TLS,1003,1
13,192.168.1.84,80,DNS,4138,2
14,192.168.1.160,80,HTTP,7480,0
15,192.168.1.232,22,SMTP,6571,1
16,172.16.1.75,80,HTTP,1024,0
17,192.168.1.184,53,HTTP,6899,2
18,192.168.1.57,53,HTTPS,8014,2
19,192.168.1.64,110,POP3,1501,0
20,192.168.1.97,80,SSH,2965,2
21,192.168.1.55,25,DNS,1479,1
22,192.168.1.44,80,FTP,1629,2
23,192.168.1.166,22,HTTP,8243,1
24,192.168.1.220,53,UDP,6843,2
25,192.168.1.169,80,TLS,1215,0
26,192.168.1.168,80,SMTP,5966,1
28,192.168.1.247,22,DNS,2469,2
29,192.168.1.115,25,HTTP,5455,1
30,172.16.1.125,80,HTTP,1024,0
31,192.168.1.119,443,UNKNOWN,9513,2
32,192.168.1.62,110,HTTPS,6388,1
33,192.168.1.219,110,POP3,3057,1
34,192.168.1.203,443,SMTP,7564,2
35,192.168.1.200,STRING_PORT,FTP,4096,?
36,192.168.1.20,443,HTTP,7841,2
37,192.168.1.24,110,HTTP,5357,0
38,192.168.1.49,80,TLS,1676,1
39,192.168.1.228,110,POP3,3059,0
40,192.168.1.249,53,DNS,2090,0
41,10.0.0.300,25,SMTP,4096,0
42,192.168.1.237,53,UDP,4855,2
43,172.16.1.200,80,HTTP,1024,0
44,192.168.1.95,443,FTP,4993,1
45,192.168.1.61,110,HTTPS,498,0
46,192.168.1.135,80,POP3,4525,0
47,192.168.1.53,443,HTTP,2504,2
48,192.168.1.228,25,DNS,4310,0
49,192.168.1.112,443,SMTP,5450,0
50,192.168.1.33,53,UDP,5919,1
51,192.168.1.99,110,HTTP,2420,1
52,10.10.10.450,STRING_PORT,FTP,4096,?
53,192.168.1.247,110,TLS,4609,0
54,192.168.1.248,110,POP3,2224,1
55,192.168.1.249,53,HTTP,6022,1
56,,53,DNS,1024,0
57,192.168.1.9,22,SMTP,9659,2
58,192.168.1.475,,UDP,2048,1
59,192.168.1.126,80,HTTP,1252,1
60,192.168.1.155,80,FTP,4410,2
61,192.168.1.166,110,HTTPS,6158,2
62,192.168.1.77,110,POP3,4096,1
63,192.168.1.21,80,HTTP,1699,1
64,MISSING_IP,53,DNS,1024,0
65,192.168.1.53,110,SMTP,2846,2
66,192.168.1.600,UNUSED_PORT,UDP,2048,1
67,192.168.1.152,110,HTTP,5314,0
68,10.10.10.77,STRING_PORT,FTP,4096,?
69,192.168.1.68,110,TLS,8935,0
70,192.168.1.172,25,POP3,3264,1
71,192.168.1.32,80,HTTP,6991,2
72,MISSING_IP,53,DNS,1024,0
73,192.168.1.29,22,SMTP,2974,0
74,192.168.1.140,443,UDP,9390,0
75,172.16.1.400,80,HTTP,1024,0
76,192.168.1.244,443,FTP,4942,2
77,192.168.1.4,110,HTTPS,1054,0
78,192.168.1.43,443,POP3,7105,1
79,172.16.254.77,999999,HTTP,2048,1
80,192.168.1.72,25,DNS,4506,1
81,192.168.1.224,25,UNKNOWN,1161,1
82,192.168.1.40,110,UDP,4659,2
83,172.16.1.450,80,HTTP,1024,0
84,192.168.1.214,80,FTP,5726,0
85,192.168.1.240,443,SMTP,4993,1
86,192.168.1.204,443,POP3,1158,2
87,192.168.1.40,80,HTTP,7083,2
88,MISSING_IP,53,DNS,1024,0
89,10.10.10.700,443,TLS,512,1
90,192.168.1.125,25,UDP,5004,1
91,192.168.1.200,110,HTTP,2433,2
92,192.168.1.219,22,FTP,6949,1
93,INVALID_IP,110,POP3,4096,1
94,192.168.1.1050,53,DNS,NON_NUMERIC,0
95,192.168.1.174,80,HTTP,1649,2
96,MISSING_IP,25,SMTP,4096,1
97,192.168.1.60,80,HTTPS,4351,1
98,192.168.1.1100,UNUSED_PORT,UDP,2048,0
99,192.168.1.16,22,HTTP,9069,0
100,192.168.1.14,25,SMTP,1494,2
1 log_id source_ip destination_port protocol bytes_transferred threat_level
2 10 10.0.0.100 STRING_PORT FTP 4096 ?
3 12 172.16.254.100 110 POP3 NEGATIVE 1
4 27 172.16.254.200 110 POP3 NON_NUMERIC 1
5 1 192.168.1.100 80 HTTP 1024 0
6 2 192.168.1.81 53 TLS 9765 0
7 3 192.168.1.147 80 SSH 1296 1
8 4 192.168.1.94 22 POP3 7185 0
9 5 192.168.1.55 110 DNS 5088 2
10 6 192.168.1.171 443 HTTPS 5989 2
11 7 192.168.1.196 22 SMTP 3166 2
12 8 192.168.1.67 443 HTTP 8279 1
13 9 192.168.1.66 22 SSH 6124 0
14 11 192.168.1.55 80 TLS 1003 1
15 13 192.168.1.84 80 DNS 4138 2
16 14 192.168.1.160 80 HTTP 7480 0
17 15 192.168.1.232 22 SMTP 6571 1
18 16 172.16.1.75 80 HTTP 1024 0
19 17 192.168.1.184 53 HTTP 6899 2
20 18 192.168.1.57 53 HTTPS 8014 2
21 19 192.168.1.64 110 POP3 1501 0
22 20 192.168.1.97 80 SSH 2965 2
23 21 192.168.1.55 25 DNS 1479 1
24 22 192.168.1.44 80 FTP 1629 2
25 23 192.168.1.166 22 HTTP 8243 1
26 24 192.168.1.220 53 UDP 6843 2
27 25 192.168.1.169 80 TLS 1215 0
28 26 192.168.1.168 80 SMTP 5966 1
29 28 192.168.1.247 22 DNS 2469 2
30 29 192.168.1.115 25 HTTP 5455 1
31 30 172.16.1.125 80 HTTP 1024 0
32 31 192.168.1.119 443 UNKNOWN 9513 2
33 32 192.168.1.62 110 HTTPS 6388 1
34 33 192.168.1.219 110 POP3 3057 1
35 34 192.168.1.203 443 SMTP 7564 2
36 35 192.168.1.200 STRING_PORT FTP 4096 ?
37 36 192.168.1.20 443 HTTP 7841 2
38 37 192.168.1.24 110 HTTP 5357 0
39 38 192.168.1.49 80 TLS 1676 1
40 39 192.168.1.228 110 POP3 3059 0
41 40 192.168.1.249 53 DNS 2090 0
42 41 10.0.0.300 25 SMTP 4096 0
43 42 192.168.1.237 53 UDP 4855 2
44 43 172.16.1.200 80 HTTP 1024 0
45 44 192.168.1.95 443 FTP 4993 1
46 45 192.168.1.61 110 HTTPS 498 0
47 46 192.168.1.135 80 POP3 4525 0
48 47 192.168.1.53 443 HTTP 2504 2
49 48 192.168.1.228 25 DNS 4310 0
50 49 192.168.1.112 443 SMTP 5450 0
51 50 192.168.1.33 53 UDP 5919 1
52 51 192.168.1.99 110 HTTP 2420 1
53 52 10.10.10.450 STRING_PORT FTP 4096 ?
54 53 192.168.1.247 110 TLS 4609 0
55 54 192.168.1.248 110 POP3 2224 1
56 55 192.168.1.249 53 HTTP 6022 1
57 56 53 DNS 1024 0
58 57 192.168.1.9 22 SMTP 9659 2
59 58 192.168.1.475 UDP 2048 1
60 59 192.168.1.126 80 HTTP 1252 1
61 60 192.168.1.155 80 FTP 4410 2
62 61 192.168.1.166 110 HTTPS 6158 2
63 62 192.168.1.77 110 POP3 4096 1
64 63 192.168.1.21 80 HTTP 1699 1
65 64 MISSING_IP 53 DNS 1024 0
66 65 192.168.1.53 110 SMTP 2846 2
67 66 192.168.1.600 UNUSED_PORT UDP 2048 1
68 67 192.168.1.152 110 HTTP 5314 0
69 68 10.10.10.77 STRING_PORT FTP 4096 ?
70 69 192.168.1.68 110 TLS 8935 0
71 70 192.168.1.172 25 POP3 3264 1
72 71 192.168.1.32 80 HTTP 6991 2
73 72 MISSING_IP 53 DNS 1024 0
74 73 192.168.1.29 22 SMTP 2974 0
75 74 192.168.1.140 443 UDP 9390 0
76 75 172.16.1.400 80 HTTP 1024 0
77 76 192.168.1.244 443 FTP 4942 2
78 77 192.168.1.4 110 HTTPS 1054 0
79 78 192.168.1.43 443 POP3 7105 1
80 79 172.16.254.77 999999 HTTP 2048 1
81 80 192.168.1.72 25 DNS 4506 1
82 81 192.168.1.224 25 UNKNOWN 1161 1
83 82 192.168.1.40 110 UDP 4659 2
84 83 172.16.1.450 80 HTTP 1024 0
85 84 192.168.1.214 80 FTP 5726 0
86 85 192.168.1.240 443 SMTP 4993 1
87 86 192.168.1.204 443 POP3 1158 2
88 87 192.168.1.40 80 HTTP 7083 2
89 88 MISSING_IP 53 DNS 1024 0
90 89 10.10.10.700 443 TLS 512 1
91 90 192.168.1.125 25 UDP 5004 1
92 91 192.168.1.200 110 HTTP 2433 2
93 92 192.168.1.219 22 FTP 6949 1
94 93 INVALID_IP 110 POP3 4096 1
95 94 192.168.1.1050 53 DNS NON_NUMERIC 0
96 95 192.168.1.174 80 HTTP 1649 2
97 96 MISSING_IP 25 SMTP 4096 1
98 97 192.168.1.60 80 HTTPS 4351 1
99 98 192.168.1.1100 UNUSED_PORT UDP 2048 0
100 99 192.168.1.16 22 HTTP 9069 0
101 100 192.168.1.14 25 SMTP 1494 2
applications-of-ai-in-infosec/demo_dataset.zip
BIN
View File
Binary file not shown.
applications-of-ai-in-infosec/download
+65
View File
@@ -0,0 +1,65 @@
label,message
spam,I'll let you know when it kicks in
spam,I've told you everything will stop. Just dont let her get dehydrated.
spam,Hmm thinking lor...
spam,I don't know but I'm raping dudes at poker
spam,What time you thinkin of goin?
ham,"FREE RINGTONE text FIRST to 87131 for a poly or text GET to 87131 for a true tone! Help? 0845 2814032 16 after 1st free, tones are 3x£150pw to e£nd txt stop"
spam,"I'm used to it. I just hope my agents don't drop me since i've only booked a few things this year. This whole me in boston, them in nyc was an experiment."
ham,Not heard from U4 a while. Call 4 rude chat private line 01223585334 to cum. Wan 2C pics of me gettin shagged then text PIX to 8552. 2End send STOP 8552 SAM xxx
spam,No b4 Thursday
spam,Ok i am on the way to railway
spam,Fancy a shag? I do.Interested? sextextuk.com txt XXUK SUZY to 69876. Txts cost 1.50 per msg. TnCs on website. X
spam,Hi there. We have now moved in2 our pub . Would be great 2 c u if u cud come up.
spam,Dear got bus directly to calicut
spam,I had a good time too. Its nice to do something a bit different with my weekends for a change. See ya soon
spam,Someonone you know is trying to contact you via our dating service! To find out who it could be call from your mobile or landline 09064015307 BOX334SK38ch
spam,Wish u many many returns of the day.. Happy birthday vikky..
spam,TBS/PERSOLVO. been chasing us since Sept for£38 definitely not paying now thanks to your information. We will ignore them. Kath. Manchester.
ham,"It wont b until 2.15 as trying 2 sort house out, is that ok?"
ham,Hello! Good week? Fancy a drink or something later?
spam,WOW! The Boys R Back. TAKE THAT 2007 UK Tour. Win VIP Tickets & pre-book with VIP Club. Txt CLUB to 81303. Trackmarque Ltd info@vipclub4u.
ham,Did you stitch his trouser
ham,S da..al r above &lt;#&gt;
ham,I'm going for bath will msg you next &lt;#&gt; min..
spam,We have new local dates in your area - Lots of new people registered in YOUR AREA. Reply DATE to start now! 18 only www.flirtparty.us REPLYS150
ham,It means u could not keep ur words.
ham,*deep sigh* ... I miss you :-( ... I am really surprised you haven't gone to the net cafe yet to get to me ... Don't you miss me?
ham,how tall are you princess?
ham,We are supposed to meet to discuss abt our trip... Thought xuhui told you? In the afternoon. Thought we can go for lesson after that
ham,Havent planning to buy later. I check already lido only got 530 show in e afternoon. U finish work already?
ham,K.k.how is your business now?still meetin 4 lunch tmr a not. Neva hear fr them lei. Ü got a lot of work ar?
ham,"Set a place for me in your heart and not in your mind, as the mind easily forgets but the heart will always remember. Wish you Happy Valentines Day!"
ham,If you have belive me. Come to my home.
ham,What are youdoing later? Sar xxx
ham,Shall i send that exe to your mail id.
spam,from www.Applausestore.com MonthlySubscription@50p/msg max6/month T&CsC web age16 2stop txt stop
ham,Did u see what I posted on your Facebook?
ham,Take us out shopping and Mark will distract Isaiah.=D
ham,Good night. Am going to sleep.
ham,Lol its ok I didn't remember til last nite
ham,Probably a couple hours tops
ham,GRAN ONLYFOUND OUT AFEW DAYS AGO.CUSOON HONI
ham,Sunshine Quiz Wkly Q! Win a top Sony DVD player if u know which country the Algarve is in? Txt ansr to 82277. £1.50 SP:Tyrone
ham,Are you driving or training?
ham,"URGENT! Your mobile number *************** WON a £2000 Bonus Caller prize on 10/06/03! This is the 2nd attempt to reach you! Call 09066368753 ASAP! Box 97N7QP, 150ppm"
spam,Serious? What like proper tongued her
spam,I will cme i want to go to hos 2morow. After that i wil cme. This what i got from her dear what to do. She didnt say any time
spam,"(I should add that I don't really care and if you can't I can at least get this dude to fuck off but hey, your money if you want it)"
spam,EY! CALM DOWNON THEACUSATIONS.. ITXT U COS IWANA KNOW WOTU R DOIN AT THEW/END... HAVENTCN U IN AGES..RING ME IF UR UP4 NETHING SAT.LOVE J XXX.
spam,Is avatar supposed to have subtoitles
spam,Have you bookedthe hut? And also your time off? How are you by the way?
spam,Have a great trip to India. And bring the light to everyone not just with the project but with everyone that is lucky to see you smile. Bye. Abiola
ham,SMS. ac Sptv: The New Jersey Devils and the Detroit Red Wings play Ice Hockey. Correct or Incorrect? End? Reply END SPTV
spam,MAKE SURE ALEX KNOWS HIS BIRTHDAY IS OVER IN FIFTEEN MINUTES AS FAR AS YOU'RE CONCERNED
spam,At what time are you coming.
spam,"Hey, I missed you tm of last night as my phone was on the charge ... *smiles* ... I am meeting a friend shortly"
spam,Was it something u ate?
spam,See? I thought it all through
spam,No problem. Talk to you later
ham,What * u wearing?
ham,U come n search tat vid..not finishd..
ham,"I don't have anybody's number, I still haven't thought up a tactful way to ask alex"
ham,I need... Coz i never go before
ham,Feel like trying kadeem again? :V
spam,Do ü all wan 2 meet up n combine all the parts? How's da rest of da project going?