Jeremy Janella
903 lines
50 KiB
Plaintext
903 lines
50 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "738bf5f9-3757-4b17-a799-7ba335c4b269",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Create the dataset"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"id": "e9a6abb6-aeba-4b0c-bbfb-5f92a4f18e21",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Download successful\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import requests\n",
|
|
"import zipfile\n",
|
|
"import io\n",
|
|
"\n",
|
|
"# URL of the dataset\n",
|
|
"url = \"https://archive.ics.uci.edu/static/public/228/sms+spam+collection.zip\"\n",
|
|
"# Download the dataset\n",
|
|
"response = requests.get(url)\n",
|
|
"if response.status_code == 200:\n",
|
|
" print(\"Download successful\")\n",
|
|
"else:\n",
|
|
" print(\"Failed to download the dataset\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "7e432811-930f-4fe5-913d-baf25f0dc1cf",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Extract"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"id": "0213809b-9b15-4aaa-8b23-54fcf0ce47a5",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Extraction successful\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# Extract the dataset\n",
|
|
"with zipfile.ZipFile(io.BytesIO(response.content)) as z:\n",
|
|
" z.extractall(\"sms_spam_collection\")\n",
|
|
" print(\"Extraction successful\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "15103f06-8dcf-466b-8a2a-9d8e5db68bda",
|
|
"metadata": {},
|
|
"source": [
|
|
"# View files"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"id": "db9de74d-fb53-4518-9765-61babfa84001",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Extracted files: ['SMSSpamCollection', 'readme']\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import os\n",
|
|
"\n",
|
|
"# List the extracted files\n",
|
|
"extracted_files = os.listdir(\"sms_spam_collection\")\n",
|
|
"print(\"Extracted files:\", extracted_files)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "19dc4d0e-407e-478a-813b-e0fe466a4912",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Load Dataset"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"id": "2fcb7819-c5ae-4932-a04d-3f7e89c23840",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"# Load the dataset\n",
|
|
"df = pd.read_csv(\n",
|
|
" \"sms_spam_collection/SMSSpamCollection\",\n",
|
|
" sep=\"\\t\",\n",
|
|
" header=None,\n",
|
|
" names=[\"label\", \"message\"],\n",
|
|
")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "a8d0056c-f7ba-416a-b973-abbfe9c9112e",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Display Dataset"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"id": "74516d20-fc54-4555-ad45-1c54061f3721",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"-------------------- HEAD --------------------\n",
|
|
" label message\n",
|
|
"0 ham Go until jurong point, crazy.. Available only ...\n",
|
|
"1 ham Ok lar... Joking wif u oni...\n",
|
|
"2 spam Free entry in 2 a wkly comp to win FA Cup fina...\n",
|
|
"3 ham U dun say so early hor... U c already then say...\n",
|
|
"4 ham Nah I don't think he goes to usf, he lives aro...\n",
|
|
"-------------------- DESCRIBE --------------------\n",
|
|
" label message\n",
|
|
"count 5572 5572\n",
|
|
"unique 2 5169\n",
|
|
"top ham Sorry, I'll call later\n",
|
|
"freq 4825 30\n",
|
|
"-------------------- INFO --------------------\n",
|
|
"<class 'pandas.core.frame.DataFrame'>\n",
|
|
"RangeIndex: 5572 entries, 0 to 5571\n",
|
|
"Data columns (total 2 columns):\n",
|
|
" # Column Non-Null Count Dtype \n",
|
|
"--- ------ -------------- ----- \n",
|
|
" 0 label 5572 non-null object\n",
|
|
" 1 message 5572 non-null object\n",
|
|
"dtypes: object(2)\n",
|
|
"memory usage: 87.2+ KB\n",
|
|
"None\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# Display basic information about the dataset\n",
|
|
"print(\"-------------------- HEAD --------------------\")\n",
|
|
"print(df.head())\n",
|
|
"print(\"-------------------- DESCRIBE --------------------\")\n",
|
|
"print(df.describe())\n",
|
|
"print(\"-------------------- INFO --------------------\")\n",
|
|
"print(df.info())\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "8ad80cb6-0aaa-4468-b17f-057db8a5d984",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Find missing Values"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"id": "d097fca1-daf8-418d-90d3-62f8831c9e86",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Missing values:\n",
|
|
" label 0\n",
|
|
"message 0\n",
|
|
"dtype: int64\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# Check for missing values\n",
|
|
"print(\"Missing values:\\n\", df.isnull().sum())\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "fe212f08-ad18-4358-a89a-4247f3322163",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Find duplicate values"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"id": "c469894c-8e88-4893-90de-180842bbc084",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Duplicate entries: 403\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# Check for duplicates\n",
|
|
"print(\"Duplicate entries:\", df.duplicated().sum())\n",
|
|
"\n",
|
|
"# Remove duplicates if any\n",
|
|
"df = df.drop_duplicates()\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "2494b426-1ce5-4f6e-92a8-9e7ffef7a651",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Preprocessing the Spam Dataset\n",
|
|
"nltk tokenizes and cleans data\n",
|
|
"- lowercases data\n",
|
|
"- removes punctuation\n",
|
|
"- tokenizes text\n",
|
|
"- removes stopwords\n",
|
|
"- does stemming"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"id": "831d49a0-0d39-44a3-894b-f9beebe82e06",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"[nltk_data] Downloading package punkt to /home/jeremy/nltk_data...\n",
|
|
"[nltk_data] Unzipping tokenizers/punkt.zip.\n",
|
|
"[nltk_data] Downloading package punkt_tab to /home/jeremy/nltk_data...\n",
|
|
"[nltk_data] Unzipping tokenizers/punkt_tab.zip.\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"=== BEFORE ANY PREPROCESSING ===\n",
|
|
" label message\n",
|
|
"0 ham Go until jurong point, crazy.. Available only ...\n",
|
|
"1 ham Ok lar... Joking wif u oni...\n",
|
|
"2 spam Free entry in 2 a wkly comp to win FA Cup fina...\n",
|
|
"3 ham U dun say so early hor... U c already then say...\n",
|
|
"4 ham Nah I don't think he goes to usf, he lives aro...\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"[nltk_data] Downloading package stopwords to /home/jeremy/nltk_data...\n",
|
|
"[nltk_data] Unzipping corpora/stopwords.zip.\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import nltk\n",
|
|
"\n",
|
|
"# Download the necessary NLTK data files\n",
|
|
"nltk.download(\"punkt\")\n",
|
|
"nltk.download(\"punkt_tab\")\n",
|
|
"nltk.download(\"stopwords\")\n",
|
|
"\n",
|
|
"print(\"=== BEFORE ANY PREPROCESSING ===\") \n",
|
|
"print(df.head(5))\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"id": "0e149fe9-a3e8-42f9-8962-eb0fa159e983",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n",
|
|
"=== AFTER LOWERCASING ===\n",
|
|
"0 go until jurong point, crazy.. available only ...\n",
|
|
"1 ok lar... joking wif u oni...\n",
|
|
"2 free entry in 2 a wkly comp to win fa cup fina...\n",
|
|
"3 u dun say so early hor... u c already then say...\n",
|
|
"4 nah i don't think he goes to usf, he lives aro...\n",
|
|
"Name: message, dtype: object\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# Convert all message text to lowercase\n",
|
|
"df[\"message\"] = df[\"message\"].str.lower()\n",
|
|
"print(\"\\n=== AFTER LOWERCASING ===\")\n",
|
|
"print(df[\"message\"].head(5))\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"id": "883576e9-52fd-4b1f-aadb-b8f75ec4ac0d",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n",
|
|
"=== AFTER REMOVING PUNCTUATION & NUMBERS (except $ and !) ===\n",
|
|
"0 go until jurong point crazy available only in ...\n",
|
|
"1 ok lar joking wif u oni\n",
|
|
"2 free entry in a wkly comp to win fa cup final...\n",
|
|
"3 u dun say so early hor u c already then say\n",
|
|
"4 nah i dont think he goes to usf he lives aroun...\n",
|
|
"Name: message, dtype: object\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import re\n",
|
|
"\n",
|
|
"# Remove non-essential punctuation and numbers, keep useful symbols like $ and !\n",
|
|
"df[\"message\"] = df[\"message\"].apply(lambda x: re.sub(r\"[^a-z\\s$!]\", \"\", x))\n",
|
|
"print(\"\\n=== AFTER REMOVING PUNCTUATION & NUMBERS (except $ and !) ===\")\n",
|
|
"print(df[\"message\"].head(5))\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 13,
|
|
"id": "ea8b673e-f1ed-499d-84c8-050144332a75",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n",
|
|
"=== AFTER TOKENIZATION ===\n",
|
|
"0 [go, until, jurong, point, crazy, available, o...\n",
|
|
"1 [ok, lar, joking, wif, u, oni]\n",
|
|
"2 [free, entry, in, a, wkly, comp, to, win, fa, ...\n",
|
|
"3 [u, dun, say, so, early, hor, u, c, already, t...\n",
|
|
"4 [nah, i, dont, think, he, goes, to, usf, he, l...\n",
|
|
"Name: message, dtype: object\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from nltk.tokenize import word_tokenize\n",
|
|
"\n",
|
|
"# Split each message into individual tokens\n",
|
|
"df[\"message\"] = df[\"message\"].apply(word_tokenize)\n",
|
|
"print(\"\\n=== AFTER TOKENIZATION ===\")\n",
|
|
"print(df[\"message\"].head(5))\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 14,
|
|
"id": "b6efb48a-f61a-4613-86b9-9ca8b38cc144",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n",
|
|
"=== AFTER REMOVING STOP WORDS ===\n",
|
|
"0 [go, jurong, point, crazy, available, bugis, n...\n",
|
|
"1 [ok, lar, joking, wif, u, oni]\n",
|
|
"2 [free, entry, wkly, comp, win, fa, cup, final,...\n",
|
|
"3 [u, dun, say, early, hor, u, c, already, say]\n",
|
|
"4 [nah, dont, think, goes, usf, lives, around, t...\n",
|
|
"Name: message, dtype: object\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from nltk.corpus import stopwords\n",
|
|
"\n",
|
|
"# Define a set of English stop words and remove them from the tokens\n",
|
|
"stop_words = set(stopwords.words(\"english\"))\n",
|
|
"df[\"message\"] = df[\"message\"].apply(lambda x: [word for word in x if word not in stop_words])\n",
|
|
"print(\"\\n=== AFTER REMOVING STOP WORDS ===\")\n",
|
|
"print(df[\"message\"].head(5))\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 15,
|
|
"id": "e556ed99-3b3e-4b77-9451-11500de66153",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n",
|
|
"=== AFTER STEMMING ===\n",
|
|
"0 [go, jurong, point, crazi, avail, bugi, n, gre...\n",
|
|
"1 [ok, lar, joke, wif, u, oni]\n",
|
|
"2 [free, entri, wkli, comp, win, fa, cup, final,...\n",
|
|
"3 [u, dun, say, earli, hor, u, c, alreadi, say]\n",
|
|
"4 [nah, dont, think, goe, usf, live, around, tho...\n",
|
|
"Name: message, dtype: object\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from nltk.stem import PorterStemmer\n",
|
|
"\n",
|
|
"# Stem each token to reduce words to their base form\n",
|
|
"stemmer = PorterStemmer()\n",
|
|
"df[\"message\"] = df[\"message\"].apply(lambda x: [stemmer.stem(word) for word in x])\n",
|
|
"print(\"\\n=== AFTER STEMMING ===\")\n",
|
|
"print(df[\"message\"].head(5))\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 16,
|
|
"id": "f1c4ecf0-79a7-4f58-b73a-b17c304ee15f",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n",
|
|
"=== AFTER JOINING TOKENS BACK INTO STRINGS ===\n",
|
|
"0 go jurong point crazi avail bugi n great world...\n",
|
|
"1 ok lar joke wif u oni\n",
|
|
"2 free entri wkli comp win fa cup final tkt st m...\n",
|
|
"3 u dun say earli hor u c alreadi say\n",
|
|
"4 nah dont think goe usf live around though\n",
|
|
"Name: message, dtype: object\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# Rejoin tokens into a single string for feature extraction\n",
|
|
"df[\"message\"] = df[\"message\"].apply(lambda x: \" \".join(x))\n",
|
|
"print(\"\\n=== AFTER JOINING TOKENS BACK INTO STRINGS ===\")\n",
|
|
"print(df[\"message\"].head(5))\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "417c3d1b-3c15-416a-a808-50c266d9bd8f",
|
|
"metadata": {},
|
|
"source": [
|
|
"# CountVectorizer Bag of Words vectorization\n",
|
|
"- Minimum occurance: 1\n",
|
|
"- Maxiumum appearce: 90%\n",
|
|
"- ngram range: 1-2\n",
|
|
"- Vectorizes (into vectors :0)\n",
|
|
"\n",
|
|
"ngrams are counted that fit this description, into vector"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 19,
|
|
"id": "83210ebb-fbf2-42b4-b2b6-9b6fdcd2a39c",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from sklearn.feature_extraction.text import CountVectorizer\n",
|
|
"\n",
|
|
"# Initialize CountVectorizer with bigrams, min_df, and max_df to focus on relevant terms\n",
|
|
"vectorizer = CountVectorizer(min_df=1, max_df=0.9, ngram_range=(1, 2))\n",
|
|
"\n",
|
|
"# Fit and transform the message column\n",
|
|
"X = vectorizer.fit_transform(df[\"message\"])\n",
|
|
"\n",
|
|
"# Labels (target variable)\n",
|
|
"y = df[\"label\"].apply(lambda x: 1 if x == \"spam\" else 0) # Converting labels to 1 and 0\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "380eae4b-e569-4bc9-b9a2-aee0214a76dd",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Train the model\n",
|
|
"\n",
|
|
"Multinomial naive Bayes to find probabilities of spam and feature relationships based on the input vectors\n",
|
|
"\n",
|
|
"Pipeline datastructure chains together vectorization and modeling steps"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 22,
|
|
"id": "6205bdc0-21f8-4d37-b682-c5d41a43610f",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from sklearn.model_selection import train_test_split, GridSearchCV\n",
|
|
"from sklearn.naive_bayes import MultinomialNB\n",
|
|
"from sklearn.pipeline import Pipeline\n",
|
|
"\n",
|
|
"# Build the pipeline by combining vectorization and classification\n",
|
|
"pipeline = Pipeline([\n",
|
|
" (\"vectorizer\", vectorizer),\n",
|
|
" (\"classifier\", MultinomialNB())\n",
|
|
"])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "ed9b3536-d2cd-4d66-8a4c-71c25f37711f",
|
|
"metadata": {},
|
|
"source": [
|
|
"GridSearchCV systematically searches through specified hyperparameter values to identify the configuration that produces the best performance\n",
|
|
"\n",
|
|
"MultinomialNB focuses on the alpha parameter, a smoothing factor that adjusts how the model handles unseen words and prevents probabilities from being zero\n",
|
|
"\n",
|
|
"The configuration is chosen by maximizing the f1 score"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 23,
|
|
"id": "2399098c-171f-400f-af2a-d47dc4af5dea",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Best model parameters: {'classifier__alpha': 0.25}\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# Define the parameter grid for hyperparameter tuning\n",
|
|
"param_grid = {\n",
|
|
" \"classifier__alpha\": [0.01, 0.1, 0.15, 0.2, 0.25, 0.5, 0.75, 1.0]\n",
|
|
"}\n",
|
|
"\n",
|
|
"# Perform the grid search with 5-fold cross-validation and the F1-score as metric\n",
|
|
"grid_search = GridSearchCV(\n",
|
|
" pipeline,\n",
|
|
" param_grid,\n",
|
|
" cv=5,\n",
|
|
" scoring=\"f1\"\n",
|
|
")\n",
|
|
"\n",
|
|
"# Fit the grid search on the full dataset\n",
|
|
"grid_search.fit(df[\"message\"], y)\n",
|
|
"\n",
|
|
"# Extract the best model identified by the grid search\n",
|
|
"best_model = grid_search.best_estimator_\n",
|
|
"print(\"Best model parameters:\", grid_search.best_params_)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "ef8dfd83-8f29-4089-b096-580e4e8bb778",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Testing the model\n",
|
|
"- create a new message\n",
|
|
"- preprocess the message"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 24,
|
|
"id": "fc0b16c6-4197-4554-86c2-54ae781ae4a8",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Example SMS messages for evaluation\n",
|
|
"new_messages = [\n",
|
|
" \"Congratulations! You've won a $1000 Walmart gift card. Go to http://bit.ly/1234 to claim now.\",\n",
|
|
" \"Hey, are we still meeting up for lunch today?\",\n",
|
|
" \"Urgent! Your account has been compromised. Verify your details here: www.fakebank.com/verify\",\n",
|
|
" \"Reminder: Your appointment is scheduled for tomorrow at 10am.\",\n",
|
|
" \"FREE entry in a weekly competition to win an iPad. Just text WIN to 80085 now!\",\n",
|
|
"]\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 25,
|
|
"id": "3f720aa9-1ea7-4eb2-8afa-20895fb1f18f",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import numpy as np\n",
|
|
"import re\n",
|
|
"\n",
|
|
"# Preprocess function that mirrors the training-time preprocessing\n",
|
|
"def preprocess_message(message):\n",
|
|
" message = message.lower()\n",
|
|
" message = re.sub(r\"[^a-z\\s$!]\", \"\", message)\n",
|
|
" tokens = word_tokenize(message)\n",
|
|
" tokens = [word for word in tokens if word not in stop_words]\n",
|
|
" tokens = [stemmer.stem(word) for word in tokens]\n",
|
|
" return \" \".join(tokens)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 26,
|
|
"id": "f81b6d54-3217-43f4-b0f7-174669631cef",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Preprocess and vectorize messages\n",
|
|
"processed_messages = [preprocess_message(msg) for msg in new_messages]\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 27,
|
|
"id": "2c2f8a99-274f-44fc-b66c-bf4e5f8d3985",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Transform preprocessed messages into feature vectors\n",
|
|
"X_new = best_model.named_steps[\"vectorizer\"].transform(processed_messages)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "d6f0d757-ee92-4985-b96c-01d1afbc7a62",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Mak predictions\n",
|
|
"- Feed the model the new message\n",
|
|
"- View the output"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 28,
|
|
"id": "d900f2af-f17a-417b-b6b1-092f4125b4ef",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Transform preprocessed messages into feature vectors\n",
|
|
"X_new = best_model.named_steps[\"vectorizer\"].transform(processed_messages)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 29,
|
|
"id": "7c5bc013-4695-47f1-a12c-08abc33aa54d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Predict with the trained classifier\n",
|
|
"predictions = best_model.named_steps[\"classifier\"].predict(X_new)\n",
|
|
"prediction_probabilities = best_model.named_steps[\"classifier\"].predict_proba(X_new)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 30,
|
|
"id": "79186e20-428d-4d92-8eeb-bb932788ef29",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Message: Congratulations! You've won a $1000 Walmart gift card. Go to http://bit.ly/1234 to claim now.\n",
|
|
"Prediction: Spam\n",
|
|
"Spam Probability: 1.00\n",
|
|
"Not-Spam Probability: 0.00\n",
|
|
"--------------------------------------------------\n",
|
|
"Message: Hey, are we still meeting up for lunch today?\n",
|
|
"Prediction: Not-Spam\n",
|
|
"Spam Probability: 0.00\n",
|
|
"Not-Spam Probability: 1.00\n",
|
|
"--------------------------------------------------\n",
|
|
"Message: Urgent! Your account has been compromised. Verify your details here: www.fakebank.com/verify\n",
|
|
"Prediction: Spam\n",
|
|
"Spam Probability: 0.96\n",
|
|
"Not-Spam Probability: 0.04\n",
|
|
"--------------------------------------------------\n",
|
|
"Message: Reminder: Your appointment is scheduled for tomorrow at 10am.\n",
|
|
"Prediction: Not-Spam\n",
|
|
"Spam Probability: 0.00\n",
|
|
"Not-Spam Probability: 1.00\n",
|
|
"--------------------------------------------------\n",
|
|
"Message: FREE entry in a weekly competition to win an iPad. Just text WIN to 80085 now!\n",
|
|
"Prediction: Spam\n",
|
|
"Spam Probability: 1.00\n",
|
|
"Not-Spam Probability: 0.00\n",
|
|
"--------------------------------------------------\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# Display predictions and probabilities for each evaluated message\n",
|
|
"for i, msg in enumerate(new_messages):\n",
|
|
" prediction = \"Spam\" if predictions[i] == 1 else \"Not-Spam\"\n",
|
|
" spam_probability = prediction_probabilities[i][1] # Probability of being spam\n",
|
|
" ham_probability = prediction_probabilities[i][0] # Probability of being not spam\n",
|
|
" \n",
|
|
" print(f\"Message: {msg}\")\n",
|
|
" print(f\"Prediction: {prediction}\")\n",
|
|
" print(f\"Spam Probability: {spam_probability:.2f}\")\n",
|
|
" print(f\"Not-Spam Probability: {ham_probability:.2f}\")\n",
|
|
" print(\"-\" * 50)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "8c42c4be-d64e-4533-959c-b3b14e57b6b8",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Saving the model with joblib"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 31,
|
|
"id": "74bf38e4-020d-400c-b581-b34fafb0737d",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Model saved to spam_detection_model.joblib\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import joblib\n",
|
|
"\n",
|
|
"# Save the trained model to a file for future use\n",
|
|
"model_filename = 'spam_detection_model.joblib'\n",
|
|
"joblib.dump(best_model, model_filename)\n",
|
|
"\n",
|
|
"print(f\"Model saved to {model_filename}\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 32,
|
|
"id": "358bc489-7758-4ac9-8b2e-cbb8888dfb53",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Load the saved model\n",
|
|
"loaded_model = joblib.load(model_filename)\n",
|
|
"\n",
|
|
"# Preprocess new messages before prediction\n",
|
|
"new_data_processed = [preprocess_message(msg) for msg in new_messages]\n",
|
|
"\n",
|
|
"# Make predictions on the preprocessed data\n",
|
|
"predictions = loaded_model.predict(new_data_processed)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "dad19530-da3c-4dbd-80de-8a5d840db7f2",
|
|
"metadata": {},
|
|
"source": [
|
|
"# HTB test the model\n",
|
|
"first run\n",
|
|
"\n",
|
|
"`sudo nix-shell -p openvpn --run \"openvpn --config /tmp/academy-regular.ovpn\"`"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 34,
|
|
"id": "b7be2e9a-d731-4351-a130-d7deb19af6c3",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"ename": "ConnectionError",
|
|
"evalue": "HTTPConnectionPool(host='localhost', port=8000): Max retries exceeded with url: /api/upload (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fcadf6b8c10>: Failed to establish a new connection: [Errno 111] Connection refused'))",
|
|
"output_type": "error",
|
|
"traceback": [
|
|
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
|
|
"\u001b[31mConnectionRefusedError\u001b[39m Traceback (most recent call last)",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/connection.py:198\u001b[39m, in \u001b[36mHTTPConnection._new_conn\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 197\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m198\u001b[39m sock = \u001b[43mconnection\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcreate_connection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 199\u001b[39m \u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_dns_host\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mport\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 200\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 201\u001b[39m \u001b[43m \u001b[49m\u001b[43msource_address\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msource_address\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 202\u001b[39m \u001b[43m \u001b[49m\u001b[43msocket_options\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msocket_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 203\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 204\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m socket.gaierror \u001b[38;5;28;01mas\u001b[39;00m e:\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/util/connection.py:85\u001b[39m, in \u001b[36mcreate_connection\u001b[39m\u001b[34m(address, timeout, source_address, socket_options)\u001b[39m\n\u001b[32m 84\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m85\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[32m 86\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m 87\u001b[39m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/util/connection.py:73\u001b[39m, in \u001b[36mcreate_connection\u001b[39m\u001b[34m(address, timeout, source_address, socket_options)\u001b[39m\n\u001b[32m 72\u001b[39m sock.bind(source_address)\n\u001b[32m---> \u001b[39m\u001b[32m73\u001b[39m \u001b[43msock\u001b[49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43msa\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 74\u001b[39m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n",
|
|
"\u001b[31mConnectionRefusedError\u001b[39m: [Errno 111] Connection refused",
|
|
"\nThe above exception was the direct cause of the following exception:\n",
|
|
"\u001b[31mNewConnectionError\u001b[39m Traceback (most recent call last)",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/connectionpool.py:787\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 786\u001b[39m \u001b[38;5;66;03m# Make the request on the HTTPConnection object\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m787\u001b[39m response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 788\u001b[39m \u001b[43m \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 789\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 790\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 791\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 792\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 793\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 794\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 795\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 796\u001b[39m \u001b[43m \u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 797\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 798\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 799\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 800\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 802\u001b[39m \u001b[38;5;66;03m# Everything went great!\u001b[39;00m\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/connectionpool.py:493\u001b[39m, in \u001b[36mHTTPConnectionPool._make_request\u001b[39m\u001b[34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 492\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m493\u001b[39m \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 494\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 495\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 496\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 497\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 498\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 499\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 500\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 501\u001b[39m \u001b[43m \u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[43m=\u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 502\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 504\u001b[39m \u001b[38;5;66;03m# We are swallowing BrokenPipeError (errno.EPIPE) since the server is\u001b[39;00m\n\u001b[32m 505\u001b[39m \u001b[38;5;66;03m# legitimately able to close the connection after sending a valid response.\u001b[39;00m\n\u001b[32m 506\u001b[39m \u001b[38;5;66;03m# With this behaviour, the received response is still readable.\u001b[39;00m\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/connection.py:494\u001b[39m, in \u001b[36mHTTPConnection.request\u001b[39m\u001b[34m(self, method, url, body, headers, chunked, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 493\u001b[39m \u001b[38;5;28mself\u001b[39m.putheader(header, value)\n\u001b[32m--> \u001b[39m\u001b[32m494\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mendheaders\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 496\u001b[39m \u001b[38;5;66;03m# If we're given a body we start sending that in chunks.\u001b[39;00m\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/http/client.py:1281\u001b[39m, in \u001b[36mHTTPConnection.endheaders\u001b[39m\u001b[34m(self, message_body, encode_chunked)\u001b[39m\n\u001b[32m 1280\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m CannotSendHeader()\n\u001b[32m-> \u001b[39m\u001b[32m1281\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_send_output\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/http/client.py:1041\u001b[39m, in \u001b[36mHTTPConnection._send_output\u001b[39m\u001b[34m(self, message_body, encode_chunked)\u001b[39m\n\u001b[32m 1040\u001b[39m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m._buffer[:]\n\u001b[32m-> \u001b[39m\u001b[32m1041\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1043\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m message_body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 1044\u001b[39m \n\u001b[32m 1045\u001b[39m \u001b[38;5;66;03m# create a consistent interface to message_body\u001b[39;00m\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/http/client.py:979\u001b[39m, in \u001b[36mHTTPConnection.send\u001b[39m\u001b[34m(self, data)\u001b[39m\n\u001b[32m 978\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.auto_open:\n\u001b[32m--> \u001b[39m\u001b[32m979\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 980\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/connection.py:325\u001b[39m, in \u001b[36mHTTPConnection.connect\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 324\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mconnect\u001b[39m(\u001b[38;5;28mself\u001b[39m) -> \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m325\u001b[39m \u001b[38;5;28mself\u001b[39m.sock = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_new_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 326\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._tunnel_host:\n\u001b[32m 327\u001b[39m \u001b[38;5;66;03m# If we're tunneling it means we're connected to our proxy.\u001b[39;00m\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/connection.py:213\u001b[39m, in \u001b[36mHTTPConnection._new_conn\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 212\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m--> \u001b[39m\u001b[32m213\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m NewConnectionError(\n\u001b[32m 214\u001b[39m \u001b[38;5;28mself\u001b[39m, \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mFailed to establish a new connection: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m 215\u001b[39m ) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01me\u001b[39;00m\n\u001b[32m 217\u001b[39m sys.audit(\u001b[33m\"\u001b[39m\u001b[33mhttp.client.connect\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28mself\u001b[39m, \u001b[38;5;28mself\u001b[39m.host, \u001b[38;5;28mself\u001b[39m.port)\n",
|
|
"\u001b[31mNewConnectionError\u001b[39m: <urllib3.connection.HTTPConnection object at 0x7fcadf6b8c10>: Failed to establish a new connection: [Errno 111] Connection refused",
|
|
"\nThe above exception was the direct cause of the following exception:\n",
|
|
"\u001b[31mMaxRetryError\u001b[39m Traceback (most recent call last)",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/requests/adapters.py:644\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 643\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m644\u001b[39m resp = \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 645\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 646\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 647\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 648\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 649\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 650\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 651\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 652\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 653\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 654\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 655\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 656\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 658\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/connectionpool.py:841\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 839\u001b[39m new_e = ProtocolError(\u001b[33m\"\u001b[39m\u001b[33mConnection aborted.\u001b[39m\u001b[33m\"\u001b[39m, new_e)\n\u001b[32m--> \u001b[39m\u001b[32m841\u001b[39m retries = \u001b[43mretries\u001b[49m\u001b[43m.\u001b[49m\u001b[43mincrement\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 842\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnew_e\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_pool\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[43m=\u001b[49m\u001b[43msys\u001b[49m\u001b[43m.\u001b[49m\u001b[43mexc_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[32;43m2\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[32m 843\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 844\u001b[39m retries.sleep()\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/urllib3/util/retry.py:519\u001b[39m, in \u001b[36mRetry.increment\u001b[39m\u001b[34m(self, method, url, response, error, _pool, _stacktrace)\u001b[39m\n\u001b[32m 518\u001b[39m reason = error \u001b[38;5;129;01mor\u001b[39;00m ResponseError(cause)\n\u001b[32m--> \u001b[39m\u001b[32m519\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m MaxRetryError(_pool, url, reason) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mreason\u001b[39;00m \u001b[38;5;66;03m# type: ignore[arg-type]\u001b[39;00m\n\u001b[32m 521\u001b[39m log.debug(\u001b[33m\"\u001b[39m\u001b[33mIncremented Retry for (url=\u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m): \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m\"\u001b[39m, url, new_retry)\n",
|
|
"\u001b[31mMaxRetryError\u001b[39m: HTTPConnectionPool(host='localhost', port=8000): Max retries exceeded with url: /api/upload (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fcadf6b8c10>: Failed to establish a new connection: [Errno 111] Connection refused'))",
|
|
"\nDuring handling of the above exception, another exception occurred:\n",
|
|
"\u001b[31mConnectionError\u001b[39m Traceback (most recent call last)",
|
|
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[34]\u001b[39m\u001b[32m, line 13\u001b[39m\n\u001b[32m 11\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(model_file_path, \u001b[33m\"\u001b[39m\u001b[33mrb\u001b[39m\u001b[33m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m model_file:\n\u001b[32m 12\u001b[39m files = {\u001b[33m\"\u001b[39m\u001b[33mmodel\u001b[39m\u001b[33m\"\u001b[39m: model_file}\n\u001b[32m---> \u001b[39m\u001b[32m13\u001b[39m response = \u001b[43mrequests\u001b[49m\u001b[43m.\u001b[49m\u001b[43mpost\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfiles\u001b[49m\u001b[43m=\u001b[49m\u001b[43mfiles\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 15\u001b[39m \u001b[38;5;66;03m# Pretty print the response from the server\u001b[39;00m\n\u001b[32m 16\u001b[39m \u001b[38;5;28mprint\u001b[39m(json.dumps(response.json(), indent=\u001b[32m4\u001b[39m))\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/requests/api.py:115\u001b[39m, in \u001b[36mpost\u001b[39m\u001b[34m(url, data, json, **kwargs)\u001b[39m\n\u001b[32m 103\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mpost\u001b[39m(url, data=\u001b[38;5;28;01mNone\u001b[39;00m, json=\u001b[38;5;28;01mNone\u001b[39;00m, **kwargs):\n\u001b[32m 104\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33mr\u001b[39m\u001b[33;03m\"\"\"Sends a POST request.\u001b[39;00m\n\u001b[32m 105\u001b[39m \n\u001b[32m 106\u001b[39m \u001b[33;03m :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 112\u001b[39m \u001b[33;03m :rtype: requests.Response\u001b[39;00m\n\u001b[32m 113\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m115\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mpost\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[43m=\u001b[49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/requests/api.py:59\u001b[39m, in \u001b[36mrequest\u001b[39m\u001b[34m(method, url, **kwargs)\u001b[39m\n\u001b[32m 55\u001b[39m \u001b[38;5;66;03m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[32m 56\u001b[39m \u001b[38;5;66;03m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[32m 57\u001b[39m \u001b[38;5;66;03m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[32m 58\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m sessions.Session() \u001b[38;5;28;01mas\u001b[39;00m session:\n\u001b[32m---> \u001b[39m\u001b[32m59\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msession\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/requests/sessions.py:589\u001b[39m, in \u001b[36mSession.request\u001b[39m\u001b[34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[39m\n\u001b[32m 584\u001b[39m send_kwargs = {\n\u001b[32m 585\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mtimeout\u001b[39m\u001b[33m\"\u001b[39m: timeout,\n\u001b[32m 586\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mallow_redirects\u001b[39m\u001b[33m\"\u001b[39m: allow_redirects,\n\u001b[32m 587\u001b[39m }\n\u001b[32m 588\u001b[39m send_kwargs.update(settings)\n\u001b[32m--> \u001b[39m\u001b[32m589\u001b[39m resp = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 591\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/requests/sessions.py:703\u001b[39m, in \u001b[36mSession.send\u001b[39m\u001b[34m(self, request, **kwargs)\u001b[39m\n\u001b[32m 700\u001b[39m start = preferred_clock()\n\u001b[32m 702\u001b[39m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m703\u001b[39m r = \u001b[43madapter\u001b[49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 705\u001b[39m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[32m 706\u001b[39m elapsed = preferred_clock() - start\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/ai/lib/python3.11/site-packages/requests/adapters.py:677\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 673\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e.reason, _SSLError):\n\u001b[32m 674\u001b[39m \u001b[38;5;66;03m# This branch is for urllib3 v1.22 and later.\u001b[39;00m\n\u001b[32m 675\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m SSLError(e, request=request)\n\u001b[32m--> \u001b[39m\u001b[32m677\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request=request)\n\u001b[32m 679\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m ClosedPoolError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 680\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request=request)\n",
|
|
"\u001b[31mConnectionError\u001b[39m: HTTPConnectionPool(host='localhost', port=8000): Max retries exceeded with url: /api/upload (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fcadf6b8c10>: Failed to establish a new connection: [Errno 111] Connection refused'))"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import requests\n",
|
|
"import json\n",
|
|
"\n",
|
|
"# Define the URL of the API endpoint\n",
|
|
"url = \"http://localhost:8000/api/upload\"\n",
|
|
"\n",
|
|
"# Path to the model file you want to upload\n",
|
|
"model_file_path = \"spam_detection_model.joblib\"\n",
|
|
"\n",
|
|
"# Open the file in binary mode and send the POST request\n",
|
|
"with open(model_file_path, \"rb\") as model_file:\n",
|
|
" files = {\"model\": model_file}\n",
|
|
" response = requests.post(url, files=files)\n",
|
|
"\n",
|
|
"# Pretty print the response from the server\n",
|
|
"print(json.dumps(response.json(), indent=4))\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d314b75e-595d-4c8a-9cb2-f81582a50bc0",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.11.6"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|