jjanella/AI-Red-Teaming-CSCD94
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
75faa5c410b60e15a7779d6f47a14b75e9ac015e
AI-Red-Teaming-CSCD94/applications-of-ai-in-infosec/Log dataset.ipynb
T
Jeremy Janella 75faa5c410 added material
2026-05-09 23:21:13 -04:00

1264 lines
44 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "733bc37f-acc9-4726-8ca2-081eb2ade7d4",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"\n",
"# Load the dataset\n",
"data = pd.read_csv(\"./demo_dataset.csv\")\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "92ea4bcb-14de-45a2-ab1f-ccf561cb5ab1",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" log_id source_ip destination_port protocol bytes_transferred \\\n",
"0 10 10.0.0.100 STRING_PORT FTP 4096 \n",
"1 12 172.16.254.100 110 POP3 NEGATIVE \n",
"2 27 172.16.254.200 110 POP3 NON_NUMERIC \n",
"3 1 192.168.1.100 80 HTTP 1024 \n",
"4 2 192.168.1.81 53 TLS 9765 \n",
"\n",
" threat_level \n",
"0 ? \n",
"1 1 \n",
"2 1 \n",
"3 0 \n",
"4 0 \n"
]
}
],
"source": [
"# Display the first few rows of the dataset\n",
"print(data.head())\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "7b45371c-12dc-4118-bcaf-9288ad57adda",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<class 'pandas.core.frame.DataFrame'>\n",
"RangeIndex: 100 entries, 0 to 99\n",
"Data columns (total 6 columns):\n",
" # Column Non-Null Count Dtype \n",
"--- ------ -------------- ----- \n",
" 0 log_id 100 non-null int64 \n",
" 1 source_ip 99 non-null object\n",
" 2 destination_port 99 non-null object\n",
" 3 protocol 100 non-null object\n",
" 4 bytes_transferred 100 non-null object\n",
" 5 threat_level 100 non-null object\n",
"dtypes: int64(1), object(5)\n",
"memory usage: 4.8+ KB\n",
"None\n"
]
}
],
"source": [
"# Get a summary of column data types and non-null counts\n",
"print(data.info())\n"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "ad1c639b-c50d-4234-ad73-9d5ccbd86be0",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"log_id 0\n",
"source_ip 1\n",
"destination_port 1\n",
"protocol 0\n",
"bytes_transferred 0\n",
"threat_level 0\n",
"dtype: int64\n"
]
}
],
"source": [
"print(data.isnull().sum())"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "869d480a-9f0d-4944-bc03-1efd83e851e7",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" log_id source_ip destination_port protocol bytes_transferred \\\n",
"40 41 10.0.0.300 25 SMTP 4096 \n",
"51 52 10.10.10.450 STRING_PORT FTP 4096 \n",
"55 56 NaN 53 DNS 1024 \n",
"57 58 192.168.1.475 NaN UDP 2048 \n",
"63 64 MISSING_IP 53 DNS 1024 \n",
"65 66 192.168.1.600 UNUSED_PORT UDP 2048 \n",
"71 72 MISSING_IP 53 DNS 1024 \n",
"74 75 172.16.1.400 80 HTTP 1024 \n",
"82 83 172.16.1.450 80 HTTP 1024 \n",
"87 88 MISSING_IP 53 DNS 1024 \n",
"88 89 10.10.10.700 443 TLS 512 \n",
"92 93 INVALID_IP 110 POP3 4096 \n",
"93 94 192.168.1.1050 53 DNS NON_NUMERIC \n",
"95 96 MISSING_IP 25 SMTP 4096 \n",
"97 98 192.168.1.1100 UNUSED_PORT UDP 2048 \n",
"\n",
" threat_level \n",
"40 0 \n",
"51 ? \n",
"55 0 \n",
"57 1 \n",
"63 0 \n",
"65 1 \n",
"71 0 \n",
"74 0 \n",
"82 0 \n",
"87 0 \n",
"88 1 \n",
"92 1 \n",
"93 0 \n",
"95 1 \n",
"97 0 \n"
]
}
],
"source": [
"import re\n",
"\n",
"def is_valid_ip(ip):\n",
" pattern = re.compile(r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$')\n",
" return bool(pattern.match(ip))\n",
"\n",
"# Check for invalid IP addresses\n",
"invalid_ips = data[~data['source_ip'].astype(str).apply(is_valid_ip)]\n",
"print(invalid_ips)\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "06ac98c3-3f31-497c-9a69-4eac3d6f8eb5",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" log_id source_ip destination_port protocol bytes_transferred \\\n",
"0 10 10.0.0.100 STRING_PORT FTP 4096 \n",
"34 35 192.168.1.200 STRING_PORT FTP 4096 \n",
"51 52 10.10.10.450 STRING_PORT FTP 4096 \n",
"57 58 192.168.1.475 NaN UDP 2048 \n",
"65 66 192.168.1.600 UNUSED_PORT UDP 2048 \n",
"67 68 10.10.10.77 STRING_PORT FTP 4096 \n",
"78 79 172.16.254.77 999999 HTTP 2048 \n",
"97 98 192.168.1.1100 UNUSED_PORT UDP 2048 \n",
"\n",
" threat_level \n",
"0 ? \n",
"34 ? \n",
"51 ? \n",
"57 1 \n",
"65 1 \n",
"67 ? \n",
"78 1 \n",
"97 0 \n"
]
}
],
"source": [
"def is_valid_port(port):\n",
" try:\n",
" port = int(port)\n",
" return 0 <= port <= 65535\n",
" except ValueError:\n",
" return False\n",
"\n",
"# Check for invalid port numbers\n",
"invalid_ports = data[~data['destination_port'].apply(is_valid_port)]\n",
"print(invalid_ports)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "b073d9f2-4000-4d98-a33d-b8b05202bcd2",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" log_id source_ip destination_port protocol bytes_transferred \\\n",
"30 31 192.168.1.119 443 UNKNOWN 9513 \n",
"80 81 192.168.1.224 25 UNKNOWN 1161 \n",
"\n",
" threat_level \n",
"30 2 \n",
"80 1 \n"
]
}
],
"source": [
"valid_protocols = ['TCP', 'TLS', 'SSH', 'POP3', 'DNS', 'HTTPS', 'SMTP', 'FTP', 'UDP', 'HTTP']\n",
"\n",
"# Check for invalid protocol values\n",
"invalid_protocols = data[~data['protocol'].isin(valid_protocols)]\n",
"print(invalid_protocols)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "fb1e6b90-cb02-4977-bcd2-faeb376a2b85",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" log_id source_ip destination_port protocol bytes_transferred \\\n",
"1 12 172.16.254.100 110 POP3 NEGATIVE \n",
"2 27 172.16.254.200 110 POP3 NON_NUMERIC \n",
"93 94 192.168.1.1050 53 DNS NON_NUMERIC \n",
"\n",
" threat_level \n",
"1 1 \n",
"2 1 \n",
"93 0 \n"
]
}
],
"source": [
"def is_valid_bytes(bytes):\n",
" try:\n",
" bytes = int(bytes)\n",
" return bytes >= 0\n",
" except ValueError:\n",
" return False\n",
"\n",
"# Check for invalid bytes transferred\n",
"invalid_bytes = data[~data['bytes_transferred'].apply(is_valid_bytes)]\n",
"print(invalid_bytes)\n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "e69ced45-c4e1-4380-92fe-3ccaff8ec9d3",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" log_id source_ip destination_port protocol bytes_transferred \\\n",
"0 10 10.0.0.100 STRING_PORT FTP 4096 \n",
"34 35 192.168.1.200 STRING_PORT FTP 4096 \n",
"51 52 10.10.10.450 STRING_PORT FTP 4096 \n",
"67 68 10.10.10.77 STRING_PORT FTP 4096 \n",
"\n",
" threat_level \n",
"0 ? \n",
"34 ? \n",
"51 ? \n",
"67 ? \n"
]
}
],
"source": [
"def is_valid_threat_level(threat_level):\n",
" try:\n",
" threat_level = int(threat_level)\n",
" return 0 <= threat_level <= 2\n",
" except ValueError:\n",
" return False\n",
"\n",
"# Check for invalid threat levels\n",
"invalid_threat_levels = data[~data['threat_level'].apply(is_valid_threat_level)]\n",
"print(invalid_threat_levels)\n"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "3862967e-60a7-407d-adc9-6c8f64b47fb9",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" log_id source_ip destination_port protocol bytes_transferred \\\n",
"count 77.000000 77 77 77 77 \n",
"unique NaN 68 6 9 73 \n",
"top NaN 192.168.1.55 80 HTTP 1024 \n",
"freq NaN 3 22 22 4 \n",
"mean 46.519481 NaN NaN NaN NaN \n",
"std 28.591317 NaN NaN NaN NaN \n",
"min 1.000000 NaN NaN NaN NaN \n",
"25% 22.000000 NaN NaN NaN NaN \n",
"50% 45.000000 NaN NaN NaN NaN \n",
"75% 70.000000 NaN NaN NaN NaN \n",
"max 100.000000 NaN NaN NaN NaN \n",
"\n",
" threat_level \n",
"count 77 \n",
"unique 3 \n",
"top 1 \n",
"freq 26 \n",
"mean NaN \n",
"std NaN \n",
"min NaN \n",
"25% NaN \n",
"50% NaN \n",
"75% NaN \n",
"max NaN \n"
]
}
],
"source": [
"# the ignore errors covers the fact that there might be some overlap between indexes that match other invalid criteria\n",
"data = data.drop(invalid_ips.index, errors='ignore') \n",
"data = data.drop(invalid_ports.index, errors='ignore')\n",
"data = data.drop(invalid_protocols.index, errors='ignore')\n",
"data = data.drop(invalid_bytes.index, errors='ignore')\n",
"data = data.drop(invalid_threat_levels.index, errors='ignore')\n",
"\n",
"print(data.describe(include='all'))\n"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "b11f6477-6c8c-454e-920c-f8e60a7eea2c",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import re\n",
"from ipaddress import ip_address\n",
"\n",
"df = pd.read_csv('demo_dataset.csv')\n",
"\n",
"invalid_ips = ['INVALID_IP', 'MISSING_IP']\n",
"invalid_ports = ['STRING_PORT', 'UNUSED_PORT']\n",
"invalid_bytes = ['NON_NUMERIC', 'NEGATIVE']\n",
"invalid_threat = ['?']\n",
"\n",
"df.replace(invalid_ips + invalid_ports + invalid_bytes + invalid_threat, np.nan, inplace=True)\n",
"\n",
"df['destination_port'] = pd.to_numeric(df['destination_port'], errors='coerce')\n",
"df['bytes_transferred'] = pd.to_numeric(df['bytes_transferred'], errors='coerce')\n",
"df['threat_level'] = pd.to_numeric(df['threat_level'], errors='coerce')\n",
"\n",
"def is_valid_ip(ip):\n",
" pattern = re.compile(r'^((25[0-5]|2[0-4][0-9]|[01]?\\d?\\d)\\.){3}(25[0-5]|2[0-4]\\d|[01]?\\d?\\d)$')\n",
" if pd.isna(ip) or not pattern.match(str(ip)):\n",
" return np.nan\n",
" return ip\n",
"\n",
"df['source_ip'] = df['source_ip'].apply(is_valid_ip)\n"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "92ff28bd-79f3-4932-8bdc-92fd13a0a2c4",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import re\n",
"from ipaddress import ip_address\n",
"\n",
"df = pd.read_csv('demo_dataset.csv')\n",
"\n",
"invalid_ips = ['INVALID_IP', 'MISSING_IP']\n",
"invalid_ports = ['STRING_PORT', 'UNUSED_PORT']\n",
"invalid_bytes = ['NON_NUMERIC', 'NEGATIVE']\n",
"invalid_threat = ['?']\n",
"\n",
"df.replace(invalid_ips + invalid_ports + invalid_bytes + invalid_threat, np.nan, inplace=True)\n",
"\n",
"df['destination_port'] = pd.to_numeric(df['destination_port'], errors='coerce')\n",
"df['bytes_transferred'] = pd.to_numeric(df['bytes_transferred'], errors='coerce')\n",
"df['threat_level'] = pd.to_numeric(df['threat_level'], errors='coerce')\n",
"\n",
"def is_valid_ip(ip):\n",
" pattern = re.compile(r'^((25[0-5]|2[0-4][0-9]|[01]?\\d?\\d)\\.){3}(25[0-5]|2[0-4]\\d|[01]?\\d?\\d)$')\n",
" if pd.isna(ip) or not pattern.match(str(ip)):\n",
" return np.nan\n",
" return ip\n",
"\n",
"df['source_ip'] = df['source_ip'].apply(is_valid_ip)\n"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "284c76b8-f1c7-4e50-85fc-02cbb4a3a39a",
"metadata": {},
"outputs": [],
"source": [
"from sklearn.impute import SimpleImputer\n",
"\n",
"numeric_cols = ['destination_port', 'bytes_transferred', 'threat_level']\n",
"categorical_cols = ['protocol']\n",
"\n",
"num_imputer = SimpleImputer(strategy='median')\n",
"df[numeric_cols] = num_imputer.fit_transform(df[numeric_cols])\n",
"\n",
"cat_imputer = SimpleImputer(strategy='most_frequent')\n",
"df[categorical_cols] = cat_imputer.fit_transform(df[categorical_cols])\n"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "e61b6da5-e951-43d0-ac2b-f2f98db665ad",
"metadata": {},
"outputs": [],
"source": [
"valid_protocols = ['TCP', 'TLS', 'SSH', 'POP3', 'DNS', 'HTTPS', 'SMTP', 'FTP', 'UDP', 'HTTP']\n",
"df.loc[~df['protocol'].isin(valid_protocols), 'protocol'] = df['protocol'].mode()[0]\n",
"\n",
"df['source_ip'] = df['source_ip'].fillna('0.0.0.0')\n",
"df['destination_port'] = df['destination_port'].clip(lower=0, upper=65535)\n"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "b847d087-1841-4abc-85d0-2ce05a3b4400",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" log_id source_ip destination_port protocol bytes_transferred \\\n",
"count 100.000000 100 100.000000 100 100.00000 \n",
"unique NaN 76 NaN 9 NaN \n",
"top NaN 0.0.0.0 NaN HTTP NaN \n",
"freq NaN 15 NaN 27 NaN \n",
"mean 50.500000 NaN 776.860000 NaN 4138.64000 \n",
"std 29.011492 NaN 6542.582099 NaN 2526.40978 \n",
"min 1.000000 NaN 22.000000 NaN 498.00000 \n",
"25% 25.750000 NaN 53.000000 NaN 1693.25000 \n",
"50% 50.500000 NaN 80.000000 NaN 4096.00000 \n",
"75% 75.250000 NaN 110.000000 NaN 5971.75000 \n",
"max 100.000000 NaN 65535.000000 NaN 9765.00000 \n",
"\n",
" threat_level \n",
"count 100.000000 \n",
"unique NaN \n",
"top NaN \n",
"freq NaN \n",
"mean 0.930000 \n",
"std 0.781801 \n",
"min 0.000000 \n",
"25% 0.000000 \n",
"50% 1.000000 \n",
"75% 2.000000 \n",
"max 2.000000 \n"
]
}
],
"source": [
"print(df.describe(include='all'))\n"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "95c1f9ef-adb5-4e0b-9ba7-bf9fbbd24231",
"metadata": {},
"outputs": [],
"source": [
"from sklearn.impute import KNNImputer\n",
"\n",
"knn_imputer = KNNImputer(n_neighbors=5)\n",
"df[numeric_cols] = knn_imputer.fit_transform(df[numeric_cols])\n"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "75cc3f27-dad5-4adc-8284-52ff7938838f",
"metadata": {},
"outputs": [],
"source": [
"from sklearn.impute import SimpleImputer\n",
"\n",
"numeric_cols = ['destination_port', 'bytes_transferred', 'threat_level']\n",
"categorical_cols = ['protocol']\n",
"\n",
"num_imputer = SimpleImputer(strategy='median')\n",
"df[numeric_cols] = num_imputer.fit_transform(df[numeric_cols])\n",
"\n",
"cat_imputer = SimpleImputer(strategy='most_frequent')\n",
"df[categorical_cols] = cat_imputer.fit_transform(df[categorical_cols])\n"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "2dce2f56-a6d7-47a7-803b-a0e6568986be",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import re\n",
"from ipaddress import ip_address\n",
"\n",
"df = pd.read_csv('demo_dataset.csv')\n",
"\n",
"invalid_ips = ['INVALID_IP', 'MISSING_IP']\n",
"invalid_ports = ['STRING_PORT', 'UNUSED_PORT']\n",
"invalid_bytes = ['NON_NUMERIC', 'NEGATIVE']\n",
"invalid_threat = ['?']\n",
"\n",
"df.replace(invalid_ips + invalid_ports + invalid_bytes + invalid_threat, np.nan, inplace=True)\n",
"\n",
"df['destination_port'] = pd.to_numeric(df['destination_port'], errors='coerce')\n",
"df['bytes_transferred'] = pd.to_numeric(df['bytes_transferred'], errors='coerce')\n",
"df['threat_level'] = pd.to_numeric(df['threat_level'], errors='coerce')\n",
"\n",
"def is_valid_ip(ip):\n",
" pattern = re.compile(r'^((25[0-5]|2[0-4][0-9]|[01]?\\d?\\d)\\.){3}(25[0-5]|2[0-4]\\d|[01]?\\d?\\d)$')\n",
" if pd.isna(ip) or not pattern.match(str(ip)):\n",
" return np.nan\n",
" return ip\n",
"\n",
"df['source_ip'] = df['source_ip'].apply(is_valid_ip)\n"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "1e1d987a-031f-4ab1-b861-f09abb8adb91",
"metadata": {},
"outputs": [],
"source": [
"from sklearn.preprocessing import OneHotEncoder\n",
"\n",
"encoder = OneHotEncoder(handle_unknown='ignore', sparse_output=False)\n",
"encoded = encoder.fit_transform(df[['protocol']])\n",
"\n",
"encoded_df = pd.DataFrame(encoded, columns=encoder.get_feature_names_out(['protocol']))\n",
"df = pd.concat([df.drop('protocol', axis=1), encoded_df], axis=1)\n"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "71481df4-7112-4814-bd64-a4b72db66f27",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>log_id</th>\n",
" <th>source_ip</th>\n",
" <th>destination_port</th>\n",
" <th>bytes_transferred</th>\n",
" <th>threat_level</th>\n",
" <th>protocol_DNS</th>\n",
" <th>protocol_FTP</th>\n",
" <th>protocol_HTTP</th>\n",
" <th>protocol_HTTPS</th>\n",
" <th>protocol_POP3</th>\n",
" <th>protocol_SMTP</th>\n",
" <th>protocol_SSH</th>\n",
" <th>protocol_TLS</th>\n",
" <th>protocol_UDP</th>\n",
" <th>protocol_UNKNOWN</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>10</td>\n",
" <td>10.0.0.100</td>\n",
" <td>NaN</td>\n",
" <td>4096.0</td>\n",
" <td>NaN</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>12</td>\n",
" <td>172.16.254.100</td>\n",
" <td>110.0</td>\n",
" <td>NaN</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>27</td>\n",
" <td>172.16.254.200</td>\n",
" <td>110.0</td>\n",
" <td>NaN</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>1</td>\n",
" <td>192.168.1.100</td>\n",
" <td>80.0</td>\n",
" <td>1024.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2</td>\n",
" <td>192.168.1.81</td>\n",
" <td>53.0</td>\n",
" <td>9765.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>95</th>\n",
" <td>96</td>\n",
" <td>NaN</td>\n",
" <td>25.0</td>\n",
" <td>4096.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>96</th>\n",
" <td>97</td>\n",
" <td>192.168.1.60</td>\n",
" <td>80.0</td>\n",
" <td>4351.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>97</th>\n",
" <td>98</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>2048.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>98</th>\n",
" <td>99</td>\n",
" <td>192.168.1.16</td>\n",
" <td>22.0</td>\n",
" <td>9069.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>99</th>\n",
" <td>100</td>\n",
" <td>192.168.1.14</td>\n",
" <td>25.0</td>\n",
" <td>1494.0</td>\n",
" <td>2.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>100 rows × 15 columns</p>\n",
"</div>"
],
"text/plain": [
" log_id source_ip destination_port bytes_transferred threat_level \\\n",
"0 10 10.0.0.100 NaN 4096.0 NaN \n",
"1 12 172.16.254.100 110.0 NaN 1.0 \n",
"2 27 172.16.254.200 110.0 NaN 1.0 \n",
"3 1 192.168.1.100 80.0 1024.0 0.0 \n",
"4 2 192.168.1.81 53.0 9765.0 0.0 \n",
".. ... ... ... ... ... \n",
"95 96 NaN 25.0 4096.0 1.0 \n",
"96 97 192.168.1.60 80.0 4351.0 1.0 \n",
"97 98 NaN NaN 2048.0 0.0 \n",
"98 99 192.168.1.16 22.0 9069.0 0.0 \n",
"99 100 192.168.1.14 25.0 1494.0 2.0 \n",
"\n",
" protocol_DNS protocol_FTP protocol_HTTP protocol_HTTPS protocol_POP3 \\\n",
"0 0.0 1.0 0.0 0.0 0.0 \n",
"1 0.0 0.0 0.0 0.0 1.0 \n",
"2 0.0 0.0 0.0 0.0 1.0 \n",
"3 0.0 0.0 1.0 0.0 0.0 \n",
"4 0.0 0.0 0.0 0.0 0.0 \n",
".. ... ... ... ... ... \n",
"95 0.0 0.0 0.0 0.0 0.0 \n",
"96 0.0 0.0 0.0 1.0 0.0 \n",
"97 0.0 0.0 0.0 0.0 0.0 \n",
"98 0.0 0.0 1.0 0.0 0.0 \n",
"99 0.0 0.0 0.0 0.0 0.0 \n",
"\n",
" protocol_SMTP protocol_SSH protocol_TLS protocol_UDP protocol_UNKNOWN \n",
"0 0.0 0.0 0.0 0.0 0.0 \n",
"1 0.0 0.0 0.0 0.0 0.0 \n",
"2 0.0 0.0 0.0 0.0 0.0 \n",
"3 0.0 0.0 0.0 0.0 0.0 \n",
"4 0.0 0.0 1.0 0.0 0.0 \n",
".. ... ... ... ... ... \n",
"95 1.0 0.0 0.0 0.0 0.0 \n",
"96 0.0 0.0 0.0 0.0 0.0 \n",
"97 0.0 0.0 0.0 1.0 0.0 \n",
"98 0.0 0.0 0.0 0.0 0.0 \n",
"99 1.0 0.0 0.0 0.0 0.0 \n",
"\n",
"[100 rows x 15 columns]"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "91ddeee5-d2be-46bc-afa4-880a5a1cccc4",
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"\n",
"# Apply logarithmic transformation to a skewed feature to reduce its skewness\n",
"df[\"bytes_transferred\"] = np.log1p(df[\"bytes_transferred\"]) # Add 1 to avoid log(0)\n"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "78c5b813-2091-44c5-8425-9d5c69303ea3",
"metadata": {},
"outputs": [],
"source": [
"from sklearn.model_selection import train_test_split\n",
"\n",
"# Separate features (X) and target (y)\n",
"X = df.drop(\"threat_level\", axis=1)\n",
"y = df[\"threat_level\"]\n",
"\n",
"# Initial split: 80% training, 20% testing\n",
"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1337)\n",
"\n",
"# Second split: from the 80% training portion, allocate 60% for final training and 20% for validation\n",
"X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.25, random_state=1337)\n"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "c1ce8e5a-c571-451e-a53c-c9230aac3258",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>log_id</th>\n",
" <th>source_ip</th>\n",
" <th>destination_port</th>\n",
" <th>bytes_transferred</th>\n",
" <th>protocol_DNS</th>\n",
" <th>protocol_FTP</th>\n",
" <th>protocol_HTTP</th>\n",
" <th>protocol_HTTPS</th>\n",
" <th>protocol_POP3</th>\n",
" <th>protocol_SMTP</th>\n",
" <th>protocol_SSH</th>\n",
" <th>protocol_TLS</th>\n",
" <th>protocol_UDP</th>\n",
" <th>protocol_UNKNOWN</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>10</td>\n",
" <td>10.0.0.100</td>\n",
" <td>NaN</td>\n",
" <td>8.318010</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>12</td>\n",
" <td>172.16.254.100</td>\n",
" <td>110.0</td>\n",
" <td>NaN</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>27</td>\n",
" <td>172.16.254.200</td>\n",
" <td>110.0</td>\n",
" <td>NaN</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>1</td>\n",
" <td>192.168.1.100</td>\n",
" <td>80.0</td>\n",
" <td>6.932448</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2</td>\n",
" <td>192.168.1.81</td>\n",
" <td>53.0</td>\n",
" <td>9.186662</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>95</th>\n",
" <td>96</td>\n",
" <td>NaN</td>\n",
" <td>25.0</td>\n",
" <td>8.318010</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>96</th>\n",
" <td>97</td>\n",
" <td>192.168.1.60</td>\n",
" <td>80.0</td>\n",
" <td>8.378391</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>97</th>\n",
" <td>98</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>7.625107</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>98</th>\n",
" <td>99</td>\n",
" <td>192.168.1.16</td>\n",
" <td>22.0</td>\n",
" <td>9.112728</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>99</th>\n",
" <td>100</td>\n",
" <td>192.168.1.14</td>\n",
" <td>25.0</td>\n",
" <td>7.309881</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>100 rows × 14 columns</p>\n",
"</div>"
],
"text/plain": [
" log_id source_ip destination_port bytes_transferred protocol_DNS \\\n",
"0 10 10.0.0.100 NaN 8.318010 0.0 \n",
"1 12 172.16.254.100 110.0 NaN 0.0 \n",
"2 27 172.16.254.200 110.0 NaN 0.0 \n",
"3 1 192.168.1.100 80.0 6.932448 0.0 \n",
"4 2 192.168.1.81 53.0 9.186662 0.0 \n",
".. ... ... ... ... ... \n",
"95 96 NaN 25.0 8.318010 0.0 \n",
"96 97 192.168.1.60 80.0 8.378391 0.0 \n",
"97 98 NaN NaN 7.625107 0.0 \n",
"98 99 192.168.1.16 22.0 9.112728 0.0 \n",
"99 100 192.168.1.14 25.0 7.309881 0.0 \n",
"\n",
" protocol_FTP protocol_HTTP protocol_HTTPS protocol_POP3 protocol_SMTP \\\n",
"0 1.0 0.0 0.0 0.0 0.0 \n",
"1 0.0 0.0 0.0 1.0 0.0 \n",
"2 0.0 0.0 0.0 1.0 0.0 \n",
"3 0.0 1.0 0.0 0.0 0.0 \n",
"4 0.0 0.0 0.0 0.0 0.0 \n",
".. ... ... ... ... ... \n",
"95 0.0 0.0 0.0 0.0 1.0 \n",
"96 0.0 0.0 1.0 0.0 0.0 \n",
"97 0.0 0.0 0.0 0.0 0.0 \n",
"98 0.0 1.0 0.0 0.0 0.0 \n",
"99 0.0 0.0 0.0 0.0 1.0 \n",
"\n",
" protocol_SSH protocol_TLS protocol_UDP protocol_UNKNOWN \n",
"0 0.0 0.0 0.0 0.0 \n",
"1 0.0 0.0 0.0 0.0 \n",
"2 0.0 0.0 0.0 0.0 \n",
"3 0.0 0.0 0.0 0.0 \n",
"4 0.0 1.0 0.0 0.0 \n",
".. ... ... ... ... \n",
"95 0.0 0.0 0.0 0.0 \n",
"96 0.0 0.0 0.0 0.0 \n",
"97 0.0 0.0 1.0 0.0 \n",
"98 0.0 0.0 0.0 0.0 \n",
"99 0.0 0.0 0.0 0.0 \n",
"\n",
"[100 rows x 14 columns]"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"X"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "4a018c42-7e19-48c7-a7fd-50d81ff91d5e",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0 NaN\n",
"1 1.0\n",
"2 1.0\n",
"3 0.0\n",
"4 0.0\n",
" ... \n",
"95 1.0\n",
"96 1.0\n",
"97 0.0\n",
"98 0.0\n",
"99 2.0\n",
"Name: threat_level, Length: 100, dtype: float64"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"y"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "93350c60-be65-4b41-bd59-41e47f1edc44",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in New Issue View Git Blame Copy Permalink