Jeremy Janella
109 lines
2.4 KiB
Plaintext
109 lines
2.4 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "08bf7c87-7357-4c1c-a185-190c2df112e6",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdin",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"> hello im jeremy\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Number of Input Characters: 15\n",
|
|
"Number of Tokens: 5\n",
|
|
"[\n",
|
|
" \"hello\",\n",
|
|
" \"\\u0120im\",\n",
|
|
" \"\\u0120j\",\n",
|
|
" \"ere\",\n",
|
|
" \"my\"\n",
|
|
"]\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"/home/jeremy/.conda/envs/ai/lib/python3.11/site-packages/huggingface_hub/file_download.py:943: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
|
|
" warnings.warn(\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdin",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"> safhawepoiefj\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Number of Input Characters: 13\n",
|
|
"Number of Tokens: 6\n",
|
|
"[\n",
|
|
" \"saf\",\n",
|
|
" \"haw\",\n",
|
|
" \"ep\",\n",
|
|
" \"o\",\n",
|
|
" \"ief\",\n",
|
|
" \"j\"\n",
|
|
"]\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from transformers import AutoTokenizer\n",
|
|
"import json\n",
|
|
"\n",
|
|
"model = 'openai-community/gpt2'\n",
|
|
"\n",
|
|
"while 1:\n",
|
|
"\ttext = input(\"> \")\n",
|
|
"\n",
|
|
"\ttokens = AutoTokenizer.from_pretrained(model).tokenize(text)\n",
|
|
"\tprint(f\"Number of Input Characters: {len(text)}\")\n",
|
|
"\tprint(f\"Number of Tokens: {len(tokens)}\")\n",
|
|
"\tprint(json.dumps(tokens, indent=2))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "9de80fcb-3145-4e9d-8265-ce885b2ed5d4",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.11.6"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|