Upload 3 files

384_bit_comp.ipynb

@@ -0,0 +1,165 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# pip -q install sentencepiece\n",
+    "# pip -q install numpy\n",
+    "# pip -q install sentence_transformers\n",
+    "# pip -q install datasets\n",
+    "import sentencepiece as spm\n",
+    "import numpy as np\n",
+    "from datasets import load_dataset\n",
+    "from collections import Counter\n",
+    "from sentence_transformers import SentenceTransformer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = SentenceTransformer('all-MiniLM-L6-v2')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[\"My favourite food is anything I didn't have to cook myself.\", 'Now if he does off himself, everyone will think hes having a laugh screwing with people instead of actually dead', 'WHY THE FUCK IS BAYLESS ISOING', 'To make her feel threatened', 'Dirty Southern Wankers', \"OmG pEyToN iSn'T gOoD eNoUgH tO hElP uS iN tHe PlAyOfFs! Dumbass Broncos fans circa December 2015.\", 'Yes I heard abt the f bombs! That has to be why. Thanks for your reply:) until then hubby and I will anxiously wait 😝', 'We need more boards and to create a bit more space for [NAME]. Then we’ll be good.', 'Damn youtube and outrage drama is super lucrative for reddit', 'It might be linked to the trust factor of your friend.']\n"
+     ]
+    }
+   ],
+   "source": [
+    "dataset = load_dataset(\"go_emotions\")\n",
+    "texts = dataset[\"train\"][\"text\"]\n",
+    "print(texts[:10])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def binarize(embeddings, sensitivity=0.1):\n",
+    "\treturn np.where(embeddings >= sensitivity, 1, 0)\n",
+    "\n",
+    "def preprocess(strings):\n",
+    "\treturn \"\\n\".join([\"\".join(map(str, s)) for s in processed_string])\n",
+    "\n",
+    "# Obtain sentence embeddings\n",
+    "embeddings = model.encode(texts)\n",
+    "binary_hashes = binarize(embeddings)\n",
+    "binary_string = preprocess(binary_hashes)\n",
+    "print(binary_string[:500])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Save passage to a temporary file\n",
+    "with open(\"passage.txt\", \"w\") as f:\n",
+    "\tf.write(binary_string)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Training options documentation: https://github.com/google/sentencepiece/blob/master/doc/options.md\n",
+    "# Training takes 3 hours to complete on GTX 1650 mobile\n",
+    "spm.SentencePieceTrainer.train(\n",
+    "\tinput='passage.txt',\n",
+    "\tmodel_prefix='384_bit_comp',\n",
+    "\tvocab_size=256 + 3, # To exclude <unk>, </s>, <s>\n",
+    "\tcharacter_coverage=1.00,\n",
+    "\tmax_sentencepiece_length=384,\n",
+    "\tmodel_type='unigram',\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "length: 13\n",
+      "encoded_tokens: ['▁0000000', '0000000000000001000000000000000000000', '00000000001000100', '1000000', '00000000000000000000000000000001000000000000000000000000000000000000000000000000000000', '00000000000000000001000000000000000000000000000000000', '0000000000000000000000000000000001000', '00000000000000000000000100000000000000000', '00000000010', '0000000000000000000000000000000000000100', '00000000000100000000000000000', '00000000010', '00001000']\n",
+      "encoded_ids: 1ab2ed09d7a9617206894e0608\n",
+      "same?: True\n",
+      "count: Counter({'00000000010': 2, '▁0000000': 1, '0000000000000001000000000000000000000': 1, '00000000001000100': 1, '1000000': 1, '00000000000000000000000000000001000000000000000000000000000000000000000000000000000000': 1, '00000000000000000001000000000000000000000000000000000': 1, '0000000000000000000000000000000001000': 1, '00000000000000000000000100000000000000000': 1, '0000000000000000000000000000000000000100': 1, '00000000000100000000000000000': 1, '00001000': 1})\n"
+     ]
+    }
+   ],
+   "source": [
+    "bpe_processor = spm.SentencePieceProcessor(model_file='384_bit_comp.model')\n",
+    "\n",
+    "def encode_id(bit_text):\n",
+    "\tencoded_pieces = bpe_processor.encode_as_pieces(bit_text)\n",
+    "\tencoded_ids = [bpe_processor.piece_to_id(s) - 3 for s in encoded_pieces]\n",
+    "\tassert any([id_ <= 255 for id_ in encoded_ids])\n",
+    "\tstring_ids = \"\".join([format(id_, \"02x\") for id_ in encoded_ids])\n",
+    "\treturn string_ids\n",
+    "\n",
+    "def decode_id(hex_string):\n",
+    "\tu8_array = np.frombuffer(bytes.fromhex(hex_string), dtype='<u1') + 3\n",
+    "\tencoded_tokens = [bpe_processor.id_to_piece(int(id_)) for id_ in u8_array]\n",
+    "\treturn encoded_tokens\n",
+    "\n",
+    "# Encode text\n",
+    "new_sentence = \"000000000000000000000010000000000000000000000000000000100010010000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000100000000000000000000000000100000000000000000000000000000000000000100000000000001000000000000000000000000001000001000\"\n",
+    "encoded_tokens = bpe_processor.encode_as_pieces(new_sentence)\n",
+    "encoded_ids = encode_id(new_sentence)\n",
+    "decoded_tokens = decode_id(encoded_ids)\n",
+    "\n",
+    "print(\"length:\", len(encoded_tokens))\n",
+    "print(\"encoded_tokens:\", encoded_tokens)\n",
+    "print(\"encoded_ids:\", encoded_ids)\n",
+    "print(\"same?:\", encoded_tokens == decoded_tokens)\n",
+    "\n",
+    "count = Counter(encoded_tokens)\n",
+    "print(\"count:\", count)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

384_bit_comp.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88c8692c639e695577c0c9fde7581d27e5fd47d34f9fda3d9ca2b9ef913276ba
+size 249691

384_bit_comp.vocab

@@ -0,0 +1,259 @@
+<unk>	0
+<s>	0
+</s>	0
+▁	-3.52845
+0000000000000000000	-3.65596
+00000000000000000000000	-3.78531
+00000000000000000000000000000	-3.91664
+0000000000000000000000000000000000000	-4.08234
+00000000000000000000000000000000000000000	-4.16796
+00000000010	-4.24139
+0010000	-4.32347
+00001000	-4.33624
+1000000	-4.43932
+000000000000000000000000000000000000000000000000000000000	-4.48283
+000000010000	-4.48902
+00000000000000000000000000000000000000000000000000000	-4.50439
+000000100	-4.60564
+00000000000000001000	-4.63665
+0001000	-4.63744
+100000000	-4.67832
+00000000000000000000000000000000000000000000000000000000000	-4.68082
+0000000000100000	-4.68338
+000000000001	-4.69539
+001000000000	-4.74417
+0000100000000000	-4.74846
+0000000010000000000	-4.75441
+01000000000000000000	-4.76876
+00000000000000000000000000000000000000000000000000000000000000	-4.78921
+0100000000000000	-4.80865
+▁0000000	-4.81039
+0000001000000000	-4.83899
+00000000000000000001	-4.88581
+000001000000	-4.92111
+000000000000000000000000000000000000000000000000000000000000000000	-4.93305
+0000000000001000	-4.93548
+0000000000000000000000000000000000000000000000000000000000000000000	-4.97344
+0000000000000100	-5.00072
+1000000000000	-5.06267
+0000000000000001	-5.06699
+0000000000000000000000000000000000000000000000000000000000000000000000	-5.07257
+00100000000000000000	-5.07809
+00000000000000000000000000000000000000000000000000000000000000000	-5.07911
+000000100000000000000000	-5.12641
+0001000000000000	-5.15307
+0000000000000001000	-5.15563
+000000000000000001	-5.17957
+1000000000000000000000000	-5.18296
+00000000000000000000010	-5.18872
+000001000000000	-5.19212
+010000000000000000	-5.21656
+0000000001000000000	-5.21729
+100000000000000000	-5.22011
+00000000000000000000000000000010	-5.22249
+000000000001000000	-5.24417
+000010000000000000000000000000	-5.25591
+00000000000000000000000000000000000000000000000000000000000000000000000000000000000000	-5.28546
+000000000000001000000000	-5.28713
+▁000000000000	-5.29248
+00000000000000000000010000000	-5.29561
+▁000000100000000	-5.31073
+000000000000001000000	-5.32714
+0001000000000000000	-5.33971
+000010000000000000	-5.34512
+▁0000000000000000000000000000000000000000000	-5.38274
+▁0000000000000000000000	-5.39199
+001000000000000000000000	-5.39265
+▁0000000000000000000000000000000000000000000000000	-5.39844
+00000000000000000000000100	-5.40173
+0000000100000000000000000000000000000	-5.41313
+0000000000000100000	-5.41855
+0100000000000000000000000000000000000	-5.42138
+000001000000000000000000	-5.42176
+0010000000000000000000000000000000000	-5.42316
+00000000100000000000000000000	-5.44607
+000100000000000000000000	-5.45402
+0000000000000000000000000000000000000000000000001	-5.46818
+000000000000000000100	-5.47311
+00000010000000000000	-5.48376
+00000000000000000000000010000	-5.49755
+000000000000000000000000001000	-5.50458
+0000000000000000000001000	-5.51823
+00000000000100000000000000000	-5.53385
+00000000000000000000000001000	-5.53677
+▁000000000000000000000000000000	-5.54138
+010000000000000000000000000000000	-5.54928
+00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000	-5.5737
+00000000000000000010000000000	-5.57618
+000001000000000000000000000000000	-5.57912
+00000000000000010000000000	-5.58209
+00000100000000000000000000	-5.58931
+▁00000000000000000	-5.60556
+000000000010000000000000	-5.61112
+00000000000000000001000000000	-5.61525
+001000000000000000000000000000000	-5.61903
+00000000000000000000001000000000	-5.61955
+00000000010000000000000000000	-5.62483
+00000000000000001000000000000	-5.64492
+00000000000000000000000000000000000010	-5.6477
+10000000000000000000000000000	-5.65699
+00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000	-5.66156
+0000000000000000000000000000000001000	-5.66588
+000000000000000000000000010000	-5.66819
+00000001000000000000000000000	-5.69042
+000000000000000000000000000000001	-5.69428
+000100000000000000000000000000	-5.70215
+1000000000000000000000000000000000000	-5.70416
+10000000000000000000000000000000000000000000000	-5.71607
+000000000000000001000000000	-5.73038
+000000010000000000000000	-5.73373
+01000000000000000000000000000000000000000000000000000000	-5.77151
+10000000000000000000000000000000000000000000000000	-5.77662
+00000000000000000000000000001	-5.77932
+0000001000000000000000000000000000000	-5.79451
+001000000000000000000000000000000000000000000000000000000	-5.7988
+00000010000000000000000000000000000000000	-5.80258
+000000000000000000000000000000000000000000010	-5.80265
+0001000000000000000000000000000000000	-5.81332
+00000000000000000000000100000000000000000	-5.81964
+00001000000000000000000000000000000000000	-5.82563
+0000000000000000000000000000001000000000	-5.83145
+00000000000000000000000000000000000000000000100	-5.84554
+000000100000000000000000000000000000000000000000000000	-5.85007
+00000000100000000000000000000000000000000	-5.86818
+00000000000000000000100000000	-5.89281
+00100000000000000000000000000000000000000000000000	-5.90395
+000000000000100000000000000000000	-5.90702
+000000000000000000010000000000000	-5.91758
+10000000000000000000000000000000000000000	-5.9197
+00000001000000000000000000000000000000000000000000	-5.92525
+00000000000000000000000000000000000000000010000	-5.92852
+0000000000000000000000000000000000000010000000000	-5.93005
+▁0000001	-5.93155
+▁0000000000000000000000000000000000000000000000000000000000000000000000000000	-5.93614
+00000000000000000010000000000000000000000	-5.93977
+110	-5.94529
+1010	-5.94822
+00010000000000000000000000000000000000000000000000000	-5.952
+▁00000000000000000000000000000000000	-5.96303
+00100000000000000000000000000000000000000000000000000000	-5.9747
+00000000000000001000000000000000000000000	-5.98606
+0000000000000000000000000000000000000100	-5.98868
+00000000000000000000000010000000000000000000000000	-5.99035
+000000000000000000001000000000000000000000000000	-5.99488
+00000000001000000000000000000000	-6.0022
+000010000000000000000000000000000000000000000000000000000	-6.01749
+000000000000000000000000000100000	-6.01957
+1001	-6.02025
+100000000000000000000000000000000000000000000	-6.03247
+00000100000000000000000000000000000000000000000000000000	-6.04545
+00000000000000000000000000100000000000000000000	-6.06651
+0000000000010000000000000000000000000000000000000	-6.06685
+0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000	-6.07915
+00000000000000000000000000000000000000001000000000	-6.08651
+00000000000010000000000000000000000000000000000	-6.08735
+00000000000001000000000000000000000000000000000000	-6.09343
+0000000000000000100000000000000000000000000000000000000000	-6.09424
+0000000000000000000000000000000100000000	-6.10272
+010010	-6.10316
+00000000000010000000000000000	-6.11175
+000000000000000100000000000000	-6.12087
+000000000000000000000000000000000000000000000000001000	-6.14209
+10001	-6.14572
+00000100000000000000000000000000000000000000000000	-6.16132
+0000000000100000000000000000000000000000000000000	-6.18804
+000000000000000000000000000000000000000000000000000000000000010	-6.21861
+00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000	-6.23145
+00000000000000000000000000000000000000000000000000000010000	-6.23637
+00001000000000000000000000000000000000000000000000000000	-6.24755
+011000	-6.2593
+000000000000001000000000000000000000000000000000	-6.26038
+00000000000010000000000000000000000000000	-6.27078
+0000000000000000000000000001000000000000000	-6.28252
+00000000000000000001000000000000000000000000000000000	-6.2829
+00000100000000000000000000000000000000000000000000000000000	-6.28362
+00000000000000000000000000000010000000000000000000000	-6.29415
+100000000000000000000000000000000000000000000000000000000000000	-6.30668
+000000000000001000000000000000000000000000000000000000000	-6.31072
+000000000000000000000000000001000000000000000000000000000	-6.3149
+0000000000000000000000001000000000000000000000000	-6.32
+00001000001000	-6.33768
+000000000001000000000000000000000000000000000000000000000000000000	-6.34143
+0000000000000001000000000000000000000	-6.35705
+0000000000001000000000000000000000000000000000000	-6.36022
+0000000011	-6.3682
+0000000000000000001000000000000000000000000000000	-6.36884
+000000000000000000000010000000000000000000000000000000000000000	-6.39045
+0000000000000000000000000000100000000000000	-6.39119
+010000000000000000000000000000000000000000000000000000000000000000000000	-6.43118
+000000000000000010000000000000000000000000000000000000000000000	-6.4329
+00001100	-6.44095
+0000000000000000000000000000000010000000000000000	-6.46427
+00000000000000000000000010000000000000000000000000000000000	-6.46542
+▁000000100000000000000000000000000000000000000000000000000000000000000	-6.48445
+00000000000000000000000001000000000000000000000000000000000000	-6.50259
+00001010	-6.51077
+1000001	-6.51688
+1000010000	-6.51997
+▁010000	-6.52565
+0000000000100000000000000000000000000000000000000000000000000000000000	-6.5289
+0000000000000000000000000000000000000000000000000000000001000000000	-6.57233
+1000000000000000000000000000000000000000000000000000000000000000000	-6.57372
+0001000000000000000000000000000000000000000000000000000000000000000000000000000	-6.59348
+0000000100000000000000000000000000000000000000000000000000000000	-6.60371
+00000001001000	-6.60859
+0101000000000	-6.61863
+0000000000000000000000000000000000000000000000000000000000000000000000001	-6.61869
+10000001000	-6.62899
+00000000000000000000100000000000000000000000000000000000000	-6.63681
+00000000000000000000000000000000000000000000000000010000000000	-6.64363
+000001000000000000000000000000000000000000000000000000000000000000000000	-6.65513
+0100000000000000000000000000000000000000000000000000000000000000000000000000	-6.65559
+0000000000000000000010000000000000000000000000000000000000000000000000000000000	-6.65961
+000000000000000000000000000000100000000000000000000000000	-6.66161
+00110000000000	-6.66161
+100000000000000000000000000000000000000000000000000000000000000000000000000000000000	-6.66332
+0000000000000000000000010000000000000000000000000000000000000000000000	-6.66687
+000000000000000000000000000000000000000000000000000000000001000	-6.67646
+00000001000000010	-6.69296
+00000000000000000000000000000001000000000000000000000000000000000000000000000000000000	-6.69633
+0000001000000001	-6.72363
+0001010000	-6.72549
+▁000000000000000000000000000000010000000000000000000000000000000000000	-6.7322
+000000000000000000000000000000000000000000000000010000000	-6.73234
+000010000000000000000000000000000000000000000000000000000000000000	-6.73386
+00000000001001000	-6.7623
+0000010000000000000000000000000000000000000000000000000000000000000000000000000	-6.79072
+0000000000000000000000000000000000000000100000000000000000000000	-6.79116
+0010100000000000	-6.79184
+00000000000100001	-6.79335
+0000000000000000000000000000001000000000000000000000000000000000000	-6.79407
+0000000000000010100	-6.80074
+01100000000000000000	-6.80084
+000000000000000010000000000000000000000000000000000000000000000000000000000000000000	-6.8132
+00000000000000000000000000100000000000000000000000000000000000000000000000	-6.8195
+000000000000000001000000000000000000000000000000000000000000000000000000	-6.82052
+00001000000000000000000000000000000000000000000000000000000000000000000000000000000000000	-6.84508
+001000000100	-6.8523
+000100100000000000000000	-6.85511
+0100000001000000000	-6.87402
+0000000000000100000000000000000000000000000000000000000000000000000000	-6.88584
+00000000001000100	-6.89382
+▁000010	-6.92121
+00000000000000000000000000000000000000000000000000000000000000000000000000000000000010	-6.92647
+000000000000000000000000000000000000010000000000000000000000000000000000	-6.93266
+▁000000000000000010000000000000000000000000000000	-6.94882
+0100010000000000	-6.95175
+0000011000000000	-6.98915
+0010000000010	-6.99205
+0000000000000101000	-6.9959
+0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000	-7.00683
+0000000000001000000000000000000000000000000000000000000000000000000000000000000	-7.04102
+000000000000100000010000000000	-7.04832
+00000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000	-7.06727
+000001010000000	-7.06969
+00010000000100	-7.0781
+00010001000000000	-7.08985
+01000000001000	-7.09474
+1	-8.4241
+0	-8.4242