{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6844d5b3-e159-426f-91ac-2c08064d513a",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import random\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from keras.layers import Input, Embedding, Activation, Flatten, Dense\n",
    "from keras.layers import Conv1D, MaxPooling1D, Dropout\n",
    "from keras.models import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "342ae3b1-6179-4502-a5ca-9f6f3ade65ea",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "random.seed(123)\n",
    "\n",
    "positive = []\n",
    "negative = []\n",
    "\n",
    "CHARS = []\n",
    "\n",
    "INPUT_FILES = [\"keysmash.txt\", \"keysmash2.txt\"]\n",
    "\n",
    "for fname in INPUT_FILES:\n",
    "    with open(fname) as f:\n",
    "        for line in f:\n",
    "            line = line.strip().lower()\n",
    "            if not line:\n",
    "                continue\n",
    "\n",
    "            positive.append(line)\n",
    "\n",
    "            for c in line:\n",
    "                CHARS.append(c)\n",
    "\n",
    "def gen_random_seq(length):\n",
    "    \n",
    "    s = \"\"\n",
    "    \n",
    "    for c in range(length):\n",
    "        s += random.choice(CHARS)\n",
    "    return s\n",
    "    \n",
    "def gen_uniform_random_seq(length):\n",
    "    s = \"\"\n",
    "    \n",
    "    for c in range(length):\n",
    "        s += random.choice(list(set(CHARS)))\n",
    "    return s\n",
    "\n",
    "        \n",
    "for line in positive:\n",
    "    negative.append(gen_random_seq(len(line)))\n",
    "            \n",
    "for line in positive[:len(positive) // 2]:\n",
    "    negative.append(gen_uniform_random_seq(len(line)))\n",
    "\n",
    "for c in set(CHARS):\n",
    "    s = c * 32\n",
    "    negative.append(s)\n",
    "\n",
    "df = pd.DataFrame()\n",
    "df[\"text\"] = positive + negative\n",
    "df[\"label\"] = [\"BOTTOM_KEY_SMASH\" for _ in positive] + [\"RANDOM\" for _ in negative]\n",
    "\n",
    "df_train, df_test = train_test_split(df, test_size=0.2, random_state=123)\n",
    "df_train.to_csv(\"train.csv\", index=False)\n",
    "df_test.to_csv(\"test.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "07a26eab-ff48-45ed-a127-c24ee040f2be",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "MAX_LEN = 96"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ffba18cc-ffa3-4714-81df-39da13b49bf3",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(\"train.csv\")\n",
    "df_test = pd.read_csv(\"test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "af8278f4-854c-4f23-a86b-bfd23fbc2d06",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "train_texts = df_train[\"text\"]\n",
    "test_texts = df_test[\"text\"]\n",
    "\n",
    "tk = Tokenizer(num_words=None, char_level=True, oov_token='UNK')\n",
    "tk.fit_on_texts(train_texts)\n",
    "\n",
    "train_sequences = tk.texts_to_sequences(train_texts)\n",
    "test_texts = tk.texts_to_sequences(test_texts)\n",
    "\n",
    "# Padding\n",
    "train_data = pad_sequences(train_sequences, maxlen=MAX_LEN, padding='post')\n",
    "test_data = pad_sequences(test_texts, maxlen=MAX_LEN, padding='post')\n",
    "\n",
    "# Convert to numpy array\n",
    "train_data = np.array(train_data, dtype='float32')\n",
    "test_data = np.array(test_data, dtype='float32')\n",
    "\n",
    "train_classes = [1 if l == \"BOTTOM_KEY_SMASH\" else 0 for l in df_train[\"label\"].values]\n",
    "test_classes = [1 if l == \"BOTTOM_KEY_SMASH\" else 0 for l in df_test[\"label\"].values]\n",
    "\n",
    "train_classes = to_categorical(train_classes)\n",
    "test_classes = to_categorical(test_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b23272cc-c47f-4508-910b-1a0591724cb1",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(146, 96)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "dc886c87-36fd-4980-94f1-daae285503d5",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "VOCAB_SIZE = len(tk.word_index)\n",
    "\n",
    "num_of_classes = 2\n",
    "optimizer = 'adam'\n",
    "loss = 'categorical_crossentropy'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "eb7be53f-eddc-4c29-a8ae-dcf79dd1e6f2",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def train_model(conv_layers, fully_connected_layers, dropout_p, epochs=10):\n",
    "    embedding_weights = []\n",
    "    embedding_weights.append(np.zeros(VOCAB_SIZE))\n",
    "\n",
    "    for char, i in tk.word_index.items():\n",
    "        onehot = np.zeros(VOCAB_SIZE)\n",
    "        onehot[i - 1] = 1\n",
    "        embedding_weights.append(onehot)\n",
    "\n",
    "    embedding_weights = np.array(embedding_weights)\n",
    "\n",
    "    embedding_layer = Embedding(VOCAB_SIZE + 1,\n",
    "                                VOCAB_SIZE,\n",
    "                                input_length=MAX_LEN,\n",
    "                                weights=[embedding_weights])\n",
    "\n",
    "    inputs = Input(shape=(MAX_LEN,), name='input', dtype='int32')\n",
    "\n",
    "    x = embedding_layer(inputs)\n",
    "\n",
    "    for filter_num, filter_size, pooling_size in conv_layers:\n",
    "        x = Conv1D(filter_num, filter_size, padding='same')(x)\n",
    "        x = Activation('relu')(x)\n",
    "        if pooling_size != -1:\n",
    "            x = MaxPooling1D(pool_size=pooling_size)(x)\n",
    "    x = Flatten()(x)\n",
    "\n",
    "    for dense_size in fully_connected_layers:\n",
    "        x = Dense(dense_size, activation='relu')(x)\n",
    "        x = Dropout(dropout_p)(x)\n",
    "\n",
    "    predictions = Dense(num_of_classes, activation='softmax')(x)\n",
    "\n",
    "    model = Model(inputs=inputs, outputs=predictions)\n",
    "    model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])  # Adam, categorical_crossentropy\n",
    "\n",
    "    indices = np.arange(train_data.shape[0])\n",
    "\n",
    "    x_train = train_data[indices]\n",
    "    y_train = train_classes[indices]\n",
    "\n",
    "    x_test = test_data\n",
    "    y_test = test_classes\n",
    "\n",
    "    hist = model.fit(x_train, y_train,\n",
    "              validation_data=(x_test, y_test),\n",
    "              batch_size=64,\n",
    "              epochs=epochs,\n",
    "              verbose=0)\n",
    "\n",
    "    return hist, model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2be62f40-7b5e-4732-88d3-36db0fa37ec3",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import ParameterGrid\n",
    "from itertools import combinations\n",
    "from tqdm import tqdm\n",
    "\n",
    "num_layers = [2,3,4]\n",
    "\n",
    "layer_params = {\n",
    "    \"filter_num\": [128, 256],\n",
    "    \"filter_size\": [3,5,7],\n",
    "    \"pooling_size\": [-1, 3],\n",
    "}\n",
    "\n",
    "fc_params = {\n",
    "    \"num_layers\": [1,2],\n",
    "    \"layer_size\": [64, 128]\n",
    "}\n",
    "\n",
    "layer_combinations = list(ParameterGrid(layer_params))\n",
    "architectures = []\n",
    "for n in num_layers:\n",
    "    for arch in combinations(layer_combinations, n):\n",
    "        architectures.append([[\n",
    "            x[\"filter_num\"], x[\"filter_size\"], x[\"pooling_size\"]\n",
    "        ] for x in arch])\n",
    "\n",
    "def tune():\n",
    "    best_acc = 0\n",
    "    for conv_layers in tqdm(architectures):\n",
    "        for fc_param in ParameterGrid(fc_params):\n",
    "            for dropout_p in [0.25, 0.5]:\n",
    "\n",
    "                fully_connected_layers = [fc_param[\"layer_size\"]] * fc_param[\"num_layers\"]\n",
    "\n",
    "                hist, _ = train_model(conv_layers, fully_connected_layers, dropout_p)\n",
    "                acc = max(hist.history[\"val_accuracy\"])\n",
    "\n",
    "                if acc > best_acc:\n",
    "                    print(f\"conv={conv_layers} fc={fully_connected_layers}, d={dropout_p} ACC={acc}\")\n",
    "                    best_acc = acc\n",
    "#tune()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b20315ed-5ebe-46e3-9e38-3befb7ec7b50",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-03-27 17:11:32.658592: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-03-27 17:11:32.663854: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-03-27 17:11:32.664023: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-03-27 17:11:32.664648: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-03-27 17:11:32.666111: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-03-27 17:11:32.666267: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-03-27 17:11:32.666395: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-03-27 17:11:33.145009: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-03-27 17:11:33.145178: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-03-27 17:11:33.145299: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-03-27 17:11:33.145394: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 6854 MB memory:  -> device: 0, name: NVIDIA GeForce RTX 3080, pci bus id: 0000:0b:00.0, compute capability: 8.6\n",
      "2022-03-27 17:11:33.321888: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n",
      "2022-03-27 17:11:34.354602: I tensorflow/stream_executor/cuda/cuda_dnn.cc:369] Loaded cuDNN version 8201\n",
      "2022-03-27 17:11:35.080134: I tensorflow/core/platform/default/subprocess.cc:304] Start cannot spawn child process: No such file or directory\n",
      "2022-03-27 17:11:35.080998: I tensorflow/core/platform/default/subprocess.cc:304] Start cannot spawn child process: No such file or directory\n",
      "2022-03-27 17:11:35.081014: W tensorflow/stream_executor/gpu/asm_compiler.cc:77] Couldn't get ptxas version string: Internal: Couldn't invoke ptxas --version\n",
      "2022-03-27 17:11:35.081425: I tensorflow/core/platform/default/subprocess.cc:304] Start cannot spawn child process: No such file or directory\n",
      "2022-03-27 17:11:35.081472: W tensorflow/stream_executor/gpu/redzone_allocator.cc:314] Internal: Failed to launch ptxas\n",
      "Relying on driver to perform ptx compilation. \n",
      "Modify $PATH to customize ptxas location.\n",
      "This message will be only logged once.\n",
      "2022-03-27 17:11:35.754249: I tensorflow/stream_executor/cuda/cuda_blas.cc:1760] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.6369863152503967, 0.6369863152503967, 0.698630154132843, 0.7534246444702148, 0.7260273694992065, 0.7739726305007935, 0.767123281955719, 0.767123281955719, 0.7945205569267273, 0.7808219194412231]\n"
     ]
    }
   ],
   "source": [
    "hist, model = train_model([[128, 3, -1], [256, 3, 3]], [64], 0.25)\n",
    "print(hist.history[\"val_accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0aaf3af7-5338-447c-a181-45cdc338b3e1",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "hist, model = train_model([[128, 3, -1], [256, 3, 3]], [64], 0.25, epochs=9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "df5b5ba8-68e1-44ae-8ba9-ff07bea3eafb",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 37616 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow\n",
    "tensorflow.keras.utils.plot_model(model, show_shapes=True, expand_nested=False, show_dtype=True, rankdir=\"LR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f4c48be0-c19c-44ae-a044-70ff15653699",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflowjs as tfjs\n",
    "import json\n",
    "\n",
    "tfjs.converters.save_keras_model(model, \"tfjs_model\")\n",
    "\n",
    "with open(\"tokens.json\", \"w\") as f:\n",
    "    json.dump(tk.word_index, f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}