test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "lexique = pd.read_csv('Lexique383/Lexique383.tsv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ortho</th>\n",
       "      <th>phon</th>\n",
       "      <th>lemme</th>\n",
       "      <th>cgram</th>\n",
       "      <th>genre</th>\n",
       "      <th>nombre</th>\n",
       "      <th>freqlemfilms2</th>\n",
       "      <th>freqlemlivres</th>\n",
       "      <th>freqfilms2</th>\n",
       "      <th>freqlivres</th>\n",
       "      <th>...</th>\n",
       "      <th>orthrenv</th>\n",
       "      <th>phonrenv</th>\n",
       "      <th>orthosyll</th>\n",
       "      <th>cgramortho</th>\n",
       "      <th>deflem</th>\n",
       "      <th>defobs</th>\n",
       "      <th>old20</th>\n",
       "      <th>pld20</th>\n",
       "      <th>morphoder</th>\n",
       "      <th>nbmorph</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>NOM</td>\n",
       "      <td>m</td>\n",
       "      <td>NaN</td>\n",
       "      <td>81.36</td>\n",
       "      <td>58.65</td>\n",
       "      <td>81.36</td>\n",
       "      <td>58.65</td>\n",
       "      <td>...</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>NOM,AUX,VER</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>a</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>avoir</td>\n",
       "      <td>AUX</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>18559.22</td>\n",
       "      <td>12800.81</td>\n",
       "      <td>6350.91</td>\n",
       "      <td>2926.69</td>\n",
       "      <td>...</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>NOM,AUX,VER</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>avoir</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>avoir</td>\n",
       "      <td>VER</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13572.40</td>\n",
       "      <td>6426.49</td>\n",
       "      <td>5498.34</td>\n",
       "      <td>1669.39</td>\n",
       "      <td>...</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>NOM,AUX,VER</td>\n",
       "      <td>93.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>avoir</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>a capella</td>\n",
       "      <td>akapEla</td>\n",
       "      <td>a capella</td>\n",
       "      <td>ADV</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.07</td>\n",
       "      <td>...</td>\n",
       "      <td>allepac a</td>\n",
       "      <td>alEpaka</td>\n",
       "      <td>a ca-pel-la</td>\n",
       "      <td>ADV</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.85</td>\n",
       "      <td>2.85</td>\n",
       "      <td>a-capella</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a cappella</td>\n",
       "      <td>akapEla</td>\n",
       "      <td>a cappella</td>\n",
       "      <td>ADV</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.07</td>\n",
       "      <td>...</td>\n",
       "      <td>alleppac a</td>\n",
       "      <td>alEpaka</td>\n",
       "      <td>a cap-pel-la</td>\n",
       "      <td>ADV</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.60</td>\n",
       "      <td>2.85</td>\n",
       "      <td>a-cappella</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        ortho     phon       lemme cgram genre nombre  freqlemfilms2  \\\n",
       "0           a        a           a   NOM     m    NaN          81.36   \n",
       "1           a        a       avoir   AUX   NaN    NaN       18559.22   \n",
       "2           a        a       avoir   VER   NaN    NaN       13572.40   \n",
       "3   a capella  akapEla   a capella   ADV   NaN    NaN           0.04   \n",
       "4  a cappella  akapEla  a cappella   ADV   NaN    NaN           0.04   \n",
       "\n",
       "   freqlemlivres  freqfilms2  freqlivres  ...    orthrenv  phonrenv  \\\n",
       "0          58.65       81.36       58.65  ...           a         a   \n",
       "1       12800.81     6350.91     2926.69  ...           a         a   \n",
       "2        6426.49     5498.34     1669.39  ...           a         a   \n",
       "3           0.07        0.04        0.07  ...   allepac a   alEpaka   \n",
       "4           0.07        0.04        0.07  ...  alleppac a   alEpaka   \n",
       "\n",
       "      orthosyll   cgramortho  deflem  defobs old20 pld20   morphoder  nbmorph  \n",
       "0             a  NOM,AUX,VER     NaN     NaN  1.00  1.00           a        1  \n",
       "1             a  NOM,AUX,VER     NaN     NaN  1.00  1.00       avoir        1  \n",
       "2             a  NOM,AUX,VER    93.0    16.0  1.00  1.00       avoir        1  \n",
       "3   a ca-pel-la          ADV     NaN     NaN  3.85  2.85   a-capella        2  \n",
       "4  a cap-pel-la          ADV     NaN     NaN  4.60  2.85  a-cappella        2  \n",
       "\n",
       "[5 rows x 35 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lexique.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 a\n"
     ]
    }
   ],
   "source": [
    "lexiqueClean = {}\n",
    "\n",
    "for i, row in lexique.iterrows():\n",
    "    \n",
    "\n",
    "    print(i, row['ortho'])\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cosine(t1, t2):\n",
    "    return (torch.nn.functional.normalize(t1)*torch.nn.functional.normalize(t2)).sum(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/hoomano/.cache/torch/hub/pytorch_fairseq_main\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unable to build Cython components. Please make sure Cython is installed if the torch.hub model you are loading depends on it.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-29 12:19:47 | INFO | fairseq.file_utils | loading archive file http://dl.fbaipublicfiles.com/fairseq/models/camembert-base.tar.gz from cache at /home/hoomano/.cache/torch/pytorch_fairseq/3a5b985f6506f03df591fc4564446205fffa6bfe1a5d4657775ed20efdf3162b.4f0d00e7c3a06d48868273ae5e32b16aeb9cafa08939dd691352a8bdae6059be\n",
      "2022-06-29 12:19:49 | INFO | fairseq.tasks.masked_lm | dictionary: 32004 types\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RobertaHubInterface(\n",
       "  (model): RobertaModel(\n",
       "    (encoder): RobertaEncoder(\n",
       "      (sentence_encoder): TransformerEncoder(\n",
       "        (dropout_module): FairseqDropout()\n",
       "        (embed_tokens): Embedding(32005, 768, padding_idx=1)\n",
       "        (embed_positions): LearnedPositionalEmbedding(514, 768, padding_idx=1)\n",
       "        (layernorm_embedding): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        (layers): ModuleList(\n",
       "          (0): TransformerEncoderLayerBase(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (dropout_module): FairseqDropout()\n",
       "              (k_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (v_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (q_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout_module): FairseqDropout()\n",
       "            (activation_dropout_module): FairseqDropout()\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (1): TransformerEncoderLayerBase(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (dropout_module): FairseqDropout()\n",
       "              (k_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (v_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (q_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout_module): FairseqDropout()\n",
       "            (activation_dropout_module): FairseqDropout()\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (2): TransformerEncoderLayerBase(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (dropout_module): FairseqDropout()\n",
       "              (k_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (v_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (q_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout_module): FairseqDropout()\n",
       "            (activation_dropout_module): FairseqDropout()\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (3): TransformerEncoderLayerBase(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (dropout_module): FairseqDropout()\n",
       "              (k_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (v_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (q_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout_module): FairseqDropout()\n",
       "            (activation_dropout_module): FairseqDropout()\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (4): TransformerEncoderLayerBase(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (dropout_module): FairseqDropout()\n",
       "              (k_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (v_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (q_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout_module): FairseqDropout()\n",
       "            (activation_dropout_module): FairseqDropout()\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (5): TransformerEncoderLayerBase(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (dropout_module): FairseqDropout()\n",
       "              (k_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (v_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (q_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout_module): FairseqDropout()\n",
       "            (activation_dropout_module): FairseqDropout()\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (6): TransformerEncoderLayerBase(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (dropout_module): FairseqDropout()\n",
       "              (k_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (v_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (q_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout_module): FairseqDropout()\n",
       "            (activation_dropout_module): FairseqDropout()\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (7): TransformerEncoderLayerBase(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (dropout_module): FairseqDropout()\n",
       "              (k_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (v_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (q_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout_module): FairseqDropout()\n",
       "            (activation_dropout_module): FairseqDropout()\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (8): TransformerEncoderLayerBase(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (dropout_module): FairseqDropout()\n",
       "              (k_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (v_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (q_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout_module): FairseqDropout()\n",
       "            (activation_dropout_module): FairseqDropout()\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (9): TransformerEncoderLayerBase(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (dropout_module): FairseqDropout()\n",
       "              (k_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (v_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (q_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout_module): FairseqDropout()\n",
       "            (activation_dropout_module): FairseqDropout()\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (10): TransformerEncoderLayerBase(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (dropout_module): FairseqDropout()\n",
       "              (k_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (v_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (q_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout_module): FairseqDropout()\n",
       "            (activation_dropout_module): FairseqDropout()\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (11): TransformerEncoderLayerBase(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (dropout_module): FairseqDropout()\n",
       "              (k_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (v_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (q_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout_module): FairseqDropout()\n",
       "            (activation_dropout_module): FairseqDropout()\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (lm_head): RobertaLMHead(\n",
       "        (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "    )\n",
       "    (classification_heads): ModuleDict()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "camembert = torch.hub.load('pytorch/fairseq', 'camembert')\n",
    "camembert.eval()  # disable dropout (or leave in train mode to finetune)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "line = \"J'aime le camembert !\"\n",
    "tokens = camembert.encode(line)\n",
    "last_layer_features = camembert.extract_features(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 10, 768])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_layer_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No sentence-transformers model found with name /home/hoomano/.cache/torch/sentence_transformers/dangvantuan_sentence-camembert-large. Creating a new one with MEAN pooling.\n"
     ]
    }
   ],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "model =  SentenceTransformer(\"dangvantuan/sentence-camembert-large\")\n",
    "\n",
    "sentences = \"\"\"Je fais souvent ce rêve étrange et pénétrant\n",
    "Je fais souvent ce rêve d'avoir l'biff de The Rock\n",
    "J'veux l'biff comme The Rock\"\"\".split('\\n')\n",
    "\n",
    "embeddings = model.encode(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.0000, 0.4999, 0.2042],\n",
       "        [0.4999, 1.0000, 0.7509],\n",
       "        [0.2042, 0.7509, 1.0000]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "torchEmb = torch.nn.functional.normalize(torch.from_numpy(embeddings))\n",
    "\n",
    "(torchEmb@torchEmb.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(1.1127)\n",
      "tensor(1.0266)\n",
      "tensor(1.0266)\n"
     ]
    }
   ],
   "source": [
    "targetSentences = [\"Je fais souvent ce rêve étrange et pénétrant\", \"J'veux l'biff comme The Rock\"]\n",
    "targetWeights = torch.Tensor([1., 1.])\n",
    "targetWeights /= targetWeights.sum()\n",
    "\n",
    "\n",
    "def getFitness(sentence):\n",
    "    embeddings = model.encode([sentence] + targetSentences)\n",
    "\n",
    "    norm_embeddings = torch.nn.functional.normalize(torch.from_numpy(embeddings))\n",
    "    return (((norm_embeddings@norm_embeddings.T)[0, 1:] * targetWeights)**0.5).sum()\n",
    "\n",
    "print(getFitness(\"Je fais souvent ce rêve d'avoir l'biff de The Rock\"))\n",
    "print(getFitness(\"Je fais souvent ce rêve étrange et pénétrant\"))\n",
    "print(getFitness(\"J'veux l'biff comme The Rock\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Je fais souvent ce rêve étrange et pénétrant 1.0266478061676025\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "Cannot choose from an empty sequence",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m/home/hoomano/Desktop/Jam/test.ipynb Cell 12'\u001b[0m in \u001b[0;36m<cell line: 98>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/hoomano/Desktop/Jam/test.ipynb#ch0000012?line=94'>95</a>\u001b[0m             \u001b[39mself\u001b[39m\u001b[39m.\u001b[39magents[i]\u001b[39m.\u001b[39mmutate()\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/hoomano/Desktop/Jam/test.ipynb#ch0000012?line=96'>97</a>\u001b[0m gen \u001b[39m=\u001b[39m Genetic()\n\u001b[0;32m---> <a href='vscode-notebook-cell:/home/hoomano/Desktop/Jam/test.ipynb#ch0000012?line=97'>98</a>\u001b[0m gen\u001b[39m.\u001b[39;49mrun()\n",
      "\u001b[1;32m/home/hoomano/Desktop/Jam/test.ipynb Cell 12'\u001b[0m in \u001b[0;36mGenetic.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/hoomano/Desktop/Jam/test.ipynb#ch0000012?line=52'>53</a>\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mselection()\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/hoomano/Desktop/Jam/test.ipynb#ch0000012?line=53'>54</a>\u001b[0m     \u001b[39mprint\u001b[39m(i, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbest\u001b[39m.\u001b[39msentenceToStr(), \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbest\u001b[39m.\u001b[39mfitness\u001b[39m.\u001b[39mitem())\n\u001b[0;32m---> <a href='vscode-notebook-cell:/home/hoomano/Desktop/Jam/test.ipynb#ch0000012?line=54'>55</a>\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmutate()\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/hoomano/Desktop/Jam/test.ipynb#ch0000012?line=55'>56</a>\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39magents[\u001b[39m0\u001b[39m] \u001b[39m=\u001b[39m cp\u001b[39m.\u001b[39mdeepcopy(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbest)\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/hoomano/Desktop/Jam/test.ipynb#ch0000012?line=57'>58</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbest\u001b[39m.\u001b[39msentenceToStr())\n",
      "\u001b[1;32m/home/hoomano/Desktop/Jam/test.ipynb Cell 12'\u001b[0m in \u001b[0;36mGenetic.mutate\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/hoomano/Desktop/Jam/test.ipynb#ch0000012?line=92'>93</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mmutate\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/hoomano/Desktop/Jam/test.ipynb#ch0000012?line=93'>94</a>\u001b[0m     \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39msize):\n\u001b[0;32m---> <a href='vscode-notebook-cell:/home/hoomano/Desktop/Jam/test.ipynb#ch0000012?line=94'>95</a>\u001b[0m         \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49magents[i]\u001b[39m.\u001b[39;49mmutate()\n",
      "\u001b[1;32m/home/hoomano/Desktop/Jam/test.ipynb Cell 12'\u001b[0m in \u001b[0;36mAgent.mutate\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/hoomano/Desktop/Jam/test.ipynb#ch0000012?line=30'>31</a>\u001b[0m index \u001b[39m=\u001b[39m rd\u001b[39m.\u001b[39mrandint(\u001b[39m0\u001b[39m, \u001b[39mlen\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39msentence) \u001b[39m-\u001b[39m \u001b[39m1\u001b[39m)\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/hoomano/Desktop/Jam/test.ipynb#ch0000012?line=31'>32</a>\u001b[0m index \u001b[39m=\u001b[39m rd\u001b[39m.\u001b[39mchoice([\u001b[39m-\u001b[39m\u001b[39m3\u001b[39m, \u001b[39m-\u001b[39m\u001b[39m1\u001b[39m])\n\u001b[0;32m---> <a href='vscode-notebook-cell:/home/hoomano/Desktop/Jam/test.ipynb#ch0000012?line=32'>33</a>\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msentence[index] \u001b[39m=\u001b[39m rd\u001b[39m.\u001b[39;49mchoice(corpus)\n",
      "File \u001b[0;32m/usr/lib/python3.8/random.py:290\u001b[0m, in \u001b[0;36mRandom.choice\u001b[0;34m(self, seq)\u001b[0m\n\u001b[1;32m    288\u001b[0m     i \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_randbelow(\u001b[39mlen\u001b[39m(seq))\n\u001b[1;32m    289\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mValueError\u001b[39;00m:\n\u001b[0;32m--> 290\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mIndexError\u001b[39;00m(\u001b[39m'\u001b[39m\u001b[39mCannot choose from an empty sequence\u001b[39m\u001b[39m'\u001b[39m) \u001b[39mfrom\u001b[39;00m \u001b[39mNone\u001b[39m\n\u001b[1;32m    291\u001b[0m \u001b[39mreturn\u001b[39;00m seq[i]\n",
      "\u001b[0;31mIndexError\u001b[0m: Cannot choose from an empty sequence"
     ]
    }
   ],
   "source": [
    "import random as rd\n",
    "import copy as cp\n",
    "\n",
    "rawCorpus = ' '.join(\"\"\"Freeze Raël, sur la prod, kicke comme Israel\n",
    "Fuck ces nègres comme Israël\n",
    "Chen Laden dans l'complot comme les Ben Laden, Chirak comme JB Binladen\n",
    "J'les vois petit comme si j'étais 100 pieds, on veut les VVS sur les dentiers\n",
    "À propos des sommes comme les banquiers, investisseurs, architectes et chefs de chantier\n",
    "J'veux l'biff comme The Rock\n",
    "20-20, le rap, c'est satanique comme le rock\n",
    "Cali Weed, Promethazine et l'fer comme D-Block Europe\n",
    "Concurrence j'laisse que des mégots\n",
    "J'suis là pour les deux peines de mort pour les pédos\n",
    "J'mixe le lin dans les Faygo, à Dakar, on empile les briques comme des Lego\"\"\".split('\\n'))\n",
    "rawCorpus = \"\"\n",
    "with open(\"./corpus.txt\") as f:\n",
    "    rawCorpus = ' '.join(f.readlines())\n",
    "\n",
    "corpus = [x for x in ' '.join(rawCorpus.split('\\'')).split(' ') if x.isalpha()]\n",
    "\n",
    "class Agent:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.fitness = 0\n",
    "        self.sentence = \"Je fais souvent ce rêve étrange et pénétrant\".split(' ')  # []  # [int]\n",
    "\n",
    "    def getFitness(self):\n",
    "        self.fitness = getFitness(self.sentenceToStr()) # ToDo\n",
    "\n",
    "    def mutate(self):\n",
    "        index = rd.randint(0, len(self.sentence) - 1)\n",
    "        index = rd.choice([-3, -1])\n",
    "        self.sentence[index] = rd.choice(corpus)  # getRandomCorpusWordIndex()\n",
    "\n",
    "    def sentenceToStr(self):\n",
    "        return ' '.join(self.sentence)\n",
    "\n",
    "\n",
    "class Genetic:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.size = 20\n",
    "        self.steps = 100\n",
    "        self.agents = []\n",
    "        self.best = None\n",
    "\n",
    "    def run(self):\n",
    "\n",
    "        self.init()\n",
    "\n",
    "        for i in range(self.steps):\n",
    "            self.getFitness()\n",
    "            self.selection()\n",
    "            print(i, self.best.sentenceToStr(), self.best.fitness.item())\n",
    "            self.mutate()\n",
    "            self.agents[0] = cp.deepcopy(self.best)\n",
    "\n",
    "        print(self.best.sentenceToStr())\n",
    "\n",
    "        \n",
    "\n",
    "    def init(self):\n",
    "        for i in range(self.size):\n",
    "            self.agents.append(Agent())\n",
    "\n",
    "    def getFitness(self):\n",
    "        for i in range(self.size):\n",
    "            self.agents[i].getFitness()\n",
    "\n",
    "    def selection(self):\n",
    "\n",
    "        newAgents = []\n",
    "        self.agents = sorted(self.agents, key=lambda x: x.fitness)\n",
    "\n",
    "        self.best = cp.deepcopy(self.agents[len(self.agents) - 1])\n",
    "\n",
    "        maxRandVal = self.size * (self.size + 1) / 2\n",
    "\n",
    "        for i in range(self.size):\n",
    "            cpt = 1\n",
    "            randVal = rd.randint(0, maxRandVal - 1)\n",
    "            randVal -= cpt\n",
    "            while randVal > 0:\n",
    "                cpt += 1\n",
    "                randVal -= cpt\n",
    "\n",
    "            newAgents.append(self.agents[cpt - 1])\n",
    "\n",
    "        #newAgents[0] = cp.deepcopy(self.best)\n",
    "        self.agents = newAgents\n",
    "\n",
    "\n",
    "    def mutate(self):\n",
    "        for i in range(self.size):\n",
    "            self.agents[i].mutate()\n",
    "\n",
    "gen = Genetic()\n",
    "gen.run()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.0 ('Jam-PaJBdDn2')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "73c349bd78c61de78c58332dcc18ecd360fed47953e9b155e86049e175178897"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}