OpenOcra.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_cpp import Llama\n",
    "import pandas as pd\n",
    "import json\n",
    "import random\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>het_857</td>\n",
       "      <td>Those whose child is going through a'phase'may...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>het_368</td>\n",
       "      <td>Bureaucrats cut red tape-- lengthwise.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>het_1204</td>\n",
       "      <td>If they say you look good in glasses that woul...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>het_1043</td>\n",
       "      <td>Nearing the end of his sentence, the convicted...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>het_1098</td>\n",
       "      <td>A logger went to his doctor because of a probl...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id                                               text  label\n",
       "0   het_857  Those whose child is going through a'phase'may...      1\n",
       "1   het_368             Bureaucrats cut red tape-- lengthwise.      0\n",
       "2  het_1204  If they say you look good in glasses that woul...      1\n",
       "3  het_1043  Nearing the end of his sentence, the convicted...      1\n",
       "4  het_1098  A logger went to his doctor because of a probl...      1"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_json('data/llm_task1_hetero.json', orient=\"records\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "445"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = df\n",
    "len(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "llama_model_loader: loaded meta data with 20 key-value pairs and 291 tensors from models/mistral-7b-openorca.Q6_K.gguf (version GGUF V2)\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q6_K     [  4096, 32002,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:              blk.0.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:              blk.0.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:         blk.0.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:            blk.0.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:              blk.0.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:              blk.1.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:              blk.1.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:         blk.1.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:            blk.1.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:              blk.1.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:              blk.2.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:              blk.2.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:         blk.2.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:            blk.2.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:              blk.2.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:              blk.3.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:              blk.3.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:         blk.3.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:            blk.3.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:              blk.3.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:              blk.4.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:              blk.4.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:         blk.4.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:            blk.4.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:              blk.4.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:              blk.5.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:              blk.5.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:         blk.5.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:            blk.5.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:              blk.5.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:              blk.6.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:              blk.6.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:         blk.6.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:            blk.6.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:              blk.6.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:              blk.7.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:              blk.7.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:         blk.7.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:            blk.7.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:              blk.7.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:              blk.8.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:              blk.8.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:         blk.8.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:            blk.8.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:              blk.8.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:              blk.9.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:              blk.9.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:         blk.9.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:            blk.9.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:              blk.9.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:             blk.10.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:             blk.10.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:        blk.10.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:           blk.10.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:             blk.10.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:             blk.11.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:             blk.11.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:        blk.11.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:           blk.11.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:             blk.11.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:             blk.12.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:             blk.12.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:        blk.12.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:           blk.12.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:             blk.12.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:             blk.13.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:             blk.13.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:        blk.13.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:           blk.13.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:             blk.13.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:             blk.14.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:             blk.14.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:        blk.14.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:           blk.14.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:             blk.14.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:             blk.15.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:             blk.15.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:        blk.15.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:           blk.15.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:             blk.15.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:             blk.16.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:             blk.16.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:        blk.16.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:           blk.16.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:             blk.16.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:             blk.17.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:             blk.17.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:        blk.17.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:           blk.17.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:             blk.17.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:             blk.18.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:             blk.18.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:        blk.18.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:           blk.18.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:             blk.18.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:             blk.19.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:             blk.19.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:        blk.19.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:           blk.19.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:             blk.19.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:             blk.20.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:             blk.20.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:        blk.20.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:           blk.20.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:             blk.20.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:             blk.21.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:             blk.21.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:        blk.21.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:           blk.21.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:             blk.21.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:             blk.22.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:             blk.22.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:        blk.22.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:           blk.22.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:             blk.22.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:             blk.23.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:             blk.23.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:        blk.23.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:           blk.23.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:             blk.23.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:             blk.24.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:             blk.24.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:        blk.24.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:           blk.24.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:             blk.24.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:             blk.25.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:             blk.25.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:        blk.25.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:           blk.25.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:             blk.25.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:             blk.26.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:             blk.26.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:        blk.26.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:           blk.26.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:             blk.26.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:             blk.27.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:             blk.27.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:        blk.27.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:           blk.27.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:             blk.27.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:             blk.28.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:             blk.28.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:        blk.28.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:           blk.28.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:             blk.28.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:             blk.29.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:             blk.29.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:        blk.29.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:           blk.29.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:             blk.29.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:             blk.30.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:             blk.30.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:        blk.30.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:           blk.30.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:             blk.30.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:             blk.31.attn_q.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:             blk.31.attn_k.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:        blk.31.attn_output.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:           blk.31.ffn_gate.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:             blk.31.ffn_up.weight q6_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:                    output.weight q6_K     [  4096, 32002,     1,     1 ]\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = open-orca_mistral-7b-openorca\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 32768\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 14336\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 8\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                       llama.rope.freq_base f32              = 10000.000000\n",
      "llama_model_loader: - kv  11:                          general.file_type u32              = 18\n",
      "llama_model_loader: - kv  12:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.tokens arr[str,32002]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  14:                      tokenizer.ggml.scores arr[f32,32002]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  15:                  tokenizer.ggml.token_type arr[i32,32002]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 32000\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q6_K:  226 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 261/32002 ).\n",
      "llm_load_print_meta: format           = GGUF V2\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32002\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 32768\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 8\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 4\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 14336\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 32768\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly Q6_K\n",
      "llm_load_print_meta: model params     = 7.24 B\n",
      "llm_load_print_meta: model size       = 5.53 GiB (6.56 BPW) \n",
      "llm_load_print_meta: general.name   = open-orca_mistral-7b-openorca\n",
      "llm_load_print_meta: BOS token = 1 '<s>'\n",
      "llm_load_print_meta: EOS token = 32000 '<dummy32000>'\n",
      "llm_load_print_meta: UNK token = 0 '<unk>'\n",
      "llm_load_print_meta: LF token  = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size =    0.11 MiB\n",
      "llm_load_tensors: mem required  = 5666.21 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 2048\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: kv self size  =  256.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 740/740\n",
      "ggml_metal_init: allocating\n",
      "ggml_metal_init: found device: Apple M1 Pro\n",
      "ggml_metal_init: picking default device: Apple M1 Pro\n",
      "ggml_metal_init: default.metallib not found, loading from source\n",
      "ggml_metal_init: loading '/Users/ryan/.local/lib/python3.9/site-packages/llama_cpp/ggml-metal.metal'\n",
      "ggml_metal_init: GPU name:   Apple M1 Pro\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyApple7 (1007)\n",
      "ggml_metal_init: hasUnifiedMemory              = true\n",
      "ggml_metal_init: recommendedMaxWorkingSetSize  = 10922.67 MiB\n",
      "ggml_metal_init: maxTransferRate               = built-in GPU\n",
      "llama_new_context_with_model: compute buffer total size = 159.07 MiB\n",
      "llama_new_context_with_model: max tensor size =   102.55 MiB\n",
      "ggml_metal_add_buffer: allocated 'data            ' buffer, size =  5666.81 MiB, (11746.28 / 10922.67)ggml_metal_add_buffer: warning: current allocated size is greater than the recommended max working set size\n",
      "ggml_metal_add_buffer: allocated 'kv              ' buffer, size =   256.02 MiB, (12002.30 / 10922.67)ggml_metal_add_buffer: warning: current allocated size is greater than the recommended max working set size\n",
      "ggml_metal_add_buffer: allocated 'alloc           ' buffer, size =   156.02 MiB, (12158.31 / 10922.67)ggml_metal_add_buffer: warning: current allocated size is greater than the recommended max working set size\n",
      "AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n",
      "ggml_metal_free: deallocating\n"
     ]
    }
   ],
   "source": [
    "PATH = 'models/mistral-7b-openorca.Q6_K.gguf'\n",
    "llm = Llama(model_path=PATH, n_gpu_layers=75, n_ctx=2048)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_json(strJson):\n",
    "  try:\n",
    "    json.loads(strJson)\n",
    "  except ValueError as e:\n",
    "    return False\n",
    "  return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate(prompt, attempt):\n",
    "  if attempt == 1:\n",
    "    temp = 0.2\n",
    "  else:\n",
    "    temp = round(random.uniform(0.3, 0.6), 1)\n",
    "  output = llm(prompt, max_tokens=-1, temperature=temp)\n",
    "  return output['choices'][0]['text']\n",
    "\n",
    "def parseResult(jsonStr, row):\n",
    "  if not is_json(jsonStr.strip()):\n",
    "    return None\n",
    "  else:\n",
    "    parsedJson = json.loads(jsonStr.strip())\n",
    "    return parsedJson"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 1: Pun Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = f''' ### INSTRUCTION:\n",
    "You are to classify whether a given sentence is a pun based on the following criteria:\n",
    "1. Ambiguity -  there exists a word in the sentence that has two similarly likely interpretations.\n",
    "2. Distinctiveness - the two interpretations are very different from each other i.e. how distinct are the words semantically related to the two interpretations from each other. There needs to be at least one different word in the set of words supporting each interpretation.\\n\n",
    "For a given sentence to be a pun, it should satisfy BOTH criteria - Ambiguity and Distinctiveness.\\n\n",
    "The final output is either true or false where true means that the sentence is a pun.\\n\n",
    "It is possible that a sentence does not have a word that is ambiguous. The result in this case is false.\\n\n",
    "It is possible that a sentence has Ambiguity but no Distinctiveness. The result in this case is false.\\n\n",
    "Here are three examples of the cases you will encounter:\n",
    "1. An example with both ambiguity and distinctiveness - \n",
    "The magician got so mad that he pulled his hare out.\n",
    "{{\n",
    "  \"output\": true,\n",
    "  \"ambiguity\": \"The pun word 'hare' supports two plausible interpretations of 'hare' meaning a rabbit and 'hair' meaning human hair.\",\n",
    "  \"distinctiveness\": \"In the given sentence, the words 'magician' relates to 'hare' while 'pulled' refers to the second interpretation of 'hair'. Both of these are distinct where one refers to a magician's animal while the other refers to an action done in anger which is pulling your hair.\"\n",
    "}}\n",
    "2. An example with only ambiguity -\n",
    "I went to the bank.\n",
    "{{\n",
    "  \"output\": false,\n",
    "  \"ambiguity\": \"The word 'bank' does have ambiguity here where it supports two plausible interpretations of bank as in a financial institution and bank as in the banks of a river.\",\n",
    "  \"distinctiveness\": \"There are no other words in the sentence that provide distinctiveness to the two interpretations, the sentence is not a pun.\"\n",
    "}}\n",
    "3. An example with neither ambiguity nor distinctiveness -\n",
    "Let us go home.\n",
    "{{\n",
    "  \"output\": false,\n",
    "  \"ambiguity\": \"There is no ambiguous word in the sentence.\",\n",
    "  \"distinctiveness\": \"Not applicable\"\n",
    "}}\n",
    "Identify whether the input sentence is a pun and explain the result based on ambiguity and distinctiveness in valid JSON format. Generate a response in the form of a valid JSON object with three keys: output, ambiguity and distinctiveness.\\n\n",
    "### INPUT:\\n\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      15.16 ms /   116 runs   (    0.13 ms per token,  7653.23 tokens per second)\n",
      "llama_print_timings: prompt eval time =    2160.70 ms /    25 tokens (   86.43 ms per token,    11.57 tokens per second)\n",
      "llama_print_timings:        eval time =    6151.46 ms /   115 runs   (   53.49 ms per token,    18.69 tokens per second)\n",
      "llama_print_timings:       total time =    8529.95 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      15.18 ms /   128 runs   (    0.12 ms per token,  8431.59 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.01 ms /    17 tokens (   11.29 ms per token,    88.54 tokens per second)\n",
      "llama_print_timings:        eval time =    6748.20 ms /   127 runs   (   53.14 ms per token,    18.82 tokens per second)\n",
      "llama_print_timings:       total time =    7147.94 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      17.39 ms /   119 runs   (    0.15 ms per token,  6844.98 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.17 ms /    22 tokens (    9.05 ms per token,   110.46 tokens per second)\n",
      "llama_print_timings:        eval time =    6368.90 ms /   118 runs   (   53.97 ms per token,    18.53 tokens per second)\n",
      "llama_print_timings:       total time =    6805.53 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       5.51 ms /    39 runs   (    0.14 ms per token,  7072.91 tokens per second)\n",
      "llama_print_timings: prompt eval time =     520.14 ms /    67 tokens (    7.76 ms per token,   128.81 tokens per second)\n",
      "llama_print_timings:        eval time =    2050.41 ms /    38 runs   (   53.96 ms per token,    18.53 tokens per second)\n",
      "llama_print_timings:       total time =    2657.83 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 -- 1\n",
      "4 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.90 ms /   126 runs   (    0.12 ms per token,  8455.24 tokens per second)\n",
      "llama_print_timings: prompt eval time =     201.11 ms /    21 tokens (    9.58 ms per token,   104.42 tokens per second)\n",
      "llama_print_timings:        eval time =    6643.62 ms /   125 runs   (   53.15 ms per token,    18.82 tokens per second)\n",
      "llama_print_timings:       total time =    7046.05 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.39 ms /   131 runs   (    0.10 ms per token,  9783.42 tokens per second)\n",
      "llama_print_timings: prompt eval time =     210.88 ms /    31 tokens (    6.80 ms per token,   147.01 tokens per second)\n",
      "llama_print_timings:        eval time =    6958.07 ms /   130 runs   (   53.52 ms per token,    18.68 tokens per second)\n",
      "llama_print_timings:       total time =    7363.33 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.17 ms /   139 runs   (    0.10 ms per token,  9812.92 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.01 ms /    17 tokens (   11.24 ms per token,    89.00 tokens per second)\n",
      "llama_print_timings:        eval time =    7313.75 ms /   138 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    7699.11 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.71 ms /    70 runs   (    0.10 ms per token, 10435.30 tokens per second)\n",
      "llama_print_timings: prompt eval time =     202.53 ms /    25 tokens (    8.10 ms per token,   123.44 tokens per second)\n",
      "llama_print_timings:        eval time =    3652.66 ms /    69 runs   (   52.94 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3951.07 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.57 ms /    45 runs   (    0.10 ms per token,  9855.45 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.27 ms /    22 tokens (    9.01 ms per token,   110.96 tokens per second)\n",
      "llama_print_timings:        eval time =    2325.97 ms /    44 runs   (   52.86 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    2588.10 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.97 ms /   145 runs   (    0.10 ms per token,  9688.63 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.54 ms /    20 tokens (    9.18 ms per token,   108.97 tokens per second)\n",
      "llama_print_timings:        eval time =    7642.76 ms /   144 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    8030.29 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.62 ms /   112 runs   (    0.10 ms per token,  9640.21 tokens per second)\n",
      "llama_print_timings: prompt eval time =     187.59 ms /    15 tokens (   12.51 ms per token,    79.96 tokens per second)\n",
      "llama_print_timings:        eval time =    5880.59 ms /   111 runs   (   52.98 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    6225.93 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      10.94 ms /   111 runs   (    0.10 ms per token, 10150.89 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.64 ms /    28 tokens (    7.34 ms per token,   136.16 tokens per second)\n",
      "llama_print_timings:        eval time =    5834.86 ms /   110 runs   (   53.04 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    6198.79 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.86 ms /   135 runs   (    0.10 ms per token,  9738.85 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.10 ms /    18 tokens (   10.73 ms per token,    93.22 tokens per second)\n",
      "llama_print_timings:        eval time =    7108.31 ms /   134 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    7494.24 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.74 ms /   127 runs   (    0.10 ms per token,  9966.26 tokens per second)\n",
      "llama_print_timings: prompt eval time =     344.37 ms /    33 tokens (   10.44 ms per token,    95.83 tokens per second)\n",
      "llama_print_timings:        eval time =    6690.65 ms /   126 runs   (   53.10 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    7212.60 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.69 ms /    45 runs   (    0.10 ms per token,  9590.79 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.38 ms /    15 tokens (   12.56 ms per token,    79.63 tokens per second)\n",
      "llama_print_timings:        eval time =    2321.01 ms /    44 runs   (   52.75 ms per token,    18.96 tokens per second)\n",
      "llama_print_timings:       total time =    2571.13 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.26 ms /    40 runs   (    0.11 ms per token,  9378.66 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.56 ms /    23 tokens (    8.68 ms per token,   115.25 tokens per second)\n",
      "llama_print_timings:        eval time =    2060.27 ms /    39 runs   (   52.83 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2316.89 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.74 ms /   119 runs   (    0.10 ms per token, 10132.83 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.10 ms /    22 tokens (    9.00 ms per token,   111.05 tokens per second)\n",
      "llama_print_timings:        eval time =    6260.33 ms /   118 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    6626.96 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.69 ms /   139 runs   (    0.11 ms per token,  9464.15 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.66 ms /    20 tokens (    9.18 ms per token,   108.90 tokens per second)\n",
      "llama_print_timings:        eval time =    7329.82 ms /   138 runs   (   53.11 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    7712.96 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.54 ms /   131 runs   (    0.09 ms per token, 11354.77 tokens per second)\n",
      "llama_print_timings: prompt eval time =     204.01 ms /    32 tokens (    6.38 ms per token,   156.85 tokens per second)\n",
      "llama_print_timings:        eval time =    6911.70 ms /   130 runs   (   53.17 ms per token,    18.81 tokens per second)\n",
      "llama_print_timings:       total time =    7298.69 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.91 ms /   120 runs   (    0.10 ms per token, 10078.95 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.48 ms /    18 tokens (   10.69 ms per token,    93.52 tokens per second)\n",
      "llama_print_timings:        eval time =    6316.63 ms /   119 runs   (   53.08 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    6679.58 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.62 ms /   115 runs   (    0.10 ms per token,  9900.14 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.16 ms /    23 tokens (    8.66 ms per token,   115.48 tokens per second)\n",
      "llama_print_timings:        eval time =    6048.18 ms /   114 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    6407.81 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.33 ms /    40 runs   (    0.11 ms per token,  9246.42 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.92 ms /    18 tokens (   10.72 ms per token,    93.30 tokens per second)\n",
      "llama_print_timings:        eval time =    2059.73 ms /    39 runs   (   52.81 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2311.80 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.53 ms /    39 runs   (    0.12 ms per token,  8607.37 tokens per second)\n",
      "llama_print_timings: prompt eval time =     182.88 ms /    19 tokens (    9.63 ms per token,   103.89 tokens per second)\n",
      "llama_print_timings:        eval time =    2007.61 ms /    38 runs   (   52.83 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2249.23 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22 -- 1\n",
      "23 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.60 ms /    45 runs   (    0.10 ms per token,  9782.61 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.85 ms /    16 tokens (   11.80 ms per token,    84.72 tokens per second)\n",
      "llama_print_timings:        eval time =    2321.27 ms /    44 runs   (   52.76 ms per token,    18.96 tokens per second)\n",
      "llama_print_timings:       total time =    2572.14 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.68 ms /   132 runs   (    0.10 ms per token, 10408.45 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.87 ms /    18 tokens (   10.72 ms per token,    93.33 tokens per second)\n",
      "llama_print_timings:        eval time =    6946.13 ms /   131 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    7322.35 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.88 ms /   126 runs   (    0.10 ms per token,  9784.13 tokens per second)\n",
      "llama_print_timings: prompt eval time =     345.25 ms /    33 tokens (   10.46 ms per token,    95.58 tokens per second)\n",
      "llama_print_timings:        eval time =    6633.70 ms /   125 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    7155.78 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.26 ms /    40 runs   (    0.11 ms per token,  9398.50 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.06 ms /    19 tokens (   10.21 ms per token,    97.91 tokens per second)\n",
      "llama_print_timings:        eval time =    2060.08 ms /    39 runs   (   52.82 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2312.57 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       7.60 ms /    73 runs   (    0.10 ms per token,  9600.21 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.22 ms /    16 tokens (   11.76 ms per token,    85.01 tokens per second)\n",
      "llama_print_timings:        eval time =    3808.65 ms /    72 runs   (   52.90 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    4100.04 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.48 ms /   113 runs   (    0.10 ms per token,  9846.64 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.88 ms /    18 tokens (   10.72 ms per token,    93.32 tokens per second)\n",
      "llama_print_timings:        eval time =    5937.87 ms /   112 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    6292.86 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.46 ms /   148 runs   (    0.10 ms per token, 10233.72 tokens per second)\n",
      "llama_print_timings: prompt eval time =     373.08 ms /    53 tokens (    7.04 ms per token,   142.06 tokens per second)\n",
      "llama_print_timings:        eval time =    7827.94 ms /   147 runs   (   53.25 ms per token,    18.78 tokens per second)\n",
      "llama_print_timings:       total time =    8411.61 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.43 ms /    40 runs   (    0.11 ms per token,  9023.23 tokens per second)\n",
      "llama_print_timings: prompt eval time =     187.66 ms /    14 tokens (   13.40 ms per token,    74.60 tokens per second)\n",
      "llama_print_timings:        eval time =    2062.29 ms /    39 runs   (   52.88 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    2310.39 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.89 ms /   125 runs   (    0.10 ms per token,  9698.94 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.09 ms /    17 tokens (   11.24 ms per token,    88.96 tokens per second)\n",
      "llama_print_timings:        eval time =    6572.82 ms /   124 runs   (   53.01 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6940.04 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.19 ms /    40 runs   (    0.10 ms per token,  9557.95 tokens per second)\n",
      "llama_print_timings: prompt eval time =     202.54 ms /    25 tokens (    8.10 ms per token,   123.43 tokens per second)\n",
      "llama_print_timings:        eval time =    2060.57 ms /    39 runs   (   52.84 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2319.76 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      10.92 ms /   108 runs   (    0.10 ms per token,  9891.92 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.16 ms /    20 tokens (    9.16 ms per token,   109.19 tokens per second)\n",
      "llama_print_timings:        eval time =    5672.48 ms /   107 runs   (   53.01 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    6008.18 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "33 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.22 ms /    40 runs   (    0.11 ms per token,  9476.43 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.05 ms /    12 tokens (   15.25 ms per token,    65.56 tokens per second)\n",
      "llama_print_timings:        eval time =    2056.34 ms /    39 runs   (   52.73 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    2296.64 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34 -- 1\n",
      "35 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.82 ms /   125 runs   (    0.10 ms per token,  9748.87 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.23 ms /    16 tokens (   11.76 ms per token,    85.00 tokens per second)\n",
      "llama_print_timings:        eval time =    6571.21 ms /   124 runs   (   52.99 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6938.10 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.92 ms /   116 runs   (    0.10 ms per token,  9734.81 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.90 ms /    21 tokens (    9.33 ms per token,   107.20 tokens per second)\n",
      "llama_print_timings:        eval time =    6092.97 ms /   115 runs   (   52.98 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6450.42 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "37 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.59 ms /   126 runs   (    0.10 ms per token, 10005.56 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.95 ms /    21 tokens (    9.33 ms per token,   107.17 tokens per second)\n",
      "llama_print_timings:        eval time =    6633.37 ms /   125 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    7006.98 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "38 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.89 ms /   121 runs   (    0.11 ms per token,  9388.58 tokens per second)\n",
      "llama_print_timings: prompt eval time =     206.23 ms /    27 tokens (    7.64 ms per token,   130.92 tokens per second)\n",
      "llama_print_timings:        eval time =    6369.36 ms /   120 runs   (   53.08 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    6748.34 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.74 ms /   140 runs   (    0.10 ms per token, 10189.97 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.37 ms /    20 tokens (    9.17 ms per token,   109.07 tokens per second)\n",
      "llama_print_timings:        eval time =    7373.54 ms /   139 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    7753.18 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.63 ms /    46 runs   (    0.10 ms per token,  9939.50 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.22 ms /    21 tokens (    9.34 ms per token,   107.02 tokens per second)\n",
      "llama_print_timings:        eval time =    2378.56 ms /    45 runs   (   52.86 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    2639.62 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "41 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.98 ms /   122 runs   (    0.11 ms per token,  9397.63 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.36 ms /    15 tokens (   12.56 ms per token,    79.64 tokens per second)\n",
      "llama_print_timings:        eval time =    6417.21 ms /   121 runs   (   53.03 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    6781.49 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "42 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.68 ms /   125 runs   (    0.10 ms per token,  9859.60 tokens per second)\n",
      "llama_print_timings: prompt eval time =     190.70 ms /    17 tokens (   11.22 ms per token,    89.15 tokens per second)\n",
      "llama_print_timings:        eval time =    6572.50 ms /   124 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6940.40 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "43 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.82 ms /    40 runs   (    0.10 ms per token, 10473.95 tokens per second)\n",
      "llama_print_timings: prompt eval time =     211.38 ms /    31 tokens (    6.82 ms per token,   146.66 tokens per second)\n",
      "llama_print_timings:        eval time =    2062.20 ms /    39 runs   (   52.88 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    2330.65 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "44 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.53 ms /    40 runs   (    0.11 ms per token,  8837.83 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.69 ms /    18 tokens (   10.71 ms per token,    93.41 tokens per second)\n",
      "llama_print_timings:        eval time =    2061.53 ms /    39 runs   (   52.86 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    2314.42 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "45 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.64 ms /    64 runs   (    0.10 ms per token,  9644.36 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.05 ms /    19 tokens (   10.21 ms per token,    97.91 tokens per second)\n",
      "llama_print_timings:        eval time =    3328.92 ms /    63 runs   (   52.84 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    3614.08 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "46 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.13 ms /    39 runs   (    0.11 ms per token,  9452.25 tokens per second)\n",
      "llama_print_timings: prompt eval time =     203.97 ms /    26 tokens (    7.85 ms per token,   127.47 tokens per second)\n",
      "llama_print_timings:        eval time =    2009.49 ms /    38 runs   (   52.88 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    2269.59 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "47 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.77 ms /   120 runs   (    0.11 ms per token,  9400.71 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.63 ms /    19 tokens (   10.24 ms per token,    97.62 tokens per second)\n",
      "llama_print_timings:        eval time =    6309.66 ms /   119 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    6677.62 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.88 ms /    66 runs   (    0.10 ms per token,  9598.60 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.24 ms /    16 tokens (   11.76 ms per token,    85.00 tokens per second)\n",
      "llama_print_timings:        eval time =    3437.43 ms /    65 runs   (   52.88 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    3719.24 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.00 ms /   125 runs   (    0.10 ms per token,  9613.17 tokens per second)\n",
      "llama_print_timings: prompt eval time =     210.46 ms /    30 tokens (    7.02 ms per token,   142.55 tokens per second)\n",
      "llama_print_timings:        eval time =    6579.30 ms /   124 runs   (   53.06 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    6967.11 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.19 ms /   119 runs   (    0.10 ms per token,  9762.90 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.84 ms /    18 tokens (   10.71 ms per token,    93.34 tokens per second)\n",
      "llama_print_timings:        eval time =    6254.19 ms /   118 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6614.31 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "51 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.59 ms /   120 runs   (    0.10 ms per token,  9530.62 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.19 ms /    22 tokens (    9.01 ms per token,   111.00 tokens per second)\n",
      "llama_print_timings:        eval time =    6313.69 ms /   119 runs   (   53.06 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    6686.32 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "52 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.35 ms /    67 runs   (    0.09 ms per token, 10556.17 tokens per second)\n",
      "llama_print_timings: prompt eval time =     207.09 ms /    29 tokens (    7.14 ms per token,   140.03 tokens per second)\n",
      "llama_print_timings:        eval time =    3494.99 ms /    66 runs   (   52.95 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3795.74 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "53 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.83 ms /   125 runs   (    0.10 ms per token,  9745.83 tokens per second)\n",
      "llama_print_timings: prompt eval time =     204.41 ms /    32 tokens (    6.39 ms per token,   156.55 tokens per second)\n",
      "llama_print_timings:        eval time =    6581.42 ms /   124 runs   (   53.08 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    6961.49 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "54 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.92 ms /    45 runs   (    0.11 ms per token,  9144.48 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.50 ms /    18 tokens (   10.69 ms per token,    93.51 tokens per second)\n",
      "llama_print_timings:        eval time =    2325.78 ms /    44 runs   (   52.86 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    2584.80 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "55 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.17 ms /    40 runs   (    0.10 ms per token,  9587.73 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.60 ms /    16 tokens (   11.79 ms per token,    84.83 tokens per second)\n",
      "llama_print_timings:        eval time =    2058.87 ms /    39 runs   (   52.79 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2304.24 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "56 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.15 ms /    40 runs   (    0.10 ms per token,  9650.18 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.80 ms /    20 tokens (    9.19 ms per token,   108.81 tokens per second)\n",
      "llama_print_timings:        eval time =    2057.19 ms /    39 runs   (   52.75 ms per token,    18.96 tokens per second)\n",
      "llama_print_timings:       total time =    2296.39 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "57 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.72 ms /   135 runs   (    0.10 ms per token,  9838.93 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.68 ms /    23 tokens (    8.68 ms per token,   115.19 tokens per second)\n",
      "llama_print_timings:        eval time =    7111.72 ms /   134 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    7502.68 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "58 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      10.60 ms /   114 runs   (    0.09 ms per token, 10756.75 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.97 ms /    23 tokens (    8.65 ms per token,   115.59 tokens per second)\n",
      "llama_print_timings:        eval time =    5991.37 ms /   113 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    6352.87 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "59 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.05 ms /    39 runs   (    0.10 ms per token,  9617.76 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.18 ms /    20 tokens (    9.16 ms per token,   109.18 tokens per second)\n",
      "llama_print_timings:        eval time =    2004.55 ms /    38 runs   (   52.75 ms per token,    18.96 tokens per second)\n",
      "llama_print_timings:       total time =    2242.49 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.07 ms /    40 runs   (    0.10 ms per token,  9825.60 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.16 ms /    18 tokens (   10.68 ms per token,    93.67 tokens per second)\n",
      "llama_print_timings:        eval time =    2055.17 ms /    39 runs   (   52.70 ms per token,    18.98 tokens per second)\n",
      "llama_print_timings:       total time =    2302.44 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "61 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.30 ms /   128 runs   (    0.10 ms per token,  9621.89 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.21 ms /    17 tokens (   11.25 ms per token,    88.91 tokens per second)\n",
      "llama_print_timings:        eval time =    6736.68 ms /   127 runs   (   53.04 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    7112.00 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "62 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       5.26 ms /    47 runs   (    0.11 ms per token,  8926.88 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.55 ms /    18 tokens (   10.70 ms per token,    93.48 tokens per second)\n",
      "llama_print_timings:        eval time =    2432.39 ms /    46 runs   (   52.88 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    2694.41 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "63 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.98 ms /    46 runs   (    0.11 ms per token,  9233.24 tokens per second)\n",
      "llama_print_timings: prompt eval time =     190.64 ms /    17 tokens (   11.21 ms per token,    89.17 tokens per second)\n",
      "llama_print_timings:        eval time =    2376.72 ms /    45 runs   (   52.82 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2633.60 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.92 ms /    47 runs   (    0.10 ms per token,  9560.62 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.51 ms /    18 tokens (   10.70 ms per token,    93.50 tokens per second)\n",
      "llama_print_timings:        eval time =    2429.05 ms /    46 runs   (   52.81 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2687.49 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.75 ms /   131 runs   (    0.10 ms per token, 10274.51 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.67 ms /    24 tokens (    8.36 ms per token,   119.60 tokens per second)\n",
      "llama_print_timings:        eval time =    6897.96 ms /   130 runs   (   53.06 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    7282.00 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.97 ms /   120 runs   (    0.10 ms per token, 10028.41 tokens per second)\n",
      "llama_print_timings: prompt eval time =     203.55 ms /    25 tokens (    8.14 ms per token,   122.82 tokens per second)\n",
      "llama_print_timings:        eval time =    6313.61 ms /   119 runs   (   53.06 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    6686.84 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "67 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.60 ms /   127 runs   (    0.10 ms per token, 10079.37 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.30 ms /    18 tokens (   10.68 ms per token,    93.60 tokens per second)\n",
      "llama_print_timings:        eval time =    6683.70 ms /   126 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    7054.58 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "68 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.13 ms /    40 runs   (    0.10 ms per token,  9685.23 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.29 ms /    20 tokens (    9.16 ms per token,   109.11 tokens per second)\n",
      "llama_print_timings:        eval time =    2055.75 ms /    39 runs   (   52.71 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    2294.84 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "69 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.05 ms /    40 runs   (    0.10 ms per token,  9874.11 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.58 ms /    16 tokens (   11.79 ms per token,    84.85 tokens per second)\n",
      "llama_print_timings:        eval time =    2057.44 ms /    39 runs   (   52.75 ms per token,    18.96 tokens per second)\n",
      "llama_print_timings:       total time =    2302.11 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "70 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.60 ms /   146 runs   (    0.10 ms per token, 10002.06 tokens per second)\n",
      "llama_print_timings: prompt eval time =     202.75 ms /    25 tokens (    8.11 ms per token,   123.30 tokens per second)\n",
      "llama_print_timings:        eval time =    7701.39 ms /   145 runs   (   53.11 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    8112.37 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "71 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.46 ms /    40 runs   (    0.11 ms per token,  8970.62 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.34 ms /    15 tokens (   12.56 ms per token,    79.64 tokens per second)\n",
      "llama_print_timings:        eval time =    2059.93 ms /    39 runs   (   52.82 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2307.34 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "72 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.13 ms /    40 runs   (    0.10 ms per token,  9680.54 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.30 ms /    17 tokens (   11.25 ms per token,    88.86 tokens per second)\n",
      "llama_print_timings:        eval time =    2057.72 ms /    39 runs   (   52.76 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2304.44 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "73 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.19 ms /   121 runs   (    0.09 ms per token, 10809.36 tokens per second)\n",
      "llama_print_timings: prompt eval time =     346.51 ms /    34 tokens (   10.19 ms per token,    98.12 tokens per second)\n",
      "llama_print_timings:        eval time =    6378.53 ms /   120 runs   (   53.15 ms per token,    18.81 tokens per second)\n",
      "llama_print_timings:       total time =    6895.78 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "74 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.21 ms /    39 runs   (    0.11 ms per token,  9272.47 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.36 ms /    21 tokens (    9.35 ms per token,   106.95 tokens per second)\n",
      "llama_print_timings:        eval time =    2015.01 ms /    38 runs   (   53.03 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    2268.86 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "75 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.35 ms /    40 runs   (    0.11 ms per token,  9203.87 tokens per second)\n",
      "llama_print_timings: prompt eval time =     203.29 ms /    25 tokens (    8.13 ms per token,   122.98 tokens per second)\n",
      "llama_print_timings:        eval time =    2070.28 ms /    39 runs   (   53.08 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    2332.12 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.95 ms /   125 runs   (    0.10 ms per token, 10458.50 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.55 ms /    18 tokens (   10.75 ms per token,    93.00 tokens per second)\n",
      "llama_print_timings:        eval time =    6582.15 ms /   124 runs   (   53.08 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    6949.88 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "76 -- 1\n",
      "77 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.43 ms /   122 runs   (    0.09 ms per token, 10669.93 tokens per second)\n",
      "llama_print_timings: prompt eval time =     211.02 ms /    31 tokens (    6.81 ms per token,   146.91 tokens per second)\n",
      "llama_print_timings:        eval time =    6430.89 ms /   121 runs   (   53.15 ms per token,    18.82 tokens per second)\n",
      "llama_print_timings:       total time =    6810.67 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "78 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.02 ms /   139 runs   (    0.10 ms per token,  9912.99 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.42 ms /    20 tokens (    9.17 ms per token,   109.04 tokens per second)\n",
      "llama_print_timings:        eval time =    7328.76 ms /   138 runs   (   53.11 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    7705.75 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "79 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.84 ms /    67 runs   (    0.10 ms per token,  9798.19 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.55 ms /    19 tokens (   10.24 ms per token,    97.66 tokens per second)\n",
      "llama_print_timings:        eval time =    3492.50 ms /    66 runs   (   52.92 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    3780.58 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "80 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.10 ms /   118 runs   (    0.10 ms per token,  9754.48 tokens per second)\n",
      "llama_print_timings: prompt eval time =     203.91 ms /    26 tokens (    7.84 ms per token,   127.51 tokens per second)\n",
      "llama_print_timings:        eval time =    6214.02 ms /   117 runs   (   53.11 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    6586.73 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.12 ms /    40 runs   (    0.10 ms per token,  9704.03 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.80 ms /    16 tokens (   11.80 ms per token,    84.75 tokens per second)\n",
      "llama_print_timings:        eval time =    2058.08 ms /    39 runs   (   52.77 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2302.47 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "82 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.78 ms /    68 runs   (    0.10 ms per token, 10025.06 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.96 ms /    18 tokens (   10.72 ms per token,    93.29 tokens per second)\n",
      "llama_print_timings:        eval time =    3544.49 ms /    67 runs   (   52.90 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    3830.73 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.38 ms /   120 runs   (    0.09 ms per token, 10549.45 tokens per second)\n",
      "llama_print_timings: prompt eval time =     359.45 ms /    43 tokens (    8.36 ms per token,   119.63 tokens per second)\n",
      "llama_print_timings:        eval time =    6326.45 ms /   119 runs   (   53.16 ms per token,    18.81 tokens per second)\n",
      "llama_print_timings:       total time =    6852.55 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "83 -- 1\n",
      "84 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.66 ms /    45 runs   (    0.10 ms per token,  9652.51 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.79 ms /    23 tokens (    8.69 ms per token,   115.12 tokens per second)\n",
      "llama_print_timings:        eval time =    2327.68 ms /    44 runs   (   52.90 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    2590.42 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "85 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.66 ms /   117 runs   (    0.10 ms per token, 10036.89 tokens per second)\n",
      "llama_print_timings: prompt eval time =     203.33 ms /    25 tokens (    8.13 ms per token,   122.96 tokens per second)\n",
      "llama_print_timings:        eval time =    6131.04 ms /   116 runs   (   52.85 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    6497.80 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "86 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.88 ms /   138 runs   (    0.09 ms per token, 10716.78 tokens per second)\n",
      "llama_print_timings: prompt eval time =     206.04 ms /    27 tokens (    7.63 ms per token,   131.04 tokens per second)\n",
      "llama_print_timings:        eval time =    7277.52 ms /   137 runs   (   53.12 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    7674.91 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "87 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.10 ms /    40 runs   (    0.10 ms per token,  9751.34 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.82 ms /    15 tokens (   12.59 ms per token,    79.44 tokens per second)\n",
      "llama_print_timings:        eval time =    2055.67 ms /    39 runs   (   52.71 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    2300.03 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "88 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.50 ms /   130 runs   (    0.10 ms per token, 10396.67 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.53 ms /    23 tokens (    8.68 ms per token,   115.27 tokens per second)\n",
      "llama_print_timings:        eval time =    6850.83 ms /   129 runs   (   53.11 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    7235.75 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "89 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.59 ms /   117 runs   (    0.10 ms per token, 10092.30 tokens per second)\n",
      "llama_print_timings: prompt eval time =     204.11 ms /    26 tokens (    7.85 ms per token,   127.38 tokens per second)\n",
      "llama_print_timings:        eval time =    6153.06 ms /   116 runs   (   53.04 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    6519.63 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "90 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.56 ms /    40 runs   (    0.11 ms per token,  8773.85 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.11 ms /    22 tokens (    9.00 ms per token,   111.05 tokens per second)\n",
      "llama_print_timings:        eval time =    2061.48 ms /    39 runs   (   52.86 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    2319.68 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "91 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.46 ms /   122 runs   (    0.10 ms per token,  9789.76 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.30 ms /    20 tokens (    9.16 ms per token,   109.11 tokens per second)\n",
      "llama_print_timings:        eval time =    6420.22 ms /   121 runs   (   53.06 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    6777.86 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "92 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.72 ms /   135 runs   (    0.10 ms per token,  9841.80 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.93 ms /    15 tokens (   12.60 ms per token,    79.40 tokens per second)\n",
      "llama_print_timings:        eval time =    7105.17 ms /   134 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    7484.78 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "93 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.22 ms /   130 runs   (    0.10 ms per token,  9835.07 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.09 ms /    20 tokens (    9.15 ms per token,   109.24 tokens per second)\n",
      "llama_print_timings:        eval time =    6841.39 ms /   129 runs   (   53.03 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    7206.87 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "94 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       5.07 ms /    44 runs   (    0.12 ms per token,  8673.37 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.34 ms /    21 tokens (    9.35 ms per token,   106.96 tokens per second)\n",
      "llama_print_timings:        eval time =    2272.63 ms /    43 runs   (   52.85 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    2536.83 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.09 ms /    39 runs   (    0.10 ms per token,  9523.81 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.77 ms /    22 tokens (    8.99 ms per token,   111.24 tokens per second)\n",
      "llama_print_timings:        eval time =    2006.23 ms /    38 runs   (   52.80 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2260.15 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "96 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.69 ms /   119 runs   (    0.11 ms per token,  9379.68 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.19 ms /    20 tokens (    9.16 ms per token,   109.17 tokens per second)\n",
      "llama_print_timings:        eval time =    6255.59 ms /   118 runs   (   53.01 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    6610.07 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.55 ms /   125 runs   (    0.09 ms per token, 10820.64 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.10 ms /    19 tokens (   10.22 ms per token,    97.89 tokens per second)\n",
      "llama_print_timings:        eval time =    6577.97 ms /   124 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    6947.08 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "97 -- 1\n",
      "98 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.36 ms /    40 runs   (    0.11 ms per token,  9176.42 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.19 ms /    17 tokens (   11.25 ms per token,    88.92 tokens per second)\n",
      "llama_print_timings:        eval time =    2057.19 ms /    39 runs   (   52.75 ms per token,    18.96 tokens per second)\n",
      "llama_print_timings:       total time =    2307.03 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.07 ms /   122 runs   (    0.10 ms per token, 10104.36 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.22 ms /    23 tokens (    8.66 ms per token,   115.45 tokens per second)\n",
      "llama_print_timings:        eval time =    6416.47 ms /   121 runs   (   53.03 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    6785.92 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.25 ms /    39 runs   (    0.11 ms per token,  9170.00 tokens per second)\n",
      "llama_print_timings: prompt eval time =     187.46 ms /    14 tokens (   13.39 ms per token,    74.68 tokens per second)\n",
      "llama_print_timings:        eval time =    2008.57 ms /    38 runs   (   52.86 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    2254.11 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "101 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.29 ms /    39 runs   (    0.11 ms per token,  9086.67 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.21 ms /    19 tokens (   10.22 ms per token,    97.83 tokens per second)\n",
      "llama_print_timings:        eval time =    2007.92 ms /    38 runs   (   52.84 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2258.28 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "102 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.79 ms /   134 runs   (    0.10 ms per token, 10473.66 tokens per second)\n",
      "llama_print_timings: prompt eval time =     190.58 ms /    17 tokens (   11.21 ms per token,    89.20 tokens per second)\n",
      "llama_print_timings:        eval time =    7049.44 ms /   133 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    7427.47 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "103 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.25 ms /    40 runs   (    0.11 ms per token,  9400.71 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.49 ms /    24 tokens (    8.35 ms per token,   119.70 tokens per second)\n",
      "llama_print_timings:        eval time =    2059.37 ms /    39 runs   (   52.80 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2316.34 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "104 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.49 ms /    44 runs   (    0.10 ms per token,  9806.11 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.73 ms /    24 tokens (    8.36 ms per token,   119.56 tokens per second)\n",
      "llama_print_timings:        eval time =    2272.91 ms /    43 runs   (   52.86 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    2534.50 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "105 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.67 ms /   110 runs   (    0.11 ms per token,  9427.49 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.20 ms /    19 tokens (   10.22 ms per token,    97.84 tokens per second)\n",
      "llama_print_timings:        eval time =    5775.40 ms /   109 runs   (   52.99 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6125.87 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.45 ms /   121 runs   (    0.10 ms per token,  9718.88 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.40 ms /    20 tokens (    9.17 ms per token,   109.05 tokens per second)\n",
      "llama_print_timings:        eval time =    6360.52 ms /   120 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6715.72 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "107 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.83 ms /   134 runs   (    0.09 ms per token, 11330.01 tokens per second)\n",
      "llama_print_timings: prompt eval time =     337.74 ms /    51 tokens (    6.62 ms per token,   151.00 tokens per second)\n",
      "llama_print_timings:        eval time =    7071.30 ms /   133 runs   (   53.17 ms per token,    18.81 tokens per second)\n",
      "llama_print_timings:       total time =    7595.38 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "108 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.11 ms /    40 runs   (    0.10 ms per token,  9729.99 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.22 ms /    22 tokens (    9.01 ms per token,   110.99 tokens per second)\n",
      "llama_print_timings:        eval time =    2058.73 ms /    39 runs   (   52.79 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2312.71 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "109 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       7.21 ms /    67 runs   (    0.11 ms per token,  9291.36 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.94 ms /    18 tokens (   10.72 ms per token,    93.29 tokens per second)\n",
      "llama_print_timings:        eval time =    3491.06 ms /    66 runs   (   52.89 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    3780.90 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "110 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.43 ms /    40 runs   (    0.11 ms per token,  9023.23 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.72 ms /    18 tokens (   10.71 ms per token,    93.40 tokens per second)\n",
      "llama_print_timings:        eval time =    2059.39 ms /    39 runs   (   52.80 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2311.66 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "111 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      15.14 ms /   147 runs   (    0.10 ms per token,  9711.30 tokens per second)\n",
      "llama_print_timings: prompt eval time =     202.31 ms /    25 tokens (    8.09 ms per token,   123.57 tokens per second)\n",
      "llama_print_timings:        eval time =    7760.94 ms /   146 runs   (   53.16 ms per token,    18.81 tokens per second)\n",
      "llama_print_timings:       total time =    8170.28 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "112 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.13 ms /    39 runs   (    0.11 ms per token,  9436.24 tokens per second)\n",
      "llama_print_timings: prompt eval time =     187.53 ms /    14 tokens (   13.39 ms per token,    74.66 tokens per second)\n",
      "llama_print_timings:        eval time =    2007.95 ms /    38 runs   (   52.84 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    2251.94 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "113 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.90 ms /    45 runs   (    0.11 ms per token,  9181.80 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.34 ms /    20 tokens (    9.17 ms per token,   109.09 tokens per second)\n",
      "llama_print_timings:        eval time =    2325.95 ms /    44 runs   (   52.86 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    2575.49 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "114 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.98 ms /   129 runs   (    0.09 ms per token, 10766.15 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.31 ms /    22 tokens (    9.01 ms per token,   110.94 tokens per second)\n",
      "llama_print_timings:        eval time =    6790.46 ms /   128 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    7169.88 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "115 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.42 ms /   118 runs   (    0.11 ms per token,  9500.81 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.53 ms /    18 tokens (   10.70 ms per token,    93.49 tokens per second)\n",
      "llama_print_timings:        eval time =    6206.06 ms /   117 runs   (   53.04 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    6569.99 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "116 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.37 ms /   143 runs   (    0.10 ms per token,  9949.90 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.50 ms /    21 tokens (    9.36 ms per token,   106.87 tokens per second)\n",
      "llama_print_timings:        eval time =    7539.26 ms /   142 runs   (   53.09 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    7939.06 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "117 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.15 ms /   134 runs   (    0.11 ms per token,  9470.63 tokens per second)\n",
      "llama_print_timings: prompt eval time =     201.94 ms /    25 tokens (    8.08 ms per token,   123.80 tokens per second)\n",
      "llama_print_timings:        eval time =    7061.98 ms /   133 runs   (   53.10 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    7458.38 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "118 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.95 ms /    40 runs   (    0.10 ms per token, 10124.02 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.52 ms /    20 tokens (    9.18 ms per token,   108.98 tokens per second)\n",
      "llama_print_timings:        eval time =    2055.32 ms /    39 runs   (   52.70 ms per token,    18.98 tokens per second)\n",
      "llama_print_timings:       total time =    2294.04 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "119 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.91 ms /   124 runs   (    0.10 ms per token,  9606.45 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.36 ms /    22 tokens (    9.02 ms per token,   110.91 tokens per second)\n",
      "llama_print_timings:        eval time =    6528.45 ms /   123 runs   (   53.08 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    6905.00 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "120 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.16 ms /    40 runs   (    0.10 ms per token,  9613.07 tokens per second)\n",
      "llama_print_timings: prompt eval time =     187.65 ms /    14 tokens (   13.40 ms per token,    74.61 tokens per second)\n",
      "llama_print_timings:        eval time =    2058.98 ms /    39 runs   (   52.79 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2302.86 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "121 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.53 ms /    64 runs   (    0.10 ms per token,  9803.92 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.36 ms /    17 tokens (   11.26 ms per token,    88.84 tokens per second)\n",
      "llama_print_timings:        eval time =    3326.28 ms /    63 runs   (   52.80 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    3606.78 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "122 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.59 ms /    64 runs   (    0.10 ms per token,  9707.27 tokens per second)\n",
      "llama_print_timings: prompt eval time =     187.80 ms /    14 tokens (   13.41 ms per token,    74.55 tokens per second)\n",
      "llama_print_timings:        eval time =    3328.68 ms /    63 runs   (   52.84 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    3607.21 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "123 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.99 ms /    40 runs   (    0.10 ms per token, 10035.12 tokens per second)\n",
      "llama_print_timings: prompt eval time =     182.47 ms /    10 tokens (   18.25 ms per token,    54.80 tokens per second)\n",
      "llama_print_timings:        eval time =    2054.47 ms /    39 runs   (   52.68 ms per token,    18.98 tokens per second)\n",
      "llama_print_timings:       total time =    2292.43 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "124 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.35 ms /    40 runs   (    0.11 ms per token,  9193.29 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.71 ms /    15 tokens (   12.58 ms per token,    79.49 tokens per second)\n",
      "llama_print_timings:        eval time =    2060.51 ms /    39 runs   (   52.83 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2308.73 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "125 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.78 ms /   133 runs   (    0.10 ms per token,  9652.37 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.45 ms /    23 tokens (    8.67 ms per token,   115.32 tokens per second)\n",
      "llama_print_timings:        eval time =    7003.74 ms /   132 runs   (   53.06 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    7392.94 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "126 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.71 ms /   123 runs   (    0.10 ms per token,  9678.94 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.20 ms /    18 tokens (   10.73 ms per token,    93.17 tokens per second)\n",
      "llama_print_timings:        eval time =    6463.39 ms /   122 runs   (   52.98 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    6832.76 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "127 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.36 ms /    39 runs   (    0.11 ms per token,  8936.76 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.81 ms /    22 tokens (    8.99 ms per token,   111.22 tokens per second)\n",
      "llama_print_timings:        eval time =    2006.79 ms /    38 runs   (   52.81 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2265.46 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      10.88 ms /   117 runs   (    0.09 ms per token, 10758.62 tokens per second)\n",
      "llama_print_timings: prompt eval time =     367.16 ms /    49 tokens (    7.49 ms per token,   133.46 tokens per second)\n",
      "llama_print_timings:        eval time =    6167.56 ms /   116 runs   (   53.17 ms per token,    18.81 tokens per second)\n",
      "llama_print_timings:       total time =    6700.10 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "128 -- 1\n",
      "129 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      10.93 ms /   107 runs   (    0.10 ms per token,  9786.88 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.08 ms /    17 tokens (   11.24 ms per token,    88.97 tokens per second)\n",
      "llama_print_timings:        eval time =    5612.72 ms /   106 runs   (   52.95 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    5952.43 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "130 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.91 ms /    39 runs   (    0.10 ms per token,  9966.78 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.46 ms /    15 tokens (   12.56 ms per token,    79.59 tokens per second)\n",
      "llama_print_timings:        eval time =    2005.00 ms /    38 runs   (   52.76 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2247.74 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "131 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.45 ms /   127 runs   (    0.11 ms per token,  9443.08 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.93 ms /    23 tokens (    8.65 ms per token,   115.62 tokens per second)\n",
      "llama_print_timings:        eval time =    6688.07 ms /   126 runs   (   53.08 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    7070.34 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "132 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.32 ms /   133 runs   (    0.11 ms per token,  9289.01 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.41 ms /    22 tokens (    9.02 ms per token,   110.88 tokens per second)\n",
      "llama_print_timings:        eval time =    7002.14 ms /   132 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    7395.19 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "133 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.36 ms /   121 runs   (    0.10 ms per token,  9786.48 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.58 ms /    24 tokens (    8.36 ms per token,   119.65 tokens per second)\n",
      "llama_print_timings:        eval time =    6358.99 ms /   120 runs   (   52.99 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6730.72 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.29 ms /   142 runs   (    0.09 ms per token, 10687.14 tokens per second)\n",
      "llama_print_timings: prompt eval time =     202.53 ms /    25 tokens (    8.10 ms per token,   123.44 tokens per second)\n",
      "llama_print_timings:        eval time =    7480.86 ms /   141 runs   (   53.06 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    7884.88 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "134 -- 1\n",
      "135 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.17 ms /    40 runs   (    0.10 ms per token,  9585.43 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.75 ms /    18 tokens (   10.71 ms per token,    93.39 tokens per second)\n",
      "llama_print_timings:        eval time =    2058.65 ms /    39 runs   (   52.79 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2308.29 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "136 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.13 ms /    40 runs   (    0.10 ms per token,  9692.27 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.61 ms /    12 tokens (   15.30 ms per token,    65.36 tokens per second)\n",
      "llama_print_timings:        eval time =    2055.68 ms /    39 runs   (   52.71 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    2294.89 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "137 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.13 ms /    40 runs   (    0.10 ms per token,  9687.58 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.50 ms /    19 tokens (   10.24 ms per token,    97.68 tokens per second)\n",
      "llama_print_timings:        eval time =    2057.64 ms /    39 runs   (   52.76 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2307.34 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "138 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       9.27 ms /    94 runs   (    0.10 ms per token, 10136.96 tokens per second)\n",
      "llama_print_timings: prompt eval time =     352.12 ms /    39 tokens (    9.03 ms per token,   110.76 tokens per second)\n",
      "llama_print_timings:        eval time =    4933.59 ms /    93 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    5420.71 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "139 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       9.65 ms /    99 runs   (    0.10 ms per token, 10260.13 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.19 ms /    17 tokens (   11.25 ms per token,    88.92 tokens per second)\n",
      "llama_print_timings:        eval time =    5192.11 ms /    98 runs   (   52.98 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    5525.77 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "140 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.02 ms /    39 runs   (    0.10 ms per token,  9711.16 tokens per second)\n",
      "llama_print_timings: prompt eval time =     185.50 ms /    13 tokens (   14.27 ms per token,    70.08 tokens per second)\n",
      "llama_print_timings:        eval time =    2007.78 ms /    38 runs   (   52.84 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2248.04 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "141 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.39 ms /    40 runs   (    0.11 ms per token,  9109.54 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.10 ms /    19 tokens (   10.22 ms per token,    97.89 tokens per second)\n",
      "llama_print_timings:        eval time =    2058.02 ms /    39 runs   (   52.77 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2312.13 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "142 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.46 ms /   134 runs   (    0.10 ms per token,  9959.12 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.14 ms /    22 tokens (    9.01 ms per token,   111.03 tokens per second)\n",
      "llama_print_timings:        eval time =    7058.96 ms /   133 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    7446.85 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "143 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.62 ms /    46 runs   (    0.10 ms per token,  9954.56 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.28 ms /    22 tokens (    9.01 ms per token,   110.96 tokens per second)\n",
      "llama_print_timings:        eval time =    2377.50 ms /    45 runs   (   52.83 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2640.92 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "144 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.82 ms /   121 runs   (    0.10 ms per token, 10237.75 tokens per second)\n",
      "llama_print_timings: prompt eval time =     206.37 ms /    27 tokens (    7.64 ms per token,   130.84 tokens per second)\n",
      "llama_print_timings:        eval time =    6369.31 ms /   120 runs   (   53.08 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    6745.12 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "145 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.04 ms /    40 runs   (    0.10 ms per token,  9910.80 tokens per second)\n",
      "llama_print_timings: prompt eval time =     187.80 ms /    16 tokens (   11.74 ms per token,    85.20 tokens per second)\n",
      "llama_print_timings:        eval time =    2059.15 ms /    39 runs   (   52.80 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2303.88 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "146 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.82 ms /    45 runs   (    0.11 ms per token,  9345.79 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.40 ms /    21 tokens (    9.35 ms per token,   106.92 tokens per second)\n",
      "llama_print_timings:        eval time =    2325.82 ms /    44 runs   (   52.86 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    2586.50 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "147 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.80 ms /   122 runs   (    0.10 ms per token, 10339.86 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.62 ms /    21 tokens (    9.36 ms per token,   106.81 tokens per second)\n",
      "llama_print_timings:        eval time =    6417.58 ms /   121 runs   (   53.04 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    6784.88 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "148 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.73 ms /    45 runs   (    0.11 ms per token,  9515.75 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.03 ms /    17 tokens (   11.24 ms per token,    88.99 tokens per second)\n",
      "llama_print_timings:        eval time =    2321.39 ms /    44 runs   (   52.76 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2576.60 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "149 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.56 ms /    39 runs   (    0.12 ms per token,  8556.38 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.92 ms /    23 tokens (    8.69 ms per token,   115.05 tokens per second)\n",
      "llama_print_timings:        eval time =    2009.02 ms /    38 runs   (   52.87 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    2270.35 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "150 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.27 ms /    40 runs   (    0.11 ms per token,  9374.27 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.08 ms /    15 tokens (   12.61 ms per token,    79.33 tokens per second)\n",
      "llama_print_timings:        eval time =    2058.56 ms /    39 runs   (   52.78 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2305.31 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "151 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      15.25 ms /   158 runs   (    0.10 ms per token, 10357.94 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.45 ms /    21 tokens (    9.35 ms per token,   106.90 tokens per second)\n",
      "llama_print_timings:        eval time =    8336.29 ms /   157 runs   (   53.10 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    8755.74 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "152 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.93 ms /   145 runs   (    0.10 ms per token, 10410.68 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.40 ms /    21 tokens (    9.35 ms per token,   106.93 tokens per second)\n",
      "llama_print_timings:        eval time =    7644.87 ms /   144 runs   (   53.09 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    8046.10 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "153 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.71 ms /    40 runs   (    0.12 ms per token,  8488.96 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.47 ms /    20 tokens (    9.17 ms per token,   109.01 tokens per second)\n",
      "llama_print_timings:        eval time =    2066.00 ms /    39 runs   (   52.97 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    2313.28 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "154 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.22 ms /    39 runs   (    0.11 ms per token,  9243.90 tokens per second)\n",
      "llama_print_timings: prompt eval time =     204.27 ms /    32 tokens (    6.38 ms per token,   156.66 tokens per second)\n",
      "llama_print_timings:        eval time =    2012.57 ms /    38 runs   (   52.96 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    2274.62 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "155 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.05 ms /    40 runs   (    0.10 ms per token,  9874.11 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.59 ms /    24 tokens (    8.36 ms per token,   119.65 tokens per second)\n",
      "llama_print_timings:        eval time =    2058.37 ms /    39 runs   (   52.78 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2315.14 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "156 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.34 ms /    40 runs   (    0.11 ms per token,  9212.34 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.01 ms /    17 tokens (   11.24 ms per token,    89.00 tokens per second)\n",
      "llama_print_timings:        eval time =    2058.26 ms /    39 runs   (   52.78 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2307.30 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "157 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.95 ms /   138 runs   (    0.10 ms per token,  9893.18 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.46 ms /    19 tokens (   10.23 ms per token,    97.71 tokens per second)\n",
      "llama_print_timings:        eval time =    7271.88 ms /   137 runs   (   53.08 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    7665.00 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "158 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.71 ms /   133 runs   (    0.10 ms per token,  9700.24 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.44 ms /    24 tokens (    8.35 ms per token,   119.74 tokens per second)\n",
      "llama_print_timings:        eval time =    7005.40 ms /   132 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    7396.95 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "159 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.93 ms /    39 runs   (    0.10 ms per token,  9931.25 tokens per second)\n",
      "llama_print_timings: prompt eval time =     350.88 ms /    37 tokens (    9.48 ms per token,   105.45 tokens per second)\n",
      "llama_print_timings:        eval time =    2011.07 ms /    38 runs   (   52.92 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    2419.75 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "160 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.25 ms /    40 runs   (    0.11 ms per token,  9418.41 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.93 ms /    15 tokens (   12.60 ms per token,    79.40 tokens per second)\n",
      "llama_print_timings:        eval time =    2057.86 ms /    39 runs   (   52.77 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2304.37 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "161 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.27 ms /   125 runs   (    0.10 ms per token, 10191.60 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.96 ms /    28 tokens (    7.36 ms per token,   135.95 tokens per second)\n",
      "llama_print_timings:        eval time =    6583.91 ms /   124 runs   (   53.10 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    6970.26 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "162 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.32 ms /    40 runs   (    0.11 ms per token,  9250.69 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.70 ms /    15 tokens (   12.58 ms per token,    79.49 tokens per second)\n",
      "llama_print_timings:        eval time =    2057.67 ms /    39 runs   (   52.76 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2304.01 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "163 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.08 ms /    39 runs   (    0.10 ms per token,  9547.12 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.90 ms /    15 tokens (   12.59 ms per token,    79.41 tokens per second)\n",
      "llama_print_timings:        eval time =    2003.32 ms /    38 runs   (   52.72 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    2248.37 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.75 ms /   126 runs   (    0.10 ms per token,  9884.68 tokens per second)\n",
      "llama_print_timings: prompt eval time =     203.00 ms /    25 tokens (    8.12 ms per token,   123.15 tokens per second)\n",
      "llama_print_timings:        eval time =    6628.58 ms /   125 runs   (   53.03 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    7008.38 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "165 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.99 ms /    46 runs   (    0.11 ms per token,  9227.68 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.04 ms /    19 tokens (   10.21 ms per token,    97.92 tokens per second)\n",
      "llama_print_timings:        eval time =    2377.67 ms /    45 runs   (   52.84 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2637.50 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "166 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.97 ms /   132 runs   (    0.10 ms per token, 10177.33 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.46 ms /    23 tokens (    8.67 ms per token,   115.31 tokens per second)\n",
      "llama_print_timings:        eval time =    6949.22 ms /   131 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    7338.31 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "167 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.12 ms /    40 runs   (    0.10 ms per token,  9715.81 tokens per second)\n",
      "llama_print_timings: prompt eval time =     185.64 ms /    13 tokens (   14.28 ms per token,    70.03 tokens per second)\n",
      "llama_print_timings:        eval time =    2057.91 ms /    39 runs   (   52.77 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2299.84 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "168 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.31 ms /    40 runs   (    0.11 ms per token,  9278.59 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.30 ms /    19 tokens (   10.23 ms per token,    97.79 tokens per second)\n",
      "llama_print_timings:        eval time =    2059.84 ms /    39 runs   (   52.82 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2312.04 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "169 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.26 ms /    40 runs   (    0.11 ms per token,  9385.27 tokens per second)\n",
      "llama_print_timings: prompt eval time =     190.81 ms /    17 tokens (   11.22 ms per token,    89.09 tokens per second)\n",
      "llama_print_timings:        eval time =    2058.28 ms /    39 runs   (   52.78 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2308.92 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "170 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.60 ms /   136 runs   (    0.10 ms per token, 10000.00 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.71 ms /    20 tokens (    9.19 ms per token,   108.87 tokens per second)\n",
      "llama_print_timings:        eval time =    7161.07 ms /   135 runs   (   53.04 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    7536.86 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "171 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.33 ms /   126 runs   (    0.11 ms per token,  9452.36 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.42 ms /    20 tokens (    9.17 ms per token,   109.04 tokens per second)\n",
      "llama_print_timings:        eval time =    6625.21 ms /   125 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6991.41 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "172 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.27 ms /    40 runs   (    0.11 ms per token,  9376.47 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.69 ms /    21 tokens (    9.37 ms per token,   106.76 tokens per second)\n",
      "llama_print_timings:        eval time =    2058.59 ms /    39 runs   (   52.78 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2312.28 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "173 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.57 ms /   129 runs   (    0.10 ms per token, 10260.08 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.43 ms /    28 tokens (    7.34 ms per token,   136.30 tokens per second)\n",
      "llama_print_timings:        eval time =    6796.52 ms /   128 runs   (   53.10 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    7189.47 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "174 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.20 ms /    39 runs   (    0.11 ms per token,  9283.50 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.47 ms /    19 tokens (   10.24 ms per token,    97.70 tokens per second)\n",
      "llama_print_timings:        eval time =    2003.61 ms /    38 runs   (   52.73 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    2255.37 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "175 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.30 ms /   138 runs   (    0.10 ms per token, 10372.82 tokens per second)\n",
      "llama_print_timings: prompt eval time =     208.76 ms /    31 tokens (    6.73 ms per token,   148.50 tokens per second)\n",
      "llama_print_timings:        eval time =    7280.40 ms /   137 runs   (   53.14 ms per token,    18.82 tokens per second)\n",
      "llama_print_timings:       total time =    7688.17 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "176 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.27 ms /   152 runs   (    0.09 ms per token, 10648.73 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.97 ms /    20 tokens (    9.20 ms per token,   108.71 tokens per second)\n",
      "llama_print_timings:        eval time =    8013.85 ms /   151 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    8414.80 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "177 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       5.07 ms /    49 runs   (    0.10 ms per token,  9674.23 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.37 ms /    17 tokens (   11.26 ms per token,    88.83 tokens per second)\n",
      "llama_print_timings:        eval time =    2536.86 ms /    48 runs   (   52.85 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    2797.73 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "178 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.51 ms /   129 runs   (    0.10 ms per token,  9547.78 tokens per second)\n",
      "llama_print_timings: prompt eval time =     202.51 ms /    25 tokens (    8.10 ms per token,   123.45 tokens per second)\n",
      "llama_print_timings:        eval time =    6794.75 ms /   128 runs   (   53.08 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    7185.11 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "179 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.17 ms /    40 runs   (    0.10 ms per token,  9583.13 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.72 ms /    16 tokens (   11.80 ms per token,    84.78 tokens per second)\n",
      "llama_print_timings:        eval time =    2059.20 ms /    39 runs   (   52.80 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2304.38 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "180 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.07 ms /    40 runs   (    0.10 ms per token,  9832.84 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.69 ms /    20 tokens (    9.18 ms per token,   108.88 tokens per second)\n",
      "llama_print_timings:        eval time =    2055.49 ms /    39 runs   (   52.70 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    2295.89 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "181 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.87 ms /   117 runs   (    0.10 ms per token,  9853.46 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.84 ms /    15 tokens (   12.59 ms per token,    79.43 tokens per second)\n",
      "llama_print_timings:        eval time =    6146.10 ms /   116 runs   (   52.98 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6500.27 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "182 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       5.04 ms /    45 runs   (    0.11 ms per token,  8935.66 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.55 ms /    20 tokens (    9.18 ms per token,   108.96 tokens per second)\n",
      "llama_print_timings:        eval time =    2324.63 ms /    44 runs   (   52.83 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2574.98 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "183 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.18 ms /    40 runs   (    0.10 ms per token,  9562.51 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.61 ms /    18 tokens (   10.70 ms per token,    93.45 tokens per second)\n",
      "llama_print_timings:        eval time =    2059.38 ms /    39 runs   (   52.80 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2307.61 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "184 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.86 ms /   123 runs   (    0.10 ms per token, 10370.99 tokens per second)\n",
      "llama_print_timings: prompt eval time =     363.23 ms /    47 tokens (    7.73 ms per token,   129.39 tokens per second)\n",
      "llama_print_timings:        eval time =    6483.68 ms /   122 runs   (   53.14 ms per token,    18.82 tokens per second)\n",
      "llama_print_timings:       total time =    7019.85 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "185 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.56 ms /   122 runs   (    0.09 ms per token, 10549.98 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.34 ms /    28 tokens (    7.33 ms per token,   136.36 tokens per second)\n",
      "llama_print_timings:        eval time =    6422.15 ms /   121 runs   (   53.08 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    6798.11 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "186 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      10.68 ms /   116 runs   (    0.09 ms per token, 10858.37 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.14 ms /    24 tokens (    8.34 ms per token,   119.92 tokens per second)\n",
      "llama_print_timings:        eval time =    6097.14 ms /   115 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    6462.54 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "187 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.13 ms /    40 runs   (    0.10 ms per token,  9685.23 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.17 ms /    14 tokens (   13.44 ms per token,    74.40 tokens per second)\n",
      "llama_print_timings:        eval time =    2058.23 ms /    39 runs   (   52.78 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2302.72 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "188 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.72 ms /    62 runs   (    0.11 ms per token,  9220.70 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.43 ms /    18 tokens (   10.75 ms per token,    93.06 tokens per second)\n",
      "llama_print_timings:        eval time =    3224.13 ms /    61 runs   (   52.85 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    3506.75 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "189 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.40 ms /    61 runs   (    0.10 ms per token,  9528.27 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.05 ms /    17 tokens (   11.24 ms per token,    88.98 tokens per second)\n",
      "llama_print_timings:        eval time =    3169.38 ms /    60 runs   (   52.82 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    3447.87 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "190 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.95 ms /   127 runs   (    0.10 ms per token,  9807.71 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.49 ms /    20 tokens (    9.17 ms per token,   109.00 tokens per second)\n",
      "llama_print_timings:        eval time =    6682.36 ms /   126 runs   (   53.03 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    7048.10 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "191 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.94 ms /   134 runs   (    0.10 ms per token,  9616.07 tokens per second)\n",
      "llama_print_timings: prompt eval time =     186.56 ms /    15 tokens (   12.44 ms per token,    80.40 tokens per second)\n",
      "llama_print_timings:        eval time =    7052.80 ms /   133 runs   (   53.03 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    7430.51 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "192 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.94 ms /   137 runs   (    0.10 ms per token,  9827.83 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.24 ms /    23 tokens (    8.66 ms per token,   115.44 tokens per second)\n",
      "llama_print_timings:        eval time =    7217.40 ms /   136 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    7614.08 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "193 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.24 ms /   131 runs   (    0.09 ms per token, 10699.99 tokens per second)\n",
      "llama_print_timings: prompt eval time =     204.09 ms /    32 tokens (    6.38 ms per token,   156.80 tokens per second)\n",
      "llama_print_timings:        eval time =    6907.99 ms /   130 runs   (   53.14 ms per token,    18.82 tokens per second)\n",
      "llama_print_timings:       total time =    7296.18 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "194 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.39 ms /   119 runs   (    0.10 ms per token,  9601.42 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.24 ms /    16 tokens (   11.83 ms per token,    84.55 tokens per second)\n",
      "llama_print_timings:        eval time =    6254.67 ms /   118 runs   (   53.01 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6613.41 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "195 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.21 ms /    40 runs   (    0.11 ms per token,  9510.22 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.73 ms /    18 tokens (   10.71 ms per token,    93.40 tokens per second)\n",
      "llama_print_timings:        eval time =    2056.85 ms /    39 runs   (   52.74 ms per token,    18.96 tokens per second)\n",
      "llama_print_timings:       total time =    2306.59 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "196 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.72 ms /   146 runs   (    0.10 ms per token,  9918.48 tokens per second)\n",
      "llama_print_timings: prompt eval time =     207.44 ms /    29 tokens (    7.15 ms per token,   139.80 tokens per second)\n",
      "llama_print_timings:        eval time =    7705.18 ms /   145 runs   (   53.14 ms per token,    18.82 tokens per second)\n",
      "llama_print_timings:       total time =    8123.49 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "197 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.26 ms /    40 runs   (    0.11 ms per token,  9398.50 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.41 ms /    15 tokens (   12.56 ms per token,    79.61 tokens per second)\n",
      "llama_print_timings:        eval time =    2059.44 ms /    39 runs   (   52.81 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2303.74 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.43 ms /    39 runs   (    0.11 ms per token,  8801.62 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.24 ms /    19 tokens (   10.22 ms per token,    97.82 tokens per second)\n",
      "llama_print_timings:        eval time =    2004.49 ms /    38 runs   (   52.75 ms per token,    18.96 tokens per second)\n",
      "llama_print_timings:       total time =    2256.54 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "198 -- 1\n",
      "199 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.88 ms /   129 runs   (    0.10 ms per token, 10014.75 tokens per second)\n",
      "llama_print_timings: prompt eval time =     209.62 ms /    30 tokens (    6.99 ms per token,   143.11 tokens per second)\n",
      "llama_print_timings:        eval time =    6798.28 ms /   128 runs   (   53.11 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    7189.73 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.07 ms /   141 runs   (    0.09 ms per token, 10789.72 tokens per second)\n",
      "llama_print_timings: prompt eval time =     345.01 ms /    34 tokens (   10.15 ms per token,    98.55 tokens per second)\n",
      "llama_print_timings:        eval time =    7432.95 ms /   140 runs   (   53.09 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    7974.86 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "201 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.05 ms /   126 runs   (    0.10 ms per token,  9651.47 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.89 ms /    27 tokens (    7.63 ms per token,   131.14 tokens per second)\n",
      "llama_print_timings:        eval time =    6637.40 ms /   125 runs   (   53.10 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    7024.69 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "202 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.14 ms /    40 runs   (    0.10 ms per token,  9659.50 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.36 ms /    16 tokens (   11.77 ms per token,    84.94 tokens per second)\n",
      "llama_print_timings:        eval time =    2056.17 ms /    39 runs   (   52.72 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    2301.44 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "203 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.37 ms /    40 runs   (    0.11 ms per token,  9149.13 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.77 ms /    12 tokens (   15.31 ms per token,    65.30 tokens per second)\n",
      "llama_print_timings:        eval time =    2060.66 ms /    39 runs   (   52.84 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2303.84 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "204 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.80 ms /   142 runs   (    0.10 ms per token,  9593.95 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.50 ms /    23 tokens (    8.67 ms per token,   115.29 tokens per second)\n",
      "llama_print_timings:        eval time =    7487.57 ms /   141 runs   (   53.10 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    7890.86 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "205 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.16 ms /    39 runs   (    0.11 ms per token,  9381.77 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.13 ms /    20 tokens (    9.76 ms per token,   102.50 tokens per second)\n",
      "llama_print_timings:        eval time =    2005.40 ms /    38 runs   (   52.77 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2255.31 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "206 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.14 ms /   123 runs   (    0.10 ms per token, 10130.13 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.15 ms /    19 tokens (   10.22 ms per token,    97.86 tokens per second)\n",
      "llama_print_timings:        eval time =    6468.60 ms /   122 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    6835.94 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "207 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.16 ms /    40 runs   (    0.10 ms per token,  9613.07 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.47 ms /    15 tokens (   12.56 ms per token,    79.59 tokens per second)\n",
      "llama_print_timings:        eval time =    2060.30 ms /    39 runs   (   52.83 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2305.70 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "208 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.07 ms /    39 runs   (    0.10 ms per token,  9582.31 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.51 ms /    23 tokens (    8.67 ms per token,   115.28 tokens per second)\n",
      "llama_print_timings:        eval time =    2006.62 ms /    38 runs   (   52.81 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2261.55 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "209 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.07 ms /   130 runs   (    0.10 ms per token,  9944.92 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.13 ms /    23 tokens (    8.66 ms per token,   115.50 tokens per second)\n",
      "llama_print_timings:        eval time =    6840.38 ms /   129 runs   (   53.03 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    7222.34 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "210 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.75 ms /   113 runs   (    0.10 ms per token,  9614.57 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.38 ms /    22 tokens (    9.02 ms per token,   110.90 tokens per second)\n",
      "llama_print_timings:        eval time =    5940.02 ms /   112 runs   (   53.04 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    6297.94 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "211 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.53 ms /    40 runs   (    0.11 ms per token,  8831.97 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.52 ms /    23 tokens (    8.67 ms per token,   115.27 tokens per second)\n",
      "llama_print_timings:        eval time =    2059.94 ms /    39 runs   (   52.82 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2321.77 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "212 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.59 ms /   133 runs   (    0.10 ms per token,  9786.61 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.26 ms /    19 tokens (   10.22 ms per token,    97.81 tokens per second)\n",
      "llama_print_timings:        eval time =    6999.29 ms /   132 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    7382.09 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "213 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.65 ms /    40 runs   (    0.12 ms per token,  8602.15 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.36 ms /    17 tokens (   11.26 ms per token,    88.84 tokens per second)\n",
      "llama_print_timings:        eval time =    2061.24 ms /    39 runs   (   52.85 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    2314.38 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "214 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.52 ms /    40 runs   (    0.11 ms per token,  8847.60 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.79 ms /    15 tokens (   12.59 ms per token,    79.45 tokens per second)\n",
      "llama_print_timings:        eval time =    2059.90 ms /    39 runs   (   52.82 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2309.15 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "215 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.30 ms /   141 runs   (    0.09 ms per token, 10603.90 tokens per second)\n",
      "llama_print_timings: prompt eval time =     353.15 ms /    39 tokens (    9.06 ms per token,   110.43 tokens per second)\n",
      "llama_print_timings:        eval time =    7439.99 ms /   140 runs   (   53.14 ms per token,    18.82 tokens per second)\n",
      "llama_print_timings:       total time =    7991.13 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.05 ms /    39 runs   (    0.10 ms per token,  9629.63 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.38 ms /    24 tokens (    8.35 ms per token,   119.77 tokens per second)\n",
      "llama_print_timings:        eval time =    2005.47 ms /    38 runs   (   52.78 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2261.32 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "217 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.08 ms /    40 runs   (    0.10 ms per token,  9803.92 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.46 ms /    15 tokens (   12.56 ms per token,    79.59 tokens per second)\n",
      "llama_print_timings:        eval time =    2057.54 ms /    39 runs   (   52.76 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2302.71 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "218 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.27 ms /    39 runs   (    0.11 ms per token,  9137.77 tokens per second)\n",
      "llama_print_timings: prompt eval time =     185.43 ms /    13 tokens (   14.26 ms per token,    70.11 tokens per second)\n",
      "llama_print_timings:        eval time =    2005.49 ms /    38 runs   (   52.78 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2247.87 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "219 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.89 ms /    67 runs   (    0.10 ms per token,  9724.24 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.79 ms /    15 tokens (   12.59 ms per token,    79.45 tokens per second)\n",
      "llama_print_timings:        eval time =    3488.89 ms /    66 runs   (   52.86 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    3770.88 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "220 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.28 ms /    40 runs   (    0.11 ms per token,  9352.35 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.41 ms /    20 tokens (    9.17 ms per token,   109.04 tokens per second)\n",
      "llama_print_timings:        eval time =    2057.66 ms /    39 runs   (   52.76 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2298.02 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "221 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.92 ms /   132 runs   (    0.11 ms per token,  9484.12 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.70 ms /    23 tokens (    8.68 ms per token,   115.17 tokens per second)\n",
      "llama_print_timings:        eval time =    6948.65 ms /   131 runs   (   53.04 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    7336.97 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "222 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.13 ms /    40 runs   (    0.10 ms per token,  9675.86 tokens per second)\n",
      "llama_print_timings: prompt eval time =     204.10 ms /    26 tokens (    7.85 ms per token,   127.39 tokens per second)\n",
      "llama_print_timings:        eval time =    2062.18 ms /    39 runs   (   52.88 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    2323.44 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "223 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.95 ms /   125 runs   (    0.10 ms per token,  9655.49 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.94 ms /    18 tokens (   10.72 ms per token,    93.30 tokens per second)\n",
      "llama_print_timings:        eval time =    6572.89 ms /   124 runs   (   53.01 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6942.82 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "224 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.46 ms /   146 runs   (    0.10 ms per token, 10098.22 tokens per second)\n",
      "llama_print_timings: prompt eval time =     345.03 ms /    33 tokens (   10.46 ms per token,    95.64 tokens per second)\n",
      "llama_print_timings:        eval time =    7708.91 ms /   145 runs   (   53.16 ms per token,    18.81 tokens per second)\n",
      "llama_print_timings:       total time =    8261.71 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "225 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.72 ms /   131 runs   (    0.10 ms per token,  9546.71 tokens per second)\n",
      "llama_print_timings: prompt eval time =     187.44 ms /    14 tokens (   13.39 ms per token,    74.69 tokens per second)\n",
      "llama_print_timings:        eval time =    6896.94 ms /   130 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    7272.42 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "226 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       7.84 ms /    71 runs   (    0.11 ms per token,  9052.66 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.91 ms /    18 tokens (   10.72 ms per token,    93.31 tokens per second)\n",
      "llama_print_timings:        eval time =    3700.93 ms /    70 runs   (   52.87 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    3997.92 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "227 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      10.61 ms /   103 runs   (    0.10 ms per token,  9705.08 tokens per second)\n",
      "llama_print_timings: prompt eval time =     182.91 ms /    20 tokens (    9.15 ms per token,   109.34 tokens per second)\n",
      "llama_print_timings:        eval time =    5399.27 ms /   102 runs   (   52.93 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    5728.93 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "228 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.30 ms /    40 runs   (    0.11 ms per token,  9304.49 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.52 ms /    12 tokens (   15.29 ms per token,    65.39 tokens per second)\n",
      "llama_print_timings:        eval time =    2058.45 ms /    39 runs   (   52.78 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2299.16 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "229 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.29 ms /    40 runs   (    0.11 ms per token,  9324.01 tokens per second)\n",
      "llama_print_timings: prompt eval time =     185.52 ms /    13 tokens (   14.27 ms per token,    70.07 tokens per second)\n",
      "llama_print_timings:        eval time =    2058.58 ms /    39 runs   (   52.78 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2302.54 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "230 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.26 ms /    40 runs   (    0.11 ms per token,  9387.47 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.44 ms /    17 tokens (   11.26 ms per token,    88.80 tokens per second)\n",
      "llama_print_timings:        eval time =    2059.88 ms /    39 runs   (   52.82 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2309.31 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "231 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.41 ms /    39 runs   (    0.11 ms per token,  8837.53 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.49 ms /    20 tokens (    9.17 ms per token,   109.00 tokens per second)\n",
      "llama_print_timings:        eval time =    2005.66 ms /    38 runs   (   52.78 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2247.62 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "232 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.68 ms /   125 runs   (    0.10 ms per token,  9855.71 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.02 ms /    24 tokens (    8.33 ms per token,   119.99 tokens per second)\n",
      "llama_print_timings:        eval time =    6578.69 ms /   124 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    6955.29 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "233 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.28 ms /    64 runs   (    0.10 ms per token, 10192.71 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.60 ms /    23 tokens (    8.63 ms per token,   115.81 tokens per second)\n",
      "llama_print_timings:        eval time =    3336.41 ms /    63 runs   (   52.96 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3623.99 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "234 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.28 ms /    40 runs   (    0.11 ms per token,  9341.43 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.46 ms /    15 tokens (   12.56 ms per token,    79.59 tokens per second)\n",
      "llama_print_timings:        eval time =    2059.32 ms /    39 runs   (   52.80 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2305.60 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "235 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.73 ms /   138 runs   (    0.10 ms per token, 10053.18 tokens per second)\n",
      "llama_print_timings: prompt eval time =     211.36 ms /    31 tokens (    6.82 ms per token,   146.67 tokens per second)\n",
      "llama_print_timings:        eval time =    7273.19 ms /   137 runs   (   53.09 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    7678.97 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "236 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.46 ms /    39 runs   (    0.11 ms per token,  8744.39 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.32 ms /    16 tokens (   11.77 ms per token,    84.96 tokens per second)\n",
      "llama_print_timings:        eval time =    2006.52 ms /    38 runs   (   52.80 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2253.32 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "237 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.80 ms /   125 runs   (    0.10 ms per token,  9765.62 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.24 ms /    23 tokens (    8.66 ms per token,   115.44 tokens per second)\n",
      "llama_print_timings:        eval time =    6578.15 ms /   124 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    6961.63 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "238 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.30 ms /    40 runs   (    0.11 ms per token,  9291.52 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.09 ms /    21 tokens (    9.34 ms per token,   107.09 tokens per second)\n",
      "llama_print_timings:        eval time =    2060.52 ms /    39 runs   (   52.83 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2313.58 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "239 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      10.89 ms /   102 runs   (    0.11 ms per token,  9369.83 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.48 ms /    15 tokens (   12.57 ms per token,    79.58 tokens per second)\n",
      "llama_print_timings:        eval time =    5348.79 ms /   101 runs   (   52.96 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    5683.59 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "240 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.11 ms /   108 runs   (    0.10 ms per token,  9717.47 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.13 ms /    21 tokens (    9.34 ms per token,   107.07 tokens per second)\n",
      "llama_print_timings:        eval time =    5675.94 ms /   107 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    6027.87 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "241 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.95 ms /   113 runs   (    0.11 ms per token,  9458.44 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.41 ms /    24 tokens (    8.35 ms per token,   119.76 tokens per second)\n",
      "llama_print_timings:        eval time =    5935.83 ms /   112 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6299.97 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "242 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.83 ms /   133 runs   (    0.10 ms per token,  9620.25 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.53 ms /    20 tokens (    9.18 ms per token,   108.98 tokens per second)\n",
      "llama_print_timings:        eval time =    6997.78 ms /   132 runs   (   53.01 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    7369.44 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "243 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      16.48 ms /   163 runs   (    0.10 ms per token,  9891.98 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.38 ms /    20 tokens (    9.17 ms per token,   109.06 tokens per second)\n",
      "llama_print_timings:        eval time =    8601.30 ms /   162 runs   (   53.09 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    9016.24 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "244 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.60 ms /    40 runs   (    0.11 ms per token,  8703.22 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.69 ms /    16 tokens (   11.79 ms per token,    84.79 tokens per second)\n",
      "llama_print_timings:        eval time =    2060.16 ms /    39 runs   (   52.82 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2310.83 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "245 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.30 ms /    40 runs   (    0.11 ms per token,  9304.49 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.11 ms /    15 tokens (   12.61 ms per token,    79.32 tokens per second)\n",
      "llama_print_timings:        eval time =    2059.00 ms /    39 runs   (   52.79 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2305.77 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "246 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      15.93 ms /   156 runs   (    0.10 ms per token,  9792.84 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.69 ms /    16 tokens (   11.79 ms per token,    84.79 tokens per second)\n",
      "llama_print_timings:        eval time =    8223.74 ms /   155 runs   (   53.06 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    8634.16 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "247 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.94 ms /   134 runs   (    0.10 ms per token,  9613.32 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.50 ms /    20 tokens (    9.18 ms per token,   108.99 tokens per second)\n",
      "llama_print_timings:        eval time =    7053.21 ms /   133 runs   (   53.03 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    7430.53 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "248 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.61 ms /    64 runs   (    0.10 ms per token,  9688.16 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.51 ms /    18 tokens (   10.70 ms per token,    93.50 tokens per second)\n",
      "llama_print_timings:        eval time =    3324.44 ms /    63 runs   (   52.77 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    3606.60 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "249 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.04 ms /   122 runs   (    0.10 ms per token, 10132.89 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.42 ms /    23 tokens (    8.71 ms per token,   114.76 tokens per second)\n",
      "llama_print_timings:        eval time =    6418.26 ms /   121 runs   (   53.04 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    6793.35 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "250 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.19 ms /   125 runs   (    0.10 ms per token, 10252.62 tokens per second)\n",
      "llama_print_timings: prompt eval time =     207.02 ms /    29 tokens (    7.14 ms per token,   140.08 tokens per second)\n",
      "llama_print_timings:        eval time =    6581.30 ms /   124 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    6967.56 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "251 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.83 ms /   114 runs   (    0.10 ms per token,  9635.70 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.78 ms /    18 tokens (   10.71 ms per token,    93.37 tokens per second)\n",
      "llama_print_timings:        eval time =    5989.46 ms /   113 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6342.81 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "252 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.16 ms /   122 runs   (    0.11 ms per token,  9269.81 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.66 ms /    15 tokens (   12.58 ms per token,    79.51 tokens per second)\n",
      "llama_print_timings:        eval time =    6413.71 ms /   121 runs   (   53.01 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6780.07 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "253 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.70 ms /   121 runs   (    0.10 ms per token,  9524.56 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.19 ms /    16 tokens (   11.76 ms per token,    85.02 tokens per second)\n",
      "llama_print_timings:        eval time =    6362.21 ms /   120 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    6723.84 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "254 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.64 ms /   114 runs   (    0.11 ms per token,  9021.13 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.37 ms /    19 tokens (   10.23 ms per token,    97.75 tokens per second)\n",
      "llama_print_timings:        eval time =    5992.61 ms /   113 runs   (   53.03 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    6356.57 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "255 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.24 ms /    40 runs   (    0.11 ms per token,  9425.07 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.96 ms /    19 tokens (   10.21 ms per token,    97.96 tokens per second)\n",
      "llama_print_timings:        eval time =    2060.19 ms /    39 runs   (   52.83 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2311.85 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "256 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      10.76 ms /   103 runs   (    0.10 ms per token,  9568.93 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.92 ms /    20 tokens (    9.20 ms per token,   108.74 tokens per second)\n",
      "llama_print_timings:        eval time =    5405.32 ms /   102 runs   (   52.99 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    5732.82 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "257 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.34 ms /   132 runs   (    0.10 ms per token,  9898.76 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.94 ms /    21 tokens (    9.33 ms per token,   107.17 tokens per second)\n",
      "llama_print_timings:        eval time =    6951.78 ms /   131 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    7336.63 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "258 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.28 ms /   130 runs   (    0.10 ms per token,  9789.16 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.02 ms /    23 tokens (    8.65 ms per token,   115.57 tokens per second)\n",
      "llama_print_timings:        eval time =    6846.08 ms /   129 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    7233.03 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "259 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.60 ms /   140 runs   (    0.10 ms per token,  9588.38 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.72 ms /    21 tokens (    9.32 ms per token,   107.29 tokens per second)\n",
      "llama_print_timings:        eval time =    7375.38 ms /   139 runs   (   53.06 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    7771.50 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "260 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.74 ms /   143 runs   (    0.10 ms per token,  9700.83 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.08 ms /    19 tokens (   10.27 ms per token,    97.40 tokens per second)\n",
      "llama_print_timings:        eval time =    7526.75 ms /   142 runs   (   53.01 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    7921.88 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "261 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.00 ms /    39 runs   (    0.10 ms per token,  9757.32 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.95 ms /    22 tokens (    9.00 ms per token,   111.14 tokens per second)\n",
      "llama_print_timings:        eval time =    2001.25 ms /    38 runs   (   52.66 ms per token,    18.99 tokens per second)\n",
      "llama_print_timings:       total time =    2252.50 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "262 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.63 ms /    67 runs   (    0.10 ms per token, 10107.11 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.89 ms /    11 tokens (   16.72 ms per token,    59.82 tokens per second)\n",
      "llama_print_timings:        eval time =    3479.53 ms /    66 runs   (   52.72 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    3754.93 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "263 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.70 ms /   116 runs   (    0.10 ms per token,  9914.53 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.83 ms /    15 tokens (   12.59 ms per token,    79.44 tokens per second)\n",
      "llama_print_timings:        eval time =    6087.32 ms /   115 runs   (   52.93 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    6438.90 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "264 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.10 ms /    40 runs   (    0.10 ms per token,  9753.72 tokens per second)\n",
      "llama_print_timings: prompt eval time =     190.79 ms /    17 tokens (   11.22 ms per token,    89.11 tokens per second)\n",
      "llama_print_timings:        eval time =    2053.99 ms /    39 runs   (   52.67 ms per token,    18.99 tokens per second)\n",
      "llama_print_timings:       total time =    2299.05 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "265 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.97 ms /    40 runs   (    0.10 ms per token, 10085.73 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.39 ms /    22 tokens (    9.02 ms per token,   110.89 tokens per second)\n",
      "llama_print_timings:        eval time =    2055.19 ms /    39 runs   (   52.70 ms per token,    18.98 tokens per second)\n",
      "llama_print_timings:       total time =    2307.97 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "266 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.99 ms /    40 runs   (    0.10 ms per token, 10027.58 tokens per second)\n",
      "llama_print_timings: prompt eval time =     187.30 ms /    14 tokens (   13.38 ms per token,    74.74 tokens per second)\n",
      "llama_print_timings:        eval time =    2052.74 ms /    39 runs   (   52.63 ms per token,    19.00 tokens per second)\n",
      "llama_print_timings:       total time =    2295.09 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "267 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       7.00 ms /    68 runs   (    0.10 ms per token,  9712.90 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.02 ms /    17 tokens (   11.24 ms per token,    89.00 tokens per second)\n",
      "llama_print_timings:        eval time =    3540.83 ms /    67 runs   (   52.85 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    3826.13 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "268 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.35 ms /   108 runs   (    0.11 ms per token,  9512.07 tokens per second)\n",
      "llama_print_timings: prompt eval time =     187.62 ms /    14 tokens (   13.40 ms per token,    74.62 tokens per second)\n",
      "llama_print_timings:        eval time =    5659.77 ms /   107 runs   (   52.90 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    5997.52 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "269 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.06 ms /    40 runs   (    0.10 ms per token,  9859.50 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.86 ms /    16 tokens (   11.80 ms per token,    84.72 tokens per second)\n",
      "llama_print_timings:        eval time =    2053.03 ms /    39 runs   (   52.64 ms per token,    19.00 tokens per second)\n",
      "llama_print_timings:       total time =    2296.08 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "270 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.10 ms /    40 runs   (    0.10 ms per token,  9765.62 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.22 ms /    24 tokens (    8.34 ms per token,   119.87 tokens per second)\n",
      "llama_print_timings:        eval time =    2058.07 ms /    39 runs   (   52.77 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2312.76 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "271 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.74 ms /    45 runs   (    0.11 ms per token,  9487.67 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.23 ms /    19 tokens (   10.22 ms per token,    97.82 tokens per second)\n",
      "llama_print_timings:        eval time =    2317.89 ms /    44 runs   (   52.68 ms per token,    18.98 tokens per second)\n",
      "llama_print_timings:       total time =    2575.83 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "272 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.26 ms /    40 runs   (    0.11 ms per token,  9396.29 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.80 ms /    18 tokens (   10.71 ms per token,    93.36 tokens per second)\n",
      "llama_print_timings:        eval time =    2054.81 ms /    39 runs   (   52.69 ms per token,    18.98 tokens per second)\n",
      "llama_print_timings:       total time =    2304.24 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "273 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.47 ms /   120 runs   (    0.10 ms per token,  9621.55 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.78 ms /    19 tokens (   10.20 ms per token,    98.05 tokens per second)\n",
      "llama_print_timings:        eval time =    6300.76 ms /   119 runs   (   52.95 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    6661.44 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "274 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.49 ms /   124 runs   (    0.09 ms per token, 10795.75 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.78 ms /    27 tokens (    7.62 ms per token,   131.21 tokens per second)\n",
      "llama_print_timings:        eval time =    6520.11 ms /   123 runs   (   53.01 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    6897.67 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "275 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.06 ms /    40 runs   (    0.10 ms per token,  9852.22 tokens per second)\n",
      "llama_print_timings: prompt eval time =     190.35 ms /    17 tokens (   11.20 ms per token,    89.31 tokens per second)\n",
      "llama_print_timings:        eval time =    2054.20 ms /    39 runs   (   52.67 ms per token,    18.99 tokens per second)\n",
      "llama_print_timings:       total time =    2299.32 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.67 ms /   140 runs   (    0.10 ms per token, 10244.40 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.28 ms /    22 tokens (    9.01 ms per token,   110.95 tokens per second)\n",
      "llama_print_timings:        eval time =    7363.78 ms /   139 runs   (   52.98 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    7761.43 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "276 -- 1\n",
      "277 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.01 ms /    39 runs   (    0.10 ms per token,  9735.40 tokens per second)\n",
      "llama_print_timings: prompt eval time =     184.06 ms /    20 tokens (    9.20 ms per token,   108.66 tokens per second)\n",
      "llama_print_timings:        eval time =    2000.64 ms /    38 runs   (   52.65 ms per token,    18.99 tokens per second)\n",
      "llama_print_timings:       total time =    2238.39 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "278 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.05 ms /    40 runs   (    0.10 ms per token,  9883.86 tokens per second)\n",
      "llama_print_timings: prompt eval time =     190.92 ms /    17 tokens (   11.23 ms per token,    89.04 tokens per second)\n",
      "llama_print_timings:        eval time =    2053.59 ms /    39 runs   (   52.66 ms per token,    18.99 tokens per second)\n",
      "llama_print_timings:       total time =    2299.49 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "279 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.55 ms /   118 runs   (    0.10 ms per token, 10218.22 tokens per second)\n",
      "llama_print_timings: prompt eval time =     187.37 ms /    14 tokens (   13.38 ms per token,    74.72 tokens per second)\n",
      "llama_print_timings:        eval time =    6193.06 ms /   117 runs   (   52.93 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    6543.41 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "280 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.11 ms /   132 runs   (    0.10 ms per token, 10065.58 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.25 ms /    21 tokens (    9.35 ms per token,   107.01 tokens per second)\n",
      "llama_print_timings:        eval time =    6947.74 ms /   131 runs   (   53.04 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    7328.03 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "281 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.41 ms /   122 runs   (    0.10 ms per token,  9831.57 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.75 ms /    18 tokens (   10.71 ms per token,    93.38 tokens per second)\n",
      "llama_print_timings:        eval time =    6402.49 ms /   121 runs   (   52.91 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    6766.50 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "282 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.64 ms /   136 runs   (    0.10 ms per token,  9972.14 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.09 ms /    16 tokens (   11.76 ms per token,    85.07 tokens per second)\n",
      "llama_print_timings:        eval time =    7149.06 ms /   135 runs   (   52.96 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    7526.41 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "283 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.09 ms /    40 runs   (    0.10 ms per token,  9782.34 tokens per second)\n",
      "llama_print_timings: prompt eval time =     187.43 ms /    14 tokens (   13.39 ms per token,    74.69 tokens per second)\n",
      "llama_print_timings:        eval time =    2056.29 ms /    39 runs   (   52.73 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    2298.19 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "284 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.29 ms /    39 runs   (    0.11 ms per token,  9097.27 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.54 ms /    16 tokens (   11.78 ms per token,    84.86 tokens per second)\n",
      "llama_print_timings:        eval time =    2000.51 ms /    38 runs   (   52.64 ms per token,    19.00 tokens per second)\n",
      "llama_print_timings:       total time =    2247.59 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "285 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      10.29 ms /   102 runs   (    0.10 ms per token,  9916.39 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.96 ms /    22 tokens (    9.00 ms per token,   111.13 tokens per second)\n",
      "llama_print_timings:        eval time =    5342.62 ms /   101 runs   (   52.90 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    5682.09 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "286 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.16 ms /    40 runs   (    0.10 ms per token,  9624.64 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.31 ms /    21 tokens (    9.35 ms per token,   106.97 tokens per second)\n",
      "llama_print_timings:        eval time =    2050.79 ms /    39 runs   (   52.58 ms per token,    19.02 tokens per second)\n",
      "llama_print_timings:       total time =    2301.98 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "287 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.51 ms /    44 runs   (    0.10 ms per token,  9749.61 tokens per second)\n",
      "llama_print_timings: prompt eval time =     190.93 ms /    17 tokens (   11.23 ms per token,    89.04 tokens per second)\n",
      "llama_print_timings:        eval time =    2264.86 ms /    43 runs   (   52.67 ms per token,    18.99 tokens per second)\n",
      "llama_print_timings:       total time =    2516.33 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "288 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.68 ms /    45 runs   (    0.10 ms per token,  9611.28 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.87 ms /    23 tokens (    8.69 ms per token,   115.08 tokens per second)\n",
      "llama_print_timings:        eval time =    2321.36 ms /    44 runs   (   52.76 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2584.33 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "289 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.77 ms /    44 runs   (    0.11 ms per token,  9222.39 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.16 ms /    20 tokens (    9.16 ms per token,   109.19 tokens per second)\n",
      "llama_print_timings:        eval time =    2268.90 ms /    43 runs   (   52.77 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2514.03 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "290 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.36 ms /   116 runs   (    0.10 ms per token, 10214.86 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.05 ms /    23 tokens (    8.65 ms per token,   115.55 tokens per second)\n",
      "llama_print_timings:        eval time =    6084.24 ms /   115 runs   (   52.91 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    6444.37 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "291 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.99 ms /    39 runs   (    0.10 ms per token,  9784.24 tokens per second)\n",
      "llama_print_timings: prompt eval time =     185.49 ms /    13 tokens (   14.27 ms per token,    70.08 tokens per second)\n",
      "llama_print_timings:        eval time =    2000.73 ms /    38 runs   (   52.65 ms per token,    18.99 tokens per second)\n",
      "llama_print_timings:       total time =    2240.03 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "292 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       7.65 ms /    78 runs   (    0.10 ms per token, 10201.41 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.10 ms /    16 tokens (   11.76 ms per token,    85.06 tokens per second)\n",
      "llama_print_timings:        eval time =    4067.57 ms /    77 runs   (   52.83 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    4362.96 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "293 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.02 ms /   127 runs   (    0.09 ms per token, 11523.46 tokens per second)\n",
      "llama_print_timings: prompt eval time =     211.34 ms /    31 tokens (    6.82 ms per token,   146.69 tokens per second)\n",
      "llama_print_timings:        eval time =    6683.73 ms /   126 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    7070.22 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "294 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.12 ms /    59 runs   (    0.10 ms per token,  9648.41 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.77 ms /    15 tokens (   12.58 ms per token,    79.46 tokens per second)\n",
      "llama_print_timings:        eval time =    3061.45 ms /    58 runs   (   52.78 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    3332.17 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "295 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.00 ms /    39 runs   (    0.10 ms per token,  9759.76 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.04 ms /    24 tokens (    8.34 ms per token,   119.97 tokens per second)\n",
      "llama_print_timings:        eval time =    2001.80 ms /    38 runs   (   52.68 ms per token,    18.98 tokens per second)\n",
      "llama_print_timings:       total time =    2255.75 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "296 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       5.02 ms /    49 runs   (    0.10 ms per token,  9759.01 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.70 ms /    19 tokens (   10.19 ms per token,    98.09 tokens per second)\n",
      "llama_print_timings:        eval time =    2530.33 ms /    48 runs   (   52.72 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    2791.16 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "297 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.88 ms /   137 runs   (    0.10 ms per token,  9869.61 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.73 ms /    19 tokens (   10.20 ms per token,    98.07 tokens per second)\n",
      "llama_print_timings:        eval time =    7210.42 ms /   136 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    7594.97 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "298 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.11 ms /    40 runs   (    0.10 ms per token,  9737.10 tokens per second)\n",
      "llama_print_timings: prompt eval time =     182.08 ms /    10 tokens (   18.21 ms per token,    54.92 tokens per second)\n",
      "llama_print_timings:        eval time =    2051.63 ms /    39 runs   (   52.61 ms per token,    19.01 tokens per second)\n",
      "llama_print_timings:       total time =    2289.46 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "299 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.86 ms /   125 runs   (    0.10 ms per token,  9719.31 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.06 ms /    21 tokens (    9.34 ms per token,   107.11 tokens per second)\n",
      "llama_print_timings:        eval time =    6570.03 ms /   124 runs   (   52.98 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6945.85 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.44 ms /   152 runs   (    0.10 ms per token, 10524.86 tokens per second)\n",
      "llama_print_timings: prompt eval time =     344.81 ms /    34 tokens (   10.14 ms per token,    98.60 tokens per second)\n",
      "llama_print_timings:        eval time =    8020.32 ms /   151 runs   (   53.11 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    8577.82 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "301 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.89 ms /    39 runs   (    0.10 ms per token, 10017.98 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.82 ms /    11 tokens (   16.71 ms per token,    59.84 tokens per second)\n",
      "llama_print_timings:        eval time =    2000.31 ms /    38 runs   (   52.64 ms per token,    19.00 tokens per second)\n",
      "llama_print_timings:       total time =    2237.02 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "302 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.88 ms /   130 runs   (    0.10 ms per token, 10097.09 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.18 ms /    18 tokens (   10.73 ms per token,    93.18 tokens per second)\n",
      "llama_print_timings:        eval time =    6828.25 ms /   129 runs   (   52.93 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    7201.63 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "303 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.84 ms /    40 runs   (    0.10 ms per token, 10413.95 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.05 ms /    19 tokens (   10.21 ms per token,    97.91 tokens per second)\n",
      "llama_print_timings:        eval time =    2053.05 ms /    39 runs   (   52.64 ms per token,    19.00 tokens per second)\n",
      "llama_print_timings:       total time =    2301.67 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "304 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.07 ms /    40 runs   (    0.10 ms per token,  9828.01 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.58 ms /    15 tokens (   12.57 ms per token,    79.54 tokens per second)\n",
      "llama_print_timings:        eval time =    2052.38 ms /    39 runs   (   52.62 ms per token,    19.00 tokens per second)\n",
      "llama_print_timings:       total time =    2295.73 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "305 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.09 ms /   122 runs   (    0.10 ms per token, 10095.16 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.57 ms /    18 tokens (   10.70 ms per token,    93.47 tokens per second)\n",
      "llama_print_timings:        eval time =    6404.48 ms /   121 runs   (   52.93 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    6766.95 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "306 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      10.53 ms /   108 runs   (    0.10 ms per token, 10255.44 tokens per second)\n",
      "llama_print_timings: prompt eval time =     203.98 ms /    32 tokens (    6.37 ms per token,   156.88 tokens per second)\n",
      "llama_print_timings:        eval time =    5671.73 ms /   107 runs   (   53.01 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6028.32 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "307 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      10.71 ms /   114 runs   (    0.09 ms per token, 10642.27 tokens per second)\n",
      "llama_print_timings: prompt eval time =     204.15 ms /    26 tokens (    7.85 ms per token,   127.36 tokens per second)\n",
      "llama_print_timings:        eval time =    5989.81 ms /   113 runs   (   53.01 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6354.03 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "308 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.40 ms /   114 runs   (    0.10 ms per token, 10000.00 tokens per second)\n",
      "llama_print_timings: prompt eval time =     184.38 ms /    20 tokens (    9.22 ms per token,   108.47 tokens per second)\n",
      "llama_print_timings:        eval time =    5981.49 ms /   113 runs   (   52.93 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    6324.60 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "309 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.98 ms /    40 runs   (    0.10 ms per token, 10037.64 tokens per second)\n",
      "llama_print_timings: prompt eval time =     187.72 ms /    14 tokens (   13.41 ms per token,    74.58 tokens per second)\n",
      "llama_print_timings:        eval time =    2053.56 ms /    39 runs   (   52.66 ms per token,    18.99 tokens per second)\n",
      "llama_print_timings:       total time =    2295.10 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "310 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.50 ms /    71 runs   (    0.09 ms per token, 10928.12 tokens per second)\n",
      "llama_print_timings: prompt eval time =     345.99 ms /    37 tokens (    9.35 ms per token,   106.94 tokens per second)\n",
      "llama_print_timings:        eval time =    3704.11 ms /    70 runs   (   52.92 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    4146.72 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "311 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.68 ms /    45 runs   (    0.10 ms per token,  9609.22 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.17 ms /    19 tokens (   10.22 ms per token,    97.85 tokens per second)\n",
      "llama_print_timings:        eval time =    2318.92 ms /    44 runs   (   52.70 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    2575.83 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "312 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.73 ms /   124 runs   (    0.10 ms per token,  9736.95 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.32 ms /    19 tokens (   10.23 ms per token,    97.78 tokens per second)\n",
      "llama_print_timings:        eval time =    6516.21 ms /   123 runs   (   52.98 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    6883.85 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "313 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.82 ms /   122 runs   (    0.11 ms per token,  9518.61 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.28 ms /    20 tokens (    9.16 ms per token,   109.12 tokens per second)\n",
      "llama_print_timings:        eval time =    6383.78 ms /   121 runs   (   52.76 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    6739.15 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "314 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.60 ms /    45 runs   (    0.10 ms per token,  9784.74 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.65 ms /    18 tokens (   10.70 ms per token,    93.43 tokens per second)\n",
      "llama_print_timings:        eval time =    2316.64 ms /    44 runs   (   52.65 ms per token,    18.99 tokens per second)\n",
      "llama_print_timings:       total time =    2571.08 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "315 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.16 ms /    40 runs   (    0.10 ms per token,  9622.32 tokens per second)\n",
      "llama_print_timings: prompt eval time =     185.38 ms /    13 tokens (   14.26 ms per token,    70.13 tokens per second)\n",
      "llama_print_timings:        eval time =    2057.65 ms /    39 runs   (   52.76 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2300.43 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "316 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.06 ms /    40 runs   (    0.10 ms per token,  9849.79 tokens per second)\n",
      "llama_print_timings: prompt eval time =     182.20 ms /    10 tokens (   18.22 ms per token,    54.89 tokens per second)\n",
      "llama_print_timings:        eval time =    2051.49 ms /    39 runs   (   52.60 ms per token,    19.01 tokens per second)\n",
      "llama_print_timings:       total time =    2288.40 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "317 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.90 ms /    39 runs   (    0.10 ms per token, 10010.27 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.42 ms /    24 tokens (    8.35 ms per token,   119.75 tokens per second)\n",
      "llama_print_timings:        eval time =    2003.87 ms /    38 runs   (   52.73 ms per token,    18.96 tokens per second)\n",
      "llama_print_timings:       total time =    2258.11 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "318 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.64 ms /   155 runs   (    0.09 ms per token, 10588.88 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.96 ms /    21 tokens (    9.33 ms per token,   107.16 tokens per second)\n",
      "llama_print_timings:        eval time =    8167.74 ms /   154 runs   (   53.04 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    8579.86 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "319 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.85 ms /    48 runs   (    0.10 ms per token,  9900.99 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.22 ms /    21 tokens (    9.34 ms per token,   107.02 tokens per second)\n",
      "llama_print_timings:        eval time =    2480.14 ms /    47 runs   (   52.77 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2742.03 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "320 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.27 ms /   109 runs   (    0.10 ms per token,  9668.26 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.17 ms /    19 tokens (   10.22 ms per token,    97.85 tokens per second)\n",
      "llama_print_timings:        eval time =    5717.67 ms /   108 runs   (   52.94 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    6064.19 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "321 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.91 ms /    40 runs   (    0.10 ms per token, 10222.34 tokens per second)\n",
      "llama_print_timings: prompt eval time =     182.00 ms /    10 tokens (   18.20 ms per token,    54.95 tokens per second)\n",
      "llama_print_timings:        eval time =    2052.57 ms /    39 runs   (   52.63 ms per token,    19.00 tokens per second)\n",
      "llama_print_timings:       total time =    2289.31 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "322 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.04 ms /    40 runs   (    0.10 ms per token,  9908.35 tokens per second)\n",
      "llama_print_timings: prompt eval time =     190.75 ms /    17 tokens (   11.22 ms per token,    89.12 tokens per second)\n",
      "llama_print_timings:        eval time =    2051.71 ms /    39 runs   (   52.61 ms per token,    19.01 tokens per second)\n",
      "llama_print_timings:       total time =    2297.14 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "323 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.31 ms /   111 runs   (    0.10 ms per token,  9815.19 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.82 ms /    18 tokens (   10.71 ms per token,    93.35 tokens per second)\n",
      "llama_print_timings:        eval time =    5822.74 ms /   110 runs   (   52.93 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    6169.61 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "324 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.10 ms /    39 runs   (    0.11 ms per token,  9502.92 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.46 ms /    18 tokens (   10.69 ms per token,    93.53 tokens per second)\n",
      "llama_print_timings:        eval time =    2003.07 ms /    38 runs   (   52.71 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    2250.46 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "325 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.70 ms /   126 runs   (    0.10 ms per token,  9921.26 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.51 ms /    15 tokens (   12.57 ms per token,    79.57 tokens per second)\n",
      "llama_print_timings:        eval time =    6617.68 ms /   125 runs   (   52.94 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    6982.91 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.10 ms /    60 runs   (    0.10 ms per token,  9831.23 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.51 ms /    15 tokens (   12.57 ms per token,    79.57 tokens per second)\n",
      "llama_print_timings:        eval time =    3109.56 ms /    59 runs   (   52.70 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    3379.81 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "326 -- 1\n",
      "327 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.05 ms /    40 runs   (    0.10 ms per token,  9866.80 tokens per second)\n",
      "llama_print_timings: prompt eval time =     190.91 ms /    17 tokens (   11.23 ms per token,    89.05 tokens per second)\n",
      "llama_print_timings:        eval time =    2053.03 ms /    39 runs   (   52.64 ms per token,    19.00 tokens per second)\n",
      "llama_print_timings:       total time =    2297.76 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "328 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.07 ms /    39 runs   (    0.10 ms per token,  9591.74 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.61 ms /    22 tokens (    9.03 ms per token,   110.77 tokens per second)\n",
      "llama_print_timings:        eval time =    2004.05 ms /    38 runs   (   52.74 ms per token,    18.96 tokens per second)\n",
      "llama_print_timings:       total time =    2257.33 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "329 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.14 ms /   129 runs   (    0.09 ms per token, 10629.53 tokens per second)\n",
      "llama_print_timings: prompt eval time =     361.63 ms /    47 tokens (    7.69 ms per token,   129.97 tokens per second)\n",
      "llama_print_timings:        eval time =    6800.24 ms /   128 runs   (   53.13 ms per token,    18.82 tokens per second)\n",
      "llama_print_timings:       total time =    7341.43 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "330 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.03 ms /    40 runs   (    0.10 ms per token,  9932.95 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.73 ms /    16 tokens (   11.80 ms per token,    84.78 tokens per second)\n",
      "llama_print_timings:        eval time =    2054.45 ms /    39 runs   (   52.68 ms per token,    18.98 tokens per second)\n",
      "llama_print_timings:       total time =    2297.08 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "331 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.53 ms /    45 runs   (    0.10 ms per token,  9942.55 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.90 ms /    19 tokens (   10.21 ms per token,    97.99 tokens per second)\n",
      "llama_print_timings:        eval time =    2317.61 ms /    44 runs   (   52.67 ms per token,    18.99 tokens per second)\n",
      "llama_print_timings:       total time =    2572.84 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "332 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.71 ms /   128 runs   (    0.10 ms per token, 10069.23 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.20 ms /    19 tokens (   10.22 ms per token,    97.84 tokens per second)\n",
      "llama_print_timings:        eval time =    6714.64 ms /   127 runs   (   52.87 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    7087.29 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "333 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.15 ms /   134 runs   (    0.10 ms per token, 10188.56 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.42 ms /    24 tokens (    8.35 ms per token,   119.75 tokens per second)\n",
      "llama_print_timings:        eval time =    7048.70 ms /   133 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    7437.15 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "334 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.84 ms /   118 runs   (    0.10 ms per token,  9967.06 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.43 ms /    22 tokens (    9.02 ms per token,   110.87 tokens per second)\n",
      "llama_print_timings:        eval time =    6193.35 ms /   117 runs   (   52.93 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    6555.59 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "335 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.45 ms /    44 runs   (    0.10 ms per token,  9880.98 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.62 ms /    23 tokens (    8.68 ms per token,   115.22 tokens per second)\n",
      "llama_print_timings:        eval time =    2267.95 ms /    43 runs   (   52.74 ms per token,    18.96 tokens per second)\n",
      "llama_print_timings:       total time =    2527.49 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "336 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.46 ms /   137 runs   (    0.10 ms per token, 10182.09 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.30 ms /    28 tokens (    7.33 ms per token,   136.39 tokens per second)\n",
      "llama_print_timings:        eval time =    7219.26 ms /   136 runs   (   53.08 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    7613.72 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "337 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.49 ms /   146 runs   (    0.10 ms per token, 10075.22 tokens per second)\n",
      "llama_print_timings: prompt eval time =     209.96 ms /    30 tokens (    7.00 ms per token,   142.89 tokens per second)\n",
      "llama_print_timings:        eval time =    7694.19 ms /   145 runs   (   53.06 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    8110.54 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "338 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.11 ms /    63 runs   (    0.10 ms per token, 10309.28 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.71 ms /    21 tokens (    9.37 ms per token,   106.76 tokens per second)\n",
      "llama_print_timings:        eval time =    3277.40 ms /    62 runs   (   52.86 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    3559.70 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "339 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.98 ms /   124 runs   (    0.10 ms per token, 10350.58 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.73 ms /    20 tokens (    9.19 ms per token,   108.85 tokens per second)\n",
      "llama_print_timings:        eval time =    6513.80 ms /   123 runs   (   52.96 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    6869.16 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "340 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.98 ms /    39 runs   (    0.10 ms per token,  9791.61 tokens per second)\n",
      "llama_print_timings: prompt eval time =     190.75 ms /    17 tokens (   11.22 ms per token,    89.12 tokens per second)\n",
      "llama_print_timings:        eval time =    2003.61 ms /    38 runs   (   52.73 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    2247.61 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "341 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.28 ms /    60 runs   (    0.10 ms per token,  9555.66 tokens per second)\n",
      "llama_print_timings: prompt eval time =     190.59 ms /    17 tokens (   11.21 ms per token,    89.20 tokens per second)\n",
      "llama_print_timings:        eval time =    3110.57 ms /    59 runs   (   52.72 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    3385.27 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "342 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.45 ms /   127 runs   (    0.10 ms per token, 10199.98 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.97 ms /    19 tokens (   10.21 ms per token,    97.95 tokens per second)\n",
      "llama_print_timings:        eval time =    6673.84 ms /   126 runs   (   52.97 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    7045.19 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "343 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.00 ms /    44 runs   (    0.09 ms per token, 11013.77 tokens per second)\n",
      "llama_print_timings: prompt eval time =     209.94 ms /    30 tokens (    7.00 ms per token,   142.90 tokens per second)\n",
      "llama_print_timings:        eval time =    2271.62 ms /    43 runs   (   52.83 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    2541.65 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "344 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.34 ms /   119 runs   (    0.10 ms per token,  9640.31 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.48 ms /    20 tokens (    9.17 ms per token,   109.01 tokens per second)\n",
      "llama_print_timings:        eval time =    6249.96 ms /   118 runs   (   52.97 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    6599.20 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "345 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.94 ms /   124 runs   (    0.10 ms per token,  9580.47 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.67 ms /    22 tokens (    9.03 ms per token,   110.73 tokens per second)\n",
      "llama_print_timings:        eval time =    6519.49 ms /   123 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6894.58 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "346 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.61 ms /    45 runs   (    0.10 ms per token,  9752.93 tokens per second)\n",
      "llama_print_timings: prompt eval time =     185.66 ms /    13 tokens (   14.28 ms per token,    70.02 tokens per second)\n",
      "llama_print_timings:        eval time =    2319.26 ms /    44 runs   (   52.71 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    2566.38 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "347 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.97 ms /    40 runs   (    0.10 ms per token, 10070.49 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.63 ms /    15 tokens (   12.58 ms per token,    79.52 tokens per second)\n",
      "llama_print_timings:        eval time =    2055.62 ms /    39 runs   (   52.71 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    2298.76 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "348 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      10.35 ms /   110 runs   (    0.09 ms per token, 10624.94 tokens per second)\n",
      "llama_print_timings: prompt eval time =     203.65 ms /    26 tokens (    7.83 ms per token,   127.67 tokens per second)\n",
      "llama_print_timings:        eval time =    5774.58 ms /   109 runs   (   52.98 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    6129.99 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "349 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.96 ms /   149 runs   (    0.10 ms per token,  9958.56 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.40 ms /    24 tokens (    8.35 ms per token,   119.76 tokens per second)\n",
      "llama_print_timings:        eval time =    7846.41 ms /   148 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    8256.26 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "350 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.04 ms /   141 runs   (    0.10 ms per token, 10045.60 tokens per second)\n",
      "llama_print_timings: prompt eval time =     346.38 ms /    36 tokens (    9.62 ms per token,   103.93 tokens per second)\n",
      "llama_print_timings:        eval time =    7433.28 ms /   140 runs   (   53.09 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    7979.53 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "351 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.00 ms /    39 runs   (    0.10 ms per token,  9752.44 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.50 ms /    16 tokens (   11.78 ms per token,    84.88 tokens per second)\n",
      "llama_print_timings:        eval time =    2003.60 ms /    38 runs   (   52.73 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    2244.52 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "352 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.92 ms /   146 runs   (    0.10 ms per token,  9784.21 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.72 ms /    23 tokens (    8.68 ms per token,   115.16 tokens per second)\n",
      "llama_print_timings:        eval time =    7691.87 ms /   145 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    8096.41 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "353 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.40 ms /   135 runs   (    0.09 ms per token, 10888.85 tokens per second)\n",
      "llama_print_timings: prompt eval time =     345.77 ms /    35 tokens (    9.88 ms per token,   101.22 tokens per second)\n",
      "llama_print_timings:        eval time =    7108.40 ms /   134 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    7641.37 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "354 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.73 ms /   123 runs   (    0.10 ms per token, 10483.25 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.30 ms /    21 tokens (    9.35 ms per token,   106.98 tokens per second)\n",
      "llama_print_timings:        eval time =    6459.20 ms /   122 runs   (   52.94 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    6826.32 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "355 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.16 ms /   128 runs   (    0.10 ms per token,  9727.18 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.68 ms /    21 tokens (    9.32 ms per token,   107.32 tokens per second)\n",
      "llama_print_timings:        eval time =    6729.44 ms /   127 runs   (   52.99 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    7105.73 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "356 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.94 ms /    39 runs   (    0.10 ms per token,  9898.48 tokens per second)\n",
      "llama_print_timings: prompt eval time =     187.94 ms /    14 tokens (   13.42 ms per token,    74.49 tokens per second)\n",
      "llama_print_timings:        eval time =    1998.34 ms /    38 runs   (   52.59 ms per token,    19.02 tokens per second)\n",
      "llama_print_timings:       total time =    2239.45 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "357 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.41 ms /   150 runs   (    0.10 ms per token, 10410.16 tokens per second)\n",
      "llama_print_timings: prompt eval time =     349.30 ms /    38 tokens (    9.19 ms per token,   108.79 tokens per second)\n",
      "llama_print_timings:        eval time =    7913.38 ms /   149 runs   (   53.11 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    8471.75 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "358 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.39 ms /   143 runs   (    0.10 ms per token,  9938.15 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.93 ms /    22 tokens (    9.00 ms per token,   111.15 tokens per second)\n",
      "llama_print_timings:        eval time =    7527.64 ms /   142 runs   (   53.01 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    7927.78 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "359 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.60 ms /   120 runs   (    0.10 ms per token, 10344.83 tokens per second)\n",
      "llama_print_timings: prompt eval time =     207.66 ms /    29 tokens (    7.16 ms per token,   139.65 tokens per second)\n",
      "llama_print_timings:        eval time =    6309.36 ms /   119 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    6683.96 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "360 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.75 ms /    46 runs   (    0.10 ms per token,  9682.17 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.85 ms /    18 tokens (   10.71 ms per token,    93.34 tokens per second)\n",
      "llama_print_timings:        eval time =    2375.46 ms /    45 runs   (   52.79 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2630.84 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "361 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.34 ms /   150 runs   (    0.10 ms per token, 10458.79 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.13 ms /    24 tokens (    8.34 ms per token,   119.92 tokens per second)\n",
      "llama_print_timings:        eval time =    7904.29 ms /   149 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    8315.56 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "362 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.83 ms /   128 runs   (    0.10 ms per token,  9978.95 tokens per second)\n",
      "llama_print_timings: prompt eval time =     204.73 ms /    26 tokens (    7.87 ms per token,   127.00 tokens per second)\n",
      "llama_print_timings:        eval time =    6740.08 ms /   127 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    7125.76 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "362 -- 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.53 ms /   122 runs   (    0.09 ms per token, 10582.93 tokens per second)\n",
      "llama_print_timings: prompt eval time =       0.00 ms /     1 tokens (    0.00 ms per token,      inf tokens per second)\n",
      "llama_print_timings:        eval time =    6466.36 ms /   122 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    6636.28 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "363 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.52 ms /    45 runs   (    0.10 ms per token,  9949.15 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.41 ms /    19 tokens (   10.23 ms per token,    97.73 tokens per second)\n",
      "llama_print_timings:        eval time =    2318.21 ms /    44 runs   (   52.69 ms per token,    18.98 tokens per second)\n",
      "llama_print_timings:       total time =    2574.34 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "364 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.87 ms /    39 runs   (    0.10 ms per token, 10090.56 tokens per second)\n",
      "llama_print_timings: prompt eval time =     183.78 ms /    12 tokens (   15.32 ms per token,    65.30 tokens per second)\n",
      "llama_print_timings:        eval time =    2001.74 ms /    38 runs   (   52.68 ms per token,    18.98 tokens per second)\n",
      "llama_print_timings:       total time =    2239.24 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "365 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.57 ms /   122 runs   (    0.10 ms per token,  9704.88 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.79 ms /    27 tokens (    7.62 ms per token,   131.20 tokens per second)\n",
      "llama_print_timings:        eval time =    6422.85 ms /   121 runs   (   53.08 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    6804.33 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "366 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.37 ms /   131 runs   (    0.10 ms per token,  9798.79 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.49 ms /    24 tokens (    8.35 ms per token,   119.71 tokens per second)\n",
      "llama_print_timings:        eval time =    6892.78 ms /   130 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    7276.17 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "367 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.31 ms /   112 runs   (    0.10 ms per token,  9899.24 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.27 ms /    19 tokens (   10.22 ms per token,    97.80 tokens per second)\n",
      "llama_print_timings:        eval time =    5877.60 ms /   111 runs   (   52.95 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    6227.90 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "368 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.99 ms /    39 runs   (    0.10 ms per token,  9781.79 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.06 ms /    19 tokens (   10.21 ms per token,    97.91 tokens per second)\n",
      "llama_print_timings:        eval time =    2004.93 ms /    38 runs   (   52.76 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2252.13 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "369 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.67 ms /    45 runs   (    0.10 ms per token,  9644.23 tokens per second)\n",
      "llama_print_timings: prompt eval time =     182.98 ms /    20 tokens (    9.15 ms per token,   109.30 tokens per second)\n",
      "llama_print_timings:        eval time =    2319.33 ms /    44 runs   (   52.71 ms per token,    18.97 tokens per second)\n",
      "llama_print_timings:       total time =    2564.99 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "370 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.34 ms /   145 runs   (    0.10 ms per token, 10113.69 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.91 ms /    21 tokens (    9.33 ms per token,   107.19 tokens per second)\n",
      "llama_print_timings:        eval time =    7631.53 ms /   144 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    8031.30 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "371 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.90 ms /    39 runs   (    0.10 ms per token,  9994.87 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.43 ms /    16 tokens (   11.78 ms per token,    84.91 tokens per second)\n",
      "llama_print_timings:        eval time =    2001.37 ms /    38 runs   (   52.67 ms per token,    18.99 tokens per second)\n",
      "llama_print_timings:       total time =    2243.04 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.56 ms /   144 runs   (    0.09 ms per token, 10617.12 tokens per second)\n",
      "llama_print_timings: prompt eval time =     357.38 ms /    43 tokens (    8.31 ms per token,   120.32 tokens per second)\n",
      "llama_print_timings:        eval time =    7596.70 ms /   143 runs   (   53.12 ms per token,    18.82 tokens per second)\n",
      "llama_print_timings:       total time =    8154.20 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "372 -- 1\n",
      "373 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.30 ms /    40 runs   (    0.11 ms per token,  9304.49 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.58 ms /    23 tokens (    8.68 ms per token,   115.24 tokens per second)\n",
      "llama_print_timings:        eval time =    2055.00 ms /    39 runs   (   52.69 ms per token,    18.98 tokens per second)\n",
      "llama_print_timings:       total time =    2311.39 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "374 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.84 ms /   125 runs   (    0.10 ms per token,  9735.20 tokens per second)\n",
      "llama_print_timings: prompt eval time =     187.64 ms /    14 tokens (   13.40 ms per token,    74.61 tokens per second)\n",
      "llama_print_timings:        eval time =    6565.44 ms /   124 runs   (   52.95 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    6928.18 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "375 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.93 ms /    40 runs   (    0.10 ms per token, 10185.89 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.86 ms /    27 tokens (    7.62 ms per token,   131.16 tokens per second)\n",
      "llama_print_timings:        eval time =    2057.57 ms /    39 runs   (   52.76 ms per token,    18.95 tokens per second)\n",
      "llama_print_timings:       total time =    2317.42 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "376 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.56 ms /   130 runs   (    0.10 ms per token,  9587.02 tokens per second)\n",
      "llama_print_timings: prompt eval time =     357.50 ms /    45 tokens (    7.94 ms per token,   125.87 tokens per second)\n",
      "llama_print_timings:        eval time =    6874.60 ms /   129 runs   (   53.29 ms per token,    18.76 tokens per second)\n",
      "llama_print_timings:       total time =    7433.66 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "377 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.34 ms /   122 runs   (    0.11 ms per token,  9142.69 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.25 ms /    19 tokens (   10.22 ms per token,    97.81 tokens per second)\n",
      "llama_print_timings:        eval time =    6505.49 ms /   121 runs   (   53.76 ms per token,    18.60 tokens per second)\n",
      "llama_print_timings:       total time =    6907.32 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "378 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.94 ms /    40 runs   (    0.12 ms per token,  8102.09 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.50 ms /    23 tokens (    8.67 ms per token,   115.29 tokens per second)\n",
      "llama_print_timings:        eval time =    2068.41 ms /    39 runs   (   53.04 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    2331.90 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       9.39 ms /    40 runs   (    0.23 ms per token,  4257.58 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.02 ms /    14 tokens (   13.79 ms per token,    72.53 tokens per second)\n",
      "llama_print_timings:        eval time =    2127.34 ms /    39 runs   (   54.55 ms per token,    18.33 tokens per second)\n",
      "llama_print_timings:       total time =    2473.06 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "379 -- 1\n",
      "380 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      28.36 ms /   125 runs   (    0.23 ms per token,  4407.77 tokens per second)\n",
      "llama_print_timings: prompt eval time =     201.64 ms /    23 tokens (    8.77 ms per token,   114.06 tokens per second)\n",
      "llama_print_timings:        eval time =    6688.51 ms /   124 runs   (   53.94 ms per token,    18.54 tokens per second)\n",
      "llama_print_timings:       total time =    7278.49 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "381 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      10.77 ms /   102 runs   (    0.11 ms per token,  9470.75 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.24 ms /    23 tokens (    8.66 ms per token,   115.44 tokens per second)\n",
      "llama_print_timings:        eval time =    5350.80 ms /   101 runs   (   52.98 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    5695.95 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "382 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.29 ms /    40 runs   (    0.11 ms per token,  9328.36 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.90 ms /    14 tokens (   13.71 ms per token,    72.95 tokens per second)\n",
      "llama_print_timings:        eval time =    2097.93 ms /    39 runs   (   53.79 ms per token,    18.59 tokens per second)\n",
      "llama_print_timings:       total time =    2360.24 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "383 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.80 ms /    46 runs   (    0.10 ms per token,  9591.33 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.01 ms /    19 tokens (   10.47 ms per token,    95.47 tokens per second)\n",
      "llama_print_timings:        eval time =    2420.28 ms /    45 runs   (   53.78 ms per token,    18.59 tokens per second)\n",
      "llama_print_timings:       total time =    2687.68 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.29 ms /    40 runs   (    0.11 ms per token,  9313.15 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.10 ms /    21 tokens (    9.48 ms per token,   105.48 tokens per second)\n",
      "llama_print_timings:        eval time =    2128.71 ms /    39 runs   (   54.58 ms per token,    18.32 tokens per second)\n",
      "llama_print_timings:       total time =    2385.86 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "384 -- 1\n",
      "385 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.55 ms /    40 runs   (    0.11 ms per token,  8783.49 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.38 ms /    21 tokens (    9.45 ms per token,   105.85 tokens per second)\n",
      "llama_print_timings:        eval time =    2188.91 ms /    39 runs   (   56.13 ms per token,    17.82 tokens per second)\n",
      "llama_print_timings:       total time =    2453.50 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.24 ms /    40 runs   (    0.11 ms per token,  9425.07 tokens per second)\n",
      "llama_print_timings: prompt eval time =     222.64 ms /    16 tokens (   13.91 ms per token,    71.87 tokens per second)\n",
      "llama_print_timings:        eval time =    2136.80 ms /    39 runs   (   54.79 ms per token,    18.25 tokens per second)\n",
      "llama_print_timings:       total time =    2422.36 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "386 -- 1\n",
      "387 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.62 ms /    44 runs   (    0.10 ms per token,  9530.00 tokens per second)\n",
      "llama_print_timings: prompt eval time =     185.49 ms /    12 tokens (   15.46 ms per token,    64.69 tokens per second)\n",
      "llama_print_timings:        eval time =    2334.82 ms /    43 runs   (   54.30 ms per token,    18.42 tokens per second)\n",
      "llama_print_timings:       total time =    2583.14 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "388 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.27 ms /   116 runs   (    0.11 ms per token,  9456.26 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.90 ms /    18 tokens (   10.88 ms per token,    91.89 tokens per second)\n",
      "llama_print_timings:        eval time =    6288.47 ms /   115 runs   (   54.68 ms per token,    18.29 tokens per second)\n",
      "llama_print_timings:       total time =    6658.92 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "389 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.53 ms /   128 runs   (    0.10 ms per token, 10217.11 tokens per second)\n",
      "llama_print_timings: prompt eval time =     210.22 ms /    28 tokens (    7.51 ms per token,   133.19 tokens per second)\n",
      "llama_print_timings:        eval time =    7042.62 ms /   127 runs   (   55.45 ms per token,    18.03 tokens per second)\n",
      "llama_print_timings:       total time =    7442.82 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "390 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.93 ms /   140 runs   (    0.11 ms per token,  9377.72 tokens per second)\n",
      "llama_print_timings: prompt eval time =     215.45 ms /    31 tokens (    6.95 ms per token,   143.88 tokens per second)\n",
      "llama_print_timings:        eval time =    7633.43 ms /   139 runs   (   54.92 ms per token,    18.21 tokens per second)\n",
      "llama_print_timings:       total time =    8059.60 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "391 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      10.49 ms /   100 runs   (    0.10 ms per token,  9530.16 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.06 ms /    18 tokens (   11.06 ms per token,    90.42 tokens per second)\n",
      "llama_print_timings:        eval time =    5509.39 ms /    99 runs   (   55.65 ms per token,    17.97 tokens per second)\n",
      "llama_print_timings:       total time =    5857.69 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "392 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       9.62 ms /   103 runs   (    0.09 ms per token, 10712.43 tokens per second)\n",
      "llama_print_timings: prompt eval time =     210.90 ms /    32 tokens (    6.59 ms per token,   151.73 tokens per second)\n",
      "llama_print_timings:        eval time =    5612.43 ms /   102 runs   (   55.02 ms per token,    18.17 tokens per second)\n",
      "llama_print_timings:       total time =    5965.31 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "393 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.37 ms /   128 runs   (    0.10 ms per token, 10349.29 tokens per second)\n",
      "llama_print_timings: prompt eval time =     213.45 ms /    32 tokens (    6.67 ms per token,   149.92 tokens per second)\n",
      "llama_print_timings:        eval time =    6832.24 ms /   127 runs   (   53.80 ms per token,    18.59 tokens per second)\n",
      "llama_print_timings:       total time =    7215.20 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "394 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.36 ms /   139 runs   (    0.09 ms per token, 11250.51 tokens per second)\n",
      "llama_print_timings: prompt eval time =     346.03 ms /    33 tokens (   10.49 ms per token,    95.37 tokens per second)\n",
      "llama_print_timings:        eval time =    7416.04 ms /   138 runs   (   53.74 ms per token,    18.61 tokens per second)\n",
      "llama_print_timings:       total time =    7941.25 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "395 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.56 ms /   125 runs   (    0.10 ms per token,  9953.81 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.54 ms /    21 tokens (    9.45 ms per token,   105.77 tokens per second)\n",
      "llama_print_timings:        eval time =    6655.60 ms /   124 runs   (   53.67 ms per token,    18.63 tokens per second)\n",
      "llama_print_timings:       total time =    7018.96 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "396 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.06 ms /   112 runs   (    0.10 ms per token, 10125.67 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.75 ms /    18 tokens (   10.82 ms per token,    92.43 tokens per second)\n",
      "llama_print_timings:        eval time =    5947.79 ms /   111 runs   (   53.58 ms per token,    18.66 tokens per second)\n",
      "llama_print_timings:       total time =    6287.76 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "397 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.93 ms /    40 runs   (    0.10 ms per token, 10170.35 tokens per second)\n",
      "llama_print_timings: prompt eval time =     190.38 ms /    16 tokens (   11.90 ms per token,    84.04 tokens per second)\n",
      "llama_print_timings:        eval time =    2079.08 ms /    39 runs   (   53.31 ms per token,    18.76 tokens per second)\n",
      "llama_print_timings:       total time =    2320.10 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "398 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.93 ms /    40 runs   (    0.10 ms per token, 10172.94 tokens per second)\n",
      "llama_print_timings: prompt eval time =     185.66 ms /    12 tokens (   15.47 ms per token,    64.63 tokens per second)\n",
      "llama_print_timings:        eval time =    2079.29 ms /    39 runs   (   53.32 ms per token,    18.76 tokens per second)\n",
      "llama_print_timings:       total time =    2316.59 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "399 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.45 ms /    44 runs   (    0.10 ms per token,  9887.64 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.97 ms /    19 tokens (   10.31 ms per token,    96.95 tokens per second)\n",
      "llama_print_timings:        eval time =    2294.65 ms /    43 runs   (   53.36 ms per token,    18.74 tokens per second)\n",
      "llama_print_timings:       total time =    2546.89 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "400 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.17 ms /   121 runs   (    0.09 ms per token, 10828.71 tokens per second)\n",
      "llama_print_timings: prompt eval time =     204.63 ms /    25 tokens (    8.19 ms per token,   122.17 tokens per second)\n",
      "llama_print_timings:        eval time =    6438.90 ms /   120 runs   (   53.66 ms per token,    18.64 tokens per second)\n",
      "llama_print_timings:       total time =    6799.19 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "401 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       7.55 ms /    88 runs   (    0.09 ms per token, 11658.72 tokens per second)\n",
      "llama_print_timings: prompt eval time =     348.05 ms /    34 tokens (   10.24 ms per token,    97.69 tokens per second)\n",
      "llama_print_timings:        eval time =    4665.13 ms /    87 runs   (   53.62 ms per token,    18.65 tokens per second)\n",
      "llama_print_timings:       total time =    5124.94 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "402 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.40 ms /    45 runs   (    0.10 ms per token, 10222.63 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.70 ms /    20 tokens (    9.88 ms per token,   101.16 tokens per second)\n",
      "llama_print_timings:        eval time =    2348.55 ms /    44 runs   (   53.38 ms per token,    18.73 tokens per second)\n",
      "llama_print_timings:       total time =    2603.33 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "403 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.38 ms /    44 runs   (    0.10 ms per token, 10036.50 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.46 ms /    23 tokens (    8.72 ms per token,   114.74 tokens per second)\n",
      "llama_print_timings:        eval time =    2297.12 ms /    43 runs   (   53.42 ms per token,    18.72 tokens per second)\n",
      "llama_print_timings:       total time =    2553.05 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "404 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.00 ms /   136 runs   (    0.10 ms per token, 10461.54 tokens per second)\n",
      "llama_print_timings: prompt eval time =     204.72 ms /    25 tokens (    8.19 ms per token,   122.12 tokens per second)\n",
      "llama_print_timings:        eval time =    7249.51 ms /   135 runs   (   53.70 ms per token,    18.62 tokens per second)\n",
      "llama_print_timings:       total time =    7633.24 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "405 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      10.75 ms /   116 runs   (    0.09 ms per token, 10787.69 tokens per second)\n",
      "llama_print_timings: prompt eval time =     213.64 ms /    31 tokens (    6.89 ms per token,   145.10 tokens per second)\n",
      "llama_print_timings:        eval time =    6174.68 ms /   115 runs   (   53.69 ms per token,    18.62 tokens per second)\n",
      "llama_print_timings:       total time =    6536.95 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "406 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.03 ms /   110 runs   (    0.10 ms per token,  9974.61 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.38 ms /    19 tokens (   10.39 ms per token,    96.26 tokens per second)\n",
      "llama_print_timings:        eval time =    5839.85 ms /   109 runs   (   53.58 ms per token,    18.66 tokens per second)\n",
      "llama_print_timings:       total time =    6179.88 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "407 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.89 ms /    40 runs   (    0.10 ms per token, 10293.36 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.67 ms /    18 tokens (   10.87 ms per token,    91.99 tokens per second)\n",
      "llama_print_timings:        eval time =    2079.12 ms /    39 runs   (   53.31 ms per token,    18.76 tokens per second)\n",
      "llama_print_timings:       total time =    2325.55 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "408 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.87 ms /    39 runs   (    0.10 ms per token, 10074.92 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.12 ms /    15 tokens (   12.74 ms per token,    78.49 tokens per second)\n",
      "llama_print_timings:        eval time =    2027.45 ms /    38 runs   (   53.35 ms per token,    18.74 tokens per second)\n",
      "llama_print_timings:       total time =    2267.54 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "409 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.53 ms /    46 runs   (    0.10 ms per token, 10161.25 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.24 ms /    13 tokens (   14.48 ms per token,    69.06 tokens per second)\n",
      "llama_print_timings:        eval time =    2401.19 ms /    45 runs   (   53.36 ms per token,    18.74 tokens per second)\n",
      "llama_print_timings:       total time =    2647.52 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "410 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.93 ms /    40 runs   (    0.10 ms per token, 10170.35 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.09 ms /    19 tokens (   10.32 ms per token,    96.90 tokens per second)\n",
      "llama_print_timings:        eval time =    2078.42 ms /    39 runs   (   53.29 ms per token,    18.76 tokens per second)\n",
      "llama_print_timings:       total time =    2325.75 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "411 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       6.24 ms /    64 runs   (    0.10 ms per token, 10249.84 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.53 ms /    22 tokens (    9.12 ms per token,   109.71 tokens per second)\n",
      "llama_print_timings:        eval time =    3372.99 ms /    63 runs   (   53.54 ms per token,    18.68 tokens per second)\n",
      "llama_print_timings:       total time =    3656.12 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "412 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.58 ms /    45 runs   (    0.10 ms per token,  9825.33 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.63 ms /    19 tokens (   10.35 ms per token,    96.63 tokens per second)\n",
      "llama_print_timings:        eval time =    2348.86 ms /    44 runs   (   53.38 ms per token,    18.73 tokens per second)\n",
      "llama_print_timings:       total time =    2604.24 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "413 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.57 ms /   126 runs   (    0.10 ms per token, 10023.07 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.69 ms /    21 tokens (    9.46 ms per token,   105.69 tokens per second)\n",
      "llama_print_timings:        eval time =    6707.47 ms /   125 runs   (   53.66 ms per token,    18.64 tokens per second)\n",
      "llama_print_timings:       total time =    7069.74 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "414 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.85 ms /   126 runs   (    0.09 ms per token, 10635.60 tokens per second)\n",
      "llama_print_timings: prompt eval time =     212.90 ms /    30 tokens (    7.10 ms per token,   140.91 tokens per second)\n",
      "llama_print_timings:        eval time =    6714.17 ms /   125 runs   (   53.71 ms per token,    18.62 tokens per second)\n",
      "llama_print_timings:       total time =    7089.07 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "415 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.88 ms /   139 runs   (    0.10 ms per token, 10017.30 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.96 ms /    19 tokens (   10.37 ms per token,    96.47 tokens per second)\n",
      "llama_print_timings:        eval time =    7408.24 ms /   138 runs   (   53.68 ms per token,    18.63 tokens per second)\n",
      "llama_print_timings:       total time =    7785.52 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "416 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.36 ms /   131 runs   (    0.09 ms per token, 10600.42 tokens per second)\n",
      "llama_print_timings: prompt eval time =     213.89 ms /    31 tokens (    6.90 ms per token,   144.93 tokens per second)\n",
      "llama_print_timings:        eval time =    6984.37 ms /   130 runs   (   53.73 ms per token,    18.61 tokens per second)\n",
      "llama_print_timings:       total time =    7369.12 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "417 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      14.42 ms /   148 runs   (    0.10 ms per token, 10262.10 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.32 ms /    18 tokens (   10.85 ms per token,    92.16 tokens per second)\n",
      "llama_print_timings:        eval time =    7887.28 ms /   147 runs   (   53.65 ms per token,    18.64 tokens per second)\n",
      "llama_print_timings:       total time =    8276.25 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "418 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.00 ms /    40 runs   (    0.10 ms per token, 10002.50 tokens per second)\n",
      "llama_print_timings: prompt eval time =     201.86 ms /    23 tokens (    8.78 ms per token,   113.94 tokens per second)\n",
      "llama_print_timings:        eval time =    2083.09 ms /    39 runs   (   53.41 ms per token,    18.72 tokens per second)\n",
      "llama_print_timings:       total time =    2337.95 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "419 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.35 ms /   121 runs   (    0.09 ms per token, 10664.55 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.22 ms /    18 tokens (   10.85 ms per token,    92.20 tokens per second)\n",
      "llama_print_timings:        eval time =    6432.27 ms /   120 runs   (   53.60 ms per token,    18.66 tokens per second)\n",
      "llama_print_timings:       total time =    6784.05 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "420 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.05 ms /    40 runs   (    0.10 ms per token,  9876.54 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.97 ms /    17 tokens (   11.35 ms per token,    88.09 tokens per second)\n",
      "llama_print_timings:        eval time =    2079.87 ms /    39 runs   (   53.33 ms per token,    18.75 tokens per second)\n",
      "llama_print_timings:       total time =    2324.00 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "421 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.85 ms /    40 runs   (    0.10 ms per token, 10386.91 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.49 ms /    17 tokens (   11.32 ms per token,    88.31 tokens per second)\n",
      "llama_print_timings:        eval time =    2077.44 ms /    39 runs   (   53.27 ms per token,    18.77 tokens per second)\n",
      "llama_print_timings:       total time =    2319.44 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "422 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       5.83 ms /    59 runs   (    0.10 ms per token, 10127.02 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.61 ms /    17 tokens (   11.39 ms per token,    87.81 tokens per second)\n",
      "llama_print_timings:        eval time =    3097.65 ms /    58 runs   (   53.41 ms per token,    18.72 tokens per second)\n",
      "llama_print_timings:       total time =    3369.56 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "423 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.59 ms /    46 runs   (    0.10 ms per token, 10030.53 tokens per second)\n",
      "llama_print_timings: prompt eval time =     201.91 ms /    23 tokens (    8.78 ms per token,   113.91 tokens per second)\n",
      "llama_print_timings:        eval time =    2405.19 ms /    45 runs   (   53.45 ms per token,    18.71 tokens per second)\n",
      "llama_print_timings:       total time =    2666.09 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "424 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       8.55 ms /    87 runs   (    0.10 ms per token, 10175.44 tokens per second)\n",
      "llama_print_timings: prompt eval time =     212.20 ms /    30 tokens (    7.07 ms per token,   141.37 tokens per second)\n",
      "llama_print_timings:        eval time =    4611.14 ms /    86 runs   (   53.62 ms per token,    18.65 tokens per second)\n",
      "llama_print_timings:       total time =    4934.28 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "425 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.88 ms /    39 runs   (    0.10 ms per token, 10046.37 tokens per second)\n",
      "llama_print_timings: prompt eval time =     209.42 ms /    29 tokens (    7.22 ms per token,   138.48 tokens per second)\n",
      "llama_print_timings:        eval time =    2034.06 ms /    38 runs   (   53.53 ms per token,    18.68 tokens per second)\n",
      "llama_print_timings:       total time =    2292.74 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "426 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.23 ms /   114 runs   (    0.10 ms per token, 10149.57 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.87 ms /    19 tokens (   10.36 ms per token,    96.51 tokens per second)\n",
      "llama_print_timings:        eval time =    6058.06 ms /   113 runs   (   53.61 ms per token,    18.65 tokens per second)\n",
      "llama_print_timings:       total time =    6401.03 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "427 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      10.52 ms /   112 runs   (    0.09 ms per token, 10650.44 tokens per second)\n",
      "llama_print_timings: prompt eval time =     353.24 ms /    39 tokens (    9.06 ms per token,   110.41 tokens per second)\n",
      "llama_print_timings:        eval time =    5960.19 ms /   111 runs   (   53.70 ms per token,    18.62 tokens per second)\n",
      "llama_print_timings:       total time =    6457.22 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "428 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      13.30 ms /   151 runs   (    0.09 ms per token, 11356.80 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.73 ms /    15 tokens (   12.65 ms per token,    79.06 tokens per second)\n",
      "llama_print_timings:        eval time =    8057.30 ms /   150 runs   (   53.72 ms per token,    18.62 tokens per second)\n",
      "llama_print_timings:       total time =    8443.41 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "429 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      10.35 ms /   112 runs   (    0.09 ms per token, 10822.30 tokens per second)\n",
      "llama_print_timings: prompt eval time =     202.73 ms /    24 tokens (    8.45 ms per token,   118.38 tokens per second)\n",
      "llama_print_timings:        eval time =    5955.50 ms /   111 runs   (   53.65 ms per token,    18.64 tokens per second)\n",
      "llama_print_timings:       total time =    6301.12 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "430 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      10.53 ms /   106 runs   (    0.10 ms per token, 10062.65 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.25 ms /    17 tokens (   11.37 ms per token,    87.97 tokens per second)\n",
      "llama_print_timings:        eval time =    5627.15 ms /   105 runs   (   53.59 ms per token,    18.66 tokens per second)\n",
      "llama_print_timings:       total time =    5956.52 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.85 ms /    40 runs   (    0.10 ms per token, 10403.12 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.10 ms /    13 tokens (   14.47 ms per token,    69.11 tokens per second)\n",
      "llama_print_timings:        eval time =    2079.16 ms /    39 runs   (   53.31 ms per token,    18.76 tokens per second)\n",
      "llama_print_timings:       total time =    2318.03 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "431 -- 1\n",
      "432 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       4.56 ms /    46 runs   (    0.10 ms per token, 10092.15 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.98 ms /    18 tokens (   10.83 ms per token,    92.32 tokens per second)\n",
      "llama_print_timings:        eval time =    2399.63 ms /    45 runs   (   53.33 ms per token,    18.75 tokens per second)\n",
      "llama_print_timings:       total time =    2652.29 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "433 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.10 ms /   123 runs   (    0.10 ms per token, 10161.93 tokens per second)\n",
      "llama_print_timings: prompt eval time =     202.39 ms /    24 tokens (    8.43 ms per token,   118.58 tokens per second)\n",
      "llama_print_timings:        eval time =    6543.48 ms /   122 runs   (   53.64 ms per token,    18.64 tokens per second)\n",
      "llama_print_timings:       total time =    6904.28 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "434 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.82 ms /    40 runs   (    0.10 ms per token, 10468.46 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.85 ms /    22 tokens (    9.13 ms per token,   109.53 tokens per second)\n",
      "llama_print_timings:        eval time =    2080.84 ms /    39 runs   (   53.35 ms per token,    18.74 tokens per second)\n",
      "llama_print_timings:       total time =    2333.06 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "435 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.89 ms /   131 runs   (    0.10 ms per token, 10165.28 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.52 ms /    26 tokens (    7.90 ms per token,   126.51 tokens per second)\n",
      "llama_print_timings:        eval time =    6984.85 ms /   130 runs   (   53.73 ms per token,    18.61 tokens per second)\n",
      "llama_print_timings:       total time =    7359.24 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "436 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.27 ms /   112 runs   (    0.10 ms per token,  9935.24 tokens per second)\n",
      "llama_print_timings: prompt eval time =     202.40 ms /    23 tokens (    8.80 ms per token,   113.64 tokens per second)\n",
      "llama_print_timings:        eval time =    5956.31 ms /   111 runs   (   53.66 ms per token,    18.64 tokens per second)\n",
      "llama_print_timings:       total time =    6302.94 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "437 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.80 ms /    39 runs   (    0.10 ms per token, 10260.46 tokens per second)\n",
      "llama_print_timings: prompt eval time =     202.75 ms /    24 tokens (    8.45 ms per token,   118.37 tokens per second)\n",
      "llama_print_timings:        eval time =    2031.04 ms /    38 runs   (   53.45 ms per token,    18.71 tokens per second)\n",
      "llama_print_timings:       total time =    2284.10 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "438 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.95 ms /    40 runs   (    0.10 ms per token, 10121.46 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.35 ms /    19 tokens (   10.33 ms per token,    96.77 tokens per second)\n",
      "llama_print_timings:        eval time =    2082.29 ms /    39 runs   (   53.39 ms per token,    18.73 tokens per second)\n",
      "llama_print_timings:       total time =    2330.25 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "439 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.20 ms /   125 runs   (    0.09 ms per token, 11159.72 tokens per second)\n",
      "llama_print_timings: prompt eval time =     206.70 ms /    26 tokens (    7.95 ms per token,   125.79 tokens per second)\n",
      "llama_print_timings:        eval time =    6660.41 ms /   124 runs   (   53.71 ms per token,    18.62 tokens per second)\n",
      "llama_print_timings:       total time =    7027.89 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "440 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      11.64 ms /   133 runs   (    0.09 ms per token, 11423.17 tokens per second)\n",
      "llama_print_timings: prompt eval time =     366.48 ms /    48 tokens (    7.63 ms per token,   130.98 tokens per second)\n",
      "llama_print_timings:        eval time =    7104.00 ms /   132 runs   (   53.82 ms per token,    18.58 tokens per second)\n",
      "llama_print_timings:       total time =    7639.48 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "441 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.87 ms /    40 runs   (    0.10 ms per token, 10349.29 tokens per second)\n",
      "llama_print_timings: prompt eval time =     184.73 ms /    10 tokens (   18.47 ms per token,    54.13 tokens per second)\n",
      "llama_print_timings:        eval time =    2079.65 ms /    39 runs   (   53.32 ms per token,    18.75 tokens per second)\n",
      "llama_print_timings:       total time =    2316.05 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "442 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =      12.36 ms /   127 runs   (    0.10 ms per token, 10273.42 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.77 ms /    21 tokens (    9.47 ms per token,   105.65 tokens per second)\n",
      "llama_print_timings:        eval time =    6760.77 ms /   126 runs   (   53.66 ms per token,    18.64 tokens per second)\n",
      "llama_print_timings:       total time =    7122.79 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "443 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.87 ms /    39 runs   (    0.10 ms per token, 10082.73 tokens per second)\n",
      "llama_print_timings: prompt eval time =     207.12 ms /    28 tokens (    7.40 ms per token,   135.19 tokens per second)\n",
      "llama_print_timings:        eval time =    2031.75 ms /    38 runs   (   53.47 ms per token,    18.70 tokens per second)\n",
      "llama_print_timings:       total time =    2288.93 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "444 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4274.02 ms\n",
      "llama_print_timings:      sample time =       3.85 ms /    40 runs   (    0.10 ms per token, 10395.01 tokens per second)\n",
      "llama_print_timings: prompt eval time =     190.95 ms /    16 tokens (   11.93 ms per token,    83.79 tokens per second)\n",
      "llama_print_timings:        eval time =    2083.72 ms /    39 runs   (   53.43 ms per token,    18.72 tokens per second)\n",
      "llama_print_timings:       total time =    2324.77 ms\n"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "i = 0\n",
    "for _, row in test.iterrows():\n",
    "  prepared_prompt = prompt + row['text'] + \"\\n### OUTPUT:\\n\"\n",
    "  json_out = ''\n",
    "  attempt = 0\n",
    "  while True:\n",
    "    attempt += 1\n",
    "    json_out = generate(prepared_prompt, attempt)\n",
    "    parsedJson = parseResult(json_out, row)\n",
    "    print(f\"{i} -- {attempt}\")\n",
    "    if parsedJson is not None:\n",
    "      parsedJson['id'] = row['id']\n",
    "      parsedJson['text'] = row['text']\n",
    "      results.append(parsedJson)\n",
    "      i+=1\n",
    "      break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "445"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df = pd.DataFrame(results, columns=['id', 'text', 'output', 'ambiguity', 'distinctiveness',])\n",
    "results_df.to_json('outputs-task1-hetero-openocra7b.json', orient=\"records\")\n",
    "len(results_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df['output'] = results_df['output'].replace({True: 1, False: 0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = [int(x) for x in list(test['label'])]\n",
    "y_pred = [int(x) for x in list(results_df['output'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.4358    0.7480    0.5507       127\n",
      "           1     0.8590    0.6132    0.7156       318\n",
      "\n",
      "    accuracy                         0.6517       445\n",
      "   macro avg     0.6474    0.6806    0.6332       445\n",
      "weighted avg     0.7382    0.6517    0.6685       445\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_true, y_pred, digits=4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 2 and 3: Pun Location & Interpretation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_sentence(text):\n",
    "  res = text.replace(\" ' ' \", '\" ')\n",
    "  res = res.replace(\" ' \", \"'\")\n",
    "  res = res.replace(\" , \", \", \")\n",
    "  res = res.replace(\" . \", \". \")\n",
    "  res = res.replace(\" .\", \". \")\n",
    "  res = res.replace(\" ! \", \"! \")\n",
    "  res = res.replace(\" !\", \"! \")\n",
    "  res = res.replace(\" ? \", \"? \")\n",
    "  res = res.replace(\" ?\", \"? \")\n",
    "  res = res.replace(\" - \", \"-\")\n",
    "  return res.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>pun_word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>In some countries there is a lot of Manuel labor.</td>\n",
       "      <td>Manuel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Someone who attempts long distance fraud is kn...</td>\n",
       "      <td>phony</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>In the delivery room, the unborn infant expect...</td>\n",
       "      <td>berth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Beautifully manicured lawns are highly sod after.</td>\n",
       "      <td>sod</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I saw something similar to moss the other day,...</td>\n",
       "      <td>lichen</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text pun_word\n",
       "0  In some countries there is a lot of Manuel labor.   Manuel\n",
       "1  Someone who attempts long distance fraud is kn...    phony\n",
       "2  In the delivery room, the unborn infant expect...    berth\n",
       "3  Beautifully manicured lawns are highly sod after.      sod\n",
       "4  I saw something similar to moss the other day,...   lichen"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_json('data/llm_task2_hetero.json', orient=\"records\")\n",
    "df['sentence'] = df['sentence'].apply(clean_sentence)\n",
    "df = df.rename(columns={'sentence': 'text'})\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "318"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = df\n",
    "len(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = f''' ### INSTRUCTION:\n",
    "You are to identify the pun word in a given sentence based on the following criteria:\n",
    "1. Ambiguity -  there exists a word in the sentence that has two similarly likely interpretations.\n",
    "2. Distinctiveness - the two interpretations are very different from each other i.e. how distinct are the words semantically related to the two interpretations from each other. There needs to be at least one different word in the set of words supporting each interpretation.\\n\n",
    "For a word to be a pun word, it should satisfy BOTH criteria - Ambiguity and Distinctiveness.\\n\n",
    "It is guaranteed that the sentence has a pun word and hence the selected pun word must be from the sentence in the same form as it appears.\\n\n",
    "Also, list all the words that realize each interpretation of the pun word. If there are multiple words that support an interpretation, separate them with a semicolon.\\n\n",
    "Here are two examples of the cases you will encounter:\n",
    "1. An example with heterographic puns (words that sound similar) - \n",
    "The magician got so mad that he pulled his hare out.\n",
    "{{\n",
    "  \"pun_word\": \"hare\",\n",
    "  \"target_word\": \"hair\",\n",
    "  \"source_sense\": \"magician\",\n",
    "  \"target_sense\": \"angry;pulled\"\n",
    "}}\n",
    "\n",
    "In the above sentence, the pun word 'hare' supports two plausible interpretations of 'hare' meaning a rabbit and 'hair' meaning human hair. In the given sentence, the words 'magician' relates to 'hare' while 'angry' and 'pulled' refer to the second interpretation of 'hair'.\n",
    "\n",
    "2. An example with homographic puns (same words with different senses) -\n",
    "I used to be a banker but I lost interest.\n",
    "{{\n",
    "  \"pun_word\": \"interest\",\n",
    "  \"target_word\": \"interest\",\n",
    "  \"source_sense\": \"banker\",\n",
    "  \"target_sense\": \"used;to;be\"\n",
    "}}\n",
    "\n",
    "In the above sentence, the pun word 'interest' supports two distinct interpretations of interest meaning 'financial interest' and interest meaning 'desire to do something'. The words 'banker' realize the first sense of financial interest while the words 'used', 'to', 'be' refer to the second sense.\n",
    "\n",
    "Identify the pun word in the input sentence is a pun and list the target word and the supporting words for each sense the result based on ambiguity and distinctiveness in valid JSON format. Do not explain. Generate a response in the form of a valid JSON object with four keys: pun_word, target_word, source_sense and target_sense.\\n\n",
    "### INPUT:\\n\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.92 ms /    52 runs   (    0.15 ms per token,  6561.51 tokens per second)\n",
      "llama_print_timings: prompt eval time =    6128.85 ms /    17 tokens (  360.52 ms per token,     2.77 tokens per second)\n",
      "llama_print_timings:        eval time =    2709.51 ms /    51 runs   (   53.13 ms per token,    18.82 tokens per second)\n",
      "llama_print_timings:       total time =    8960.64 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.39 ms /    56 runs   (    0.10 ms per token, 10391.54 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.59 ms /    20 tokens (    9.73 ms per token,   102.78 tokens per second)\n",
      "llama_print_timings:        eval time =    2906.99 ms /    55 runs   (   52.85 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    3177.65 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.75 ms /    57 runs   (    0.14 ms per token,  7356.74 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.27 ms /    24 tokens (    8.34 ms per token,   119.84 tokens per second)\n",
      "llama_print_timings:        eval time =    2973.79 ms /    56 runs   (   53.10 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    3276.50 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.36 ms /    57 runs   (    0.09 ms per token, 10632.34 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.82 ms /    20 tokens (    9.79 ms per token,   102.14 tokens per second)\n",
      "llama_print_timings:        eval time =    2958.69 ms /    56 runs   (   52.83 ms per token,    18.93 tokens per second)\n",
      "llama_print_timings:       total time =    3231.67 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.81 ms /    57 runs   (    0.14 ms per token,  7297.40 tokens per second)\n",
      "llama_print_timings: prompt eval time =     210.32 ms /    30 tokens (    7.01 ms per token,   142.64 tokens per second)\n",
      "llama_print_timings:        eval time =    2982.88 ms /    56 runs   (   53.27 ms per token,    18.77 tokens per second)\n",
      "llama_print_timings:       total time =    3296.10 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.76 ms /    54 runs   (    0.13 ms per token,  7991.71 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.72 ms /    22 tokens (    9.08 ms per token,   110.15 tokens per second)\n",
      "llama_print_timings:        eval time =    2820.93 ms /    53 runs   (   53.23 ms per token,    18.79 tokens per second)\n",
      "llama_print_timings:       total time =    3110.69 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.61 ms /    57 runs   (    0.13 ms per token,  7488.18 tokens per second)\n",
      "llama_print_timings: prompt eval time =     190.63 ms /    16 tokens (   11.91 ms per token,    83.93 tokens per second)\n",
      "llama_print_timings:        eval time =    2984.69 ms /    56 runs   (   53.30 ms per token,    18.76 tokens per second)\n",
      "llama_print_timings:       total time =    3281.56 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6 -- 1\n",
      "7 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.25 ms /    58 runs   (    0.12 ms per token,  8001.10 tokens per second)\n",
      "llama_print_timings: prompt eval time =     204.26 ms /    25 tokens (    8.17 ms per token,   122.39 tokens per second)\n",
      "llama_print_timings:        eval time =    3038.69 ms /    57 runs   (   53.31 ms per token,    18.76 tokens per second)\n",
      "llama_print_timings:       total time =    3338.49 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.50 ms /    56 runs   (    0.13 ms per token,  7465.67 tokens per second)\n",
      "llama_print_timings: prompt eval time =     209.81 ms /    29 tokens (    7.23 ms per token,   138.22 tokens per second)\n",
      "llama_print_timings:        eval time =    2936.02 ms /    55 runs   (   53.38 ms per token,    18.73 tokens per second)\n",
      "llama_print_timings:       total time =    3246.28 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.39 ms /    56 runs   (    0.11 ms per token,  8769.18 tokens per second)\n",
      "llama_print_timings: prompt eval time =     346.86 ms /    34 tokens (   10.20 ms per token,    98.02 tokens per second)\n",
      "llama_print_timings:        eval time =    2926.73 ms /    55 runs   (   53.21 ms per token,    18.79 tokens per second)\n",
      "llama_print_timings:       total time =    3356.57 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.09 ms /    53 runs   (    0.11 ms per token,  8697.08 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.82 ms /    24 tokens (    8.37 ms per token,   119.51 tokens per second)\n",
      "llama_print_timings:        eval time =    2773.39 ms /    52 runs   (   53.33 ms per token,    18.75 tokens per second)\n",
      "llama_print_timings:       total time =    3057.66 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.83 ms /    55 runs   (    0.12 ms per token,  8050.35 tokens per second)\n",
      "llama_print_timings: prompt eval time =     216.72 ms /    31 tokens (    6.99 ms per token,   143.04 tokens per second)\n",
      "llama_print_timings:        eval time =    2890.77 ms /    54 runs   (   53.53 ms per token,    18.68 tokens per second)\n",
      "llama_print_timings:       total time =    3197.57 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.44 ms /    56 runs   (    0.11 ms per token,  8698.35 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.03 ms /    22 tokens (    9.05 ms per token,   110.54 tokens per second)\n",
      "llama_print_timings:        eval time =    2930.64 ms /    55 runs   (   53.28 ms per token,    18.77 tokens per second)\n",
      "llama_print_timings:       total time =    3219.37 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12 -- 1\n",
      "13 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.42 ms /    53 runs   (    0.10 ms per token,  9774.99 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.82 ms /    19 tokens (   10.25 ms per token,    97.53 tokens per second)\n",
      "llama_print_timings:        eval time =    2781.39 ms /    52 runs   (   53.49 ms per token,    18.70 tokens per second)\n",
      "llama_print_timings:       total time =    3052.89 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.68 ms /    55 runs   (    0.12 ms per token,  8232.30 tokens per second)\n",
      "llama_print_timings: prompt eval time =     201.77 ms /    24 tokens (    8.41 ms per token,   118.94 tokens per second)\n",
      "llama_print_timings:        eval time =    2865.48 ms /    54 runs   (   53.06 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3156.28 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14 -- 1\n",
      "15 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.75 ms /    55 runs   (    0.12 ms per token,  8142.12 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.19 ms /    28 tokens (    6.76 ms per token,   148.00 tokens per second)\n",
      "llama_print_timings:        eval time =    2882.84 ms /    54 runs   (   53.39 ms per token,    18.73 tokens per second)\n",
      "llama_print_timings:       total time =    3165.04 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.01 ms /    53 runs   (    0.09 ms per token, 10574.62 tokens per second)\n",
      "llama_print_timings: prompt eval time =     182.60 ms /    10 tokens (   18.26 ms per token,    54.76 tokens per second)\n",
      "llama_print_timings:        eval time =    2750.45 ms /    52 runs   (   52.89 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    3004.93 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.09 ms /    54 runs   (    0.11 ms per token,  8869.91 tokens per second)\n",
      "llama_print_timings: prompt eval time =     211.22 ms /    30 tokens (    7.04 ms per token,   142.03 tokens per second)\n",
      "llama_print_timings:        eval time =    2811.74 ms /    53 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3104.33 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17 -- 1\n",
      "18 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.84 ms /    64 runs   (    0.12 ms per token,  8158.06 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.66 ms /    20 tokens (    9.83 ms per token,   101.70 tokens per second)\n",
      "llama_print_timings:        eval time =    3398.61 ms /    63 runs   (   53.95 ms per token,    18.54 tokens per second)\n",
      "llama_print_timings:       total time =    3709.75 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.83 ms /    52 runs   (    0.13 ms per token,  7613.47 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.18 ms /    17 tokens (   11.30 ms per token,    88.46 tokens per second)\n",
      "llama_print_timings:        eval time =    2725.75 ms /    51 runs   (   53.45 ms per token,    18.71 tokens per second)\n",
      "llama_print_timings:       total time =    3015.79 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.15 ms /    52 runs   (    0.12 ms per token,  8456.66 tokens per second)\n",
      "llama_print_timings: prompt eval time =     201.12 ms /    21 tokens (    9.58 ms per token,   104.42 tokens per second)\n",
      "llama_print_timings:        eval time =    2755.42 ms /    51 runs   (   54.03 ms per token,    18.51 tokens per second)\n",
      "llama_print_timings:       total time =    3042.55 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.88 ms /    53 runs   (    0.13 ms per token,  7706.85 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.27 ms /    19 tokens (   10.28 ms per token,    97.30 tokens per second)\n",
      "llama_print_timings:        eval time =    2782.39 ms /    52 runs   (   53.51 ms per token,    18.69 tokens per second)\n",
      "llama_print_timings:       total time =    3072.40 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.24 ms /    54 runs   (    0.12 ms per token,  8655.23 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.23 ms /    15 tokens (   13.08 ms per token,    76.44 tokens per second)\n",
      "llama_print_timings:        eval time =    2815.35 ms /    53 runs   (   53.12 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    3099.63 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.75 ms /    53 runs   (    0.13 ms per token,  7851.85 tokens per second)\n",
      "llama_print_timings: prompt eval time =     212.61 ms /    31 tokens (    6.86 ms per token,   145.80 tokens per second)\n",
      "llama_print_timings:        eval time =    2786.56 ms /    52 runs   (   53.59 ms per token,    18.66 tokens per second)\n",
      "llama_print_timings:       total time =    3087.51 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.30 ms /    59 runs   (    0.12 ms per token,  8083.30 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.73 ms /    28 tokens (    6.78 ms per token,   147.58 tokens per second)\n",
      "llama_print_timings:        eval time =    3083.88 ms /    58 runs   (   53.17 ms per token,    18.81 tokens per second)\n",
      "llama_print_timings:       total time =    3374.10 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.99 ms /    56 runs   (    0.12 ms per token,  8010.30 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.19 ms /    17 tokens (   11.25 ms per token,    88.92 tokens per second)\n",
      "llama_print_timings:        eval time =    2927.39 ms /    55 runs   (   53.23 ms per token,    18.79 tokens per second)\n",
      "llama_print_timings:       total time =    3223.59 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25 -- 1\n",
      "26 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.65 ms /    56 runs   (    0.12 ms per token,  8417.26 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.72 ms /    28 tokens (    6.78 ms per token,   147.58 tokens per second)\n",
      "llama_print_timings:        eval time =    2912.76 ms /    55 runs   (   52.96 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3194.65 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.52 ms /    52 runs   (    0.13 ms per token,  7979.13 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.21 ms /    21 tokens (    9.39 ms per token,   106.49 tokens per second)\n",
      "llama_print_timings:        eval time =    2708.84 ms /    51 runs   (   53.11 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    2996.14 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      10.17 ms /    62 runs   (    0.16 ms per token,  6098.76 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.31 ms /    19 tokens (   10.28 ms per token,    97.28 tokens per second)\n",
      "llama_print_timings:        eval time =    3261.90 ms /    61 runs   (   53.47 ms per token,    18.70 tokens per second)\n",
      "llama_print_timings:       total time =    3592.56 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.17 ms /    51 runs   (    0.10 ms per token,  9866.51 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.12 ms /    20 tokens (    9.76 ms per token,   102.50 tokens per second)\n",
      "llama_print_timings:        eval time =    2655.77 ms /    50 runs   (   53.12 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    2926.21 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.14 ms /    53 runs   (    0.10 ms per token, 10311.28 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.93 ms /    24 tokens (    8.37 ms per token,   119.44 tokens per second)\n",
      "llama_print_timings:        eval time =    2753.20 ms /    52 runs   (   52.95 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3025.39 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.30 ms /    55 runs   (    0.10 ms per token, 10371.49 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.26 ms /    21 tokens (    9.39 ms per token,   106.46 tokens per second)\n",
      "llama_print_timings:        eval time =    2865.49 ms /    54 runs   (   53.06 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3137.27 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.51 ms /    57 runs   (    0.10 ms per token, 10348.58 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.54 ms /    23 tokens (    8.72 ms per token,   114.69 tokens per second)\n",
      "llama_print_timings:        eval time =    2971.33 ms /    56 runs   (   53.06 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3249.38 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "33 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.76 ms /    59 runs   (    0.10 ms per token, 10244.83 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.08 ms /    26 tokens (    7.89 ms per token,   126.78 tokens per second)\n",
      "llama_print_timings:        eval time =    3071.92 ms /    58 runs   (   52.96 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3356.28 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.00 ms /    58 runs   (    0.10 ms per token,  9673.12 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.59 ms /    22 tokens (    9.07 ms per token,   110.23 tokens per second)\n",
      "llama_print_timings:        eval time =    3018.96 ms /    57 runs   (   52.96 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3300.84 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.23 ms /    63 runs   (    0.10 ms per token, 10107.49 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.49 ms /    32 tokens (    6.42 ms per token,   155.73 tokens per second)\n",
      "llama_print_timings:        eval time =    3292.95 ms /    62 runs   (   53.11 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    3583.47 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.72 ms /    60 runs   (    0.10 ms per token, 10482.18 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.47 ms /    21 tokens (    9.40 ms per token,   106.34 tokens per second)\n",
      "llama_print_timings:        eval time =    3123.93 ms /    59 runs   (   52.95 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3401.31 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "37 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.50 ms /    56 runs   (    0.10 ms per token, 10178.12 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.29 ms /    19 tokens (   10.23 ms per token,    97.79 tokens per second)\n",
      "llama_print_timings:        eval time =    2914.57 ms /    55 runs   (   52.99 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3185.33 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "38 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.57 ms /    57 runs   (    0.10 ms per token, 10237.07 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.48 ms /    22 tokens (    9.07 ms per token,   110.29 tokens per second)\n",
      "llama_print_timings:        eval time =    2963.00 ms /    56 runs   (   52.91 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    3239.04 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      34.16 ms /    61 runs   (    0.56 ms per token,  1785.82 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.32 ms /    28 tokens (    6.76 ms per token,   147.90 tokens per second)\n",
      "llama_print_timings:        eval time =    3293.84 ms /    60 runs   (   54.90 ms per token,    18.22 tokens per second)\n",
      "llama_print_timings:       total time =    3835.69 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39 -- 1\n",
      "40 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      15.71 ms /    56 runs   (    0.28 ms per token,  3564.15 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.52 ms /    22 tokens (    9.11 ms per token,   109.72 tokens per second)\n",
      "llama_print_timings:        eval time =    2966.62 ms /    55 runs   (   53.94 ms per token,    18.54 tokens per second)\n",
      "llama_print_timings:       total time =    3344.82 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "41 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      15.05 ms /    56 runs   (    0.27 ms per token,  3721.67 tokens per second)\n",
      "llama_print_timings: prompt eval time =     342.54 ms /    33 tokens (   10.38 ms per token,    96.34 tokens per second)\n",
      "llama_print_timings:        eval time =    3001.40 ms /    55 runs   (   54.57 ms per token,    18.32 tokens per second)\n",
      "llama_print_timings:       total time =    3580.43 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "42 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       8.00 ms /    56 runs   (    0.14 ms per token,  6996.50 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.95 ms /    18 tokens (   10.72 ms per token,    93.29 tokens per second)\n",
      "llama_print_timings:        eval time =    2920.48 ms /    55 runs   (   53.10 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    3224.65 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "43 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.46 ms /    59 runs   (    0.11 ms per token,  9140.20 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.57 ms /    21 tokens (    9.41 ms per token,   106.29 tokens per second)\n",
      "llama_print_timings:        eval time =    3073.80 ms /    58 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3359.49 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "44 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.78 ms /    52 runs   (    0.13 ms per token,  7668.49 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.21 ms /    17 tokens (   11.25 ms per token,    88.91 tokens per second)\n",
      "llama_print_timings:        eval time =    2706.53 ms /    51 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    2993.54 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.38 ms /    52 runs   (    0.10 ms per token,  9661.84 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.88 ms /    20 tokens (    9.74 ms per token,   102.63 tokens per second)\n",
      "llama_print_timings:        eval time =    2699.03 ms /    51 runs   (   52.92 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    2969.57 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "45 -- 1\n",
      "46 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.57 ms /    57 runs   (    0.10 ms per token, 10227.88 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.45 ms /    21 tokens (    9.40 ms per token,   106.36 tokens per second)\n",
      "llama_print_timings:        eval time =    2966.45 ms /    56 runs   (   52.97 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3242.97 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.11 ms /    55 runs   (    0.13 ms per token,  7734.50 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.78 ms /    24 tokens (    8.37 ms per token,   119.53 tokens per second)\n",
      "llama_print_timings:        eval time =    2886.71 ms /    54 runs   (   53.46 ms per token,    18.71 tokens per second)\n",
      "llama_print_timings:       total time =    3187.71 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "47 -- 1\n",
      "48 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.75 ms /    54 runs   (    0.14 ms per token,  6966.84 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.92 ms /    27 tokens (    7.63 ms per token,   131.12 tokens per second)\n",
      "llama_print_timings:        eval time =    2819.70 ms /    53 runs   (   53.20 ms per token,    18.80 tokens per second)\n",
      "llama_print_timings:       total time =    3128.38 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.91 ms /    56 runs   (    0.14 ms per token,  7082.33 tokens per second)\n",
      "llama_print_timings: prompt eval time =     202.07 ms /    25 tokens (    8.08 ms per token,   123.72 tokens per second)\n",
      "llama_print_timings:        eval time =    2919.95 ms /    55 runs   (   53.09 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3229.38 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       8.31 ms /    62 runs   (    0.13 ms per token,  7464.48 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.29 ms /    20 tokens (    9.71 ms per token,   102.94 tokens per second)\n",
      "llama_print_timings:        eval time =    3255.95 ms /    61 runs   (   53.38 ms per token,    18.73 tokens per second)\n",
      "llama_print_timings:       total time =    3571.52 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50 -- 1\n",
      "51 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.97 ms /    52 runs   (    0.10 ms per token, 10466.99 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.60 ms /    16 tokens (   11.79 ms per token,    84.84 tokens per second)\n",
      "llama_print_timings:        eval time =    2737.07 ms /    51 runs   (   53.67 ms per token,    18.63 tokens per second)\n",
      "llama_print_timings:       total time =    2997.93 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "52 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      20.62 ms /    57 runs   (    0.36 ms per token,  2763.77 tokens per second)\n",
      "llama_print_timings: prompt eval time =     368.40 ms /    49 tokens (    7.52 ms per token,   133.01 tokens per second)\n",
      "llama_print_timings:        eval time =    3060.21 ms /    56 runs   (   54.65 ms per token,    18.30 tokens per second)\n",
      "llama_print_timings:       total time =    3716.86 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      27.10 ms /    50 runs   (    0.54 ms per token,  1844.75 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.33 ms /    22 tokens (    9.06 ms per token,   110.37 tokens per second)\n",
      "llama_print_timings:        eval time =    2709.64 ms /    49 runs   (   55.30 ms per token,    18.08 tokens per second)\n",
      "llama_print_timings:       total time =    3177.46 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "53 -- 1\n",
      "54 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      21.07 ms /    52 runs   (    0.41 ms per token,  2467.50 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.14 ms /    20 tokens (    9.96 ms per token,   100.43 tokens per second)\n",
      "llama_print_timings:        eval time =    2819.26 ms /    51 runs   (   55.28 ms per token,    18.09 tokens per second)\n",
      "llama_print_timings:       total time =    3270.37 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "55 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.40 ms /    53 runs   (    0.10 ms per token,  9811.18 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.55 ms /    24 tokens (    8.36 ms per token,   119.67 tokens per second)\n",
      "llama_print_timings:        eval time =    2756.22 ms /    52 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3031.07 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "56 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.97 ms /    54 runs   (    0.13 ms per token,  7745.27 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.99 ms /    19 tokens (   10.21 ms per token,    97.94 tokens per second)\n",
      "llama_print_timings:        eval time =    2815.30 ms /    53 runs   (   53.12 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    3097.72 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "57 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      14.57 ms /    51 runs   (    0.29 ms per token,  3500.58 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.47 ms /    21 tokens (    9.40 ms per token,   106.34 tokens per second)\n",
      "llama_print_timings:        eval time =    2711.97 ms /    50 runs   (   54.24 ms per token,    18.44 tokens per second)\n",
      "llama_print_timings:       total time =    3097.78 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      25.47 ms /    52 runs   (    0.49 ms per token,  2041.46 tokens per second)\n",
      "llama_print_timings: prompt eval time =     201.81 ms /    23 tokens (    8.77 ms per token,   113.97 tokens per second)\n",
      "llama_print_timings:        eval time =    2802.79 ms /    51 runs   (   54.96 ms per token,    18.20 tokens per second)\n",
      "llama_print_timings:       total time =    3297.62 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "58 -- 1\n",
      "59 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.82 ms /    57 runs   (    0.10 ms per token,  9797.18 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.71 ms /    19 tokens (   10.25 ms per token,    97.58 tokens per second)\n",
      "llama_print_timings:        eval time =    2971.93 ms /    56 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3248.27 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      10.18 ms /    56 runs   (    0.18 ms per token,  5500.44 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.12 ms /    18 tokens (   10.73 ms per token,    93.21 tokens per second)\n",
      "llama_print_timings:        eval time =    2934.78 ms /    55 runs   (   53.36 ms per token,    18.74 tokens per second)\n",
      "llama_print_timings:       total time =    3238.15 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "61 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.44 ms /    57 runs   (    0.10 ms per token, 10485.65 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.08 ms /    17 tokens (   11.24 ms per token,    88.97 tokens per second)\n",
      "llama_print_timings:        eval time =    2968.11 ms /    56 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3235.97 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "62 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      20.39 ms /    59 runs   (    0.35 ms per token,  2893.01 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.14 ms /    19 tokens (   10.22 ms per token,    97.87 tokens per second)\n",
      "llama_print_timings:        eval time =    3162.60 ms /    58 runs   (   54.53 ms per token,    18.34 tokens per second)\n",
      "llama_print_timings:       total time =    3610.25 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      32.95 ms /    56 runs   (    0.59 ms per token,  1699.44 tokens per second)\n",
      "llama_print_timings: prompt eval time =     203.33 ms /    25 tokens (    8.13 ms per token,   122.96 tokens per second)\n",
      "llama_print_timings:        eval time =    3004.39 ms /    55 runs   (   54.63 ms per token,    18.31 tokens per second)\n",
      "llama_print_timings:       total time =    3555.41 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "63 -- 1\n",
      "64 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       8.60 ms /    56 runs   (    0.15 ms per token,  6510.11 tokens per second)\n",
      "llama_print_timings: prompt eval time =     190.00 ms /    16 tokens (   11.88 ms per token,    84.21 tokens per second)\n",
      "llama_print_timings:        eval time =    2929.22 ms /    55 runs   (   53.26 ms per token,    18.78 tokens per second)\n",
      "llama_print_timings:       total time =    3242.85 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      15.78 ms /    60 runs   (    0.26 ms per token,  3801.32 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.72 ms /    22 tokens (    9.03 ms per token,   110.71 tokens per second)\n",
      "llama_print_timings:        eval time =    3194.65 ms /    59 runs   (   54.15 ms per token,    18.47 tokens per second)\n",
      "llama_print_timings:       total time =    3595.00 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      22.67 ms /    55 runs   (    0.41 ms per token,  2426.54 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.24 ms /    16 tokens (   11.83 ms per token,    84.55 tokens per second)\n",
      "llama_print_timings:        eval time =    2964.73 ms /    54 runs   (   54.90 ms per token,    18.21 tokens per second)\n",
      "llama_print_timings:       total time =    3459.56 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "67 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.21 ms /    54 runs   (    0.12 ms per token,  8690.05 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.78 ms /    21 tokens (    9.42 ms per token,   106.18 tokens per second)\n",
      "llama_print_timings:        eval time =    2820.45 ms /    53 runs   (   53.22 ms per token,    18.79 tokens per second)\n",
      "llama_print_timings:       total time =    3106.10 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "68 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.62 ms /    57 runs   (    0.10 ms per token, 10140.54 tokens per second)\n",
      "llama_print_timings: prompt eval time =     209.61 ms /    29 tokens (    7.23 ms per token,   138.35 tokens per second)\n",
      "llama_print_timings:        eval time =    2969.03 ms /    56 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3255.57 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "69 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.20 ms /    53 runs   (    0.10 ms per token, 10192.31 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.55 ms /    15 tokens (   12.64 ms per token,    79.13 tokens per second)\n",
      "llama_print_timings:        eval time =    2754.22 ms /    52 runs   (   52.97 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3018.06 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "70 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.23 ms /    52 runs   (    0.10 ms per token,  9942.64 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.35 ms /    17 tokens (   11.26 ms per token,    88.84 tokens per second)\n",
      "llama_print_timings:        eval time =    2698.93 ms /    51 runs   (   52.92 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    2962.27 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "71 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      13.80 ms /    54 runs   (    0.26 ms per token,  3911.91 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.68 ms /    19 tokens (   10.25 ms per token,    97.60 tokens per second)\n",
      "llama_print_timings:        eval time =    2869.18 ms /    53 runs   (   54.14 ms per token,    18.47 tokens per second)\n",
      "llama_print_timings:       total time =    3246.40 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "72 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      11.08 ms /    54 runs   (    0.21 ms per token,  4873.65 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.05 ms /    16 tokens (   11.82 ms per token,    84.63 tokens per second)\n",
      "llama_print_timings:        eval time =    2840.26 ms /    53 runs   (   53.59 ms per token,    18.66 tokens per second)\n",
      "llama_print_timings:       total time =    3176.25 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "73 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.55 ms /    60 runs   (    0.09 ms per token, 10806.92 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.10 ms /    21 tokens (    9.39 ms per token,   106.54 tokens per second)\n",
      "llama_print_timings:        eval time =    3128.38 ms /    59 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3405.27 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "74 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       9.05 ms /    52 runs   (    0.17 ms per token,  5743.32 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.07 ms /    18 tokens (   10.73 ms per token,    93.23 tokens per second)\n",
      "llama_print_timings:        eval time =    2721.32 ms /    51 runs   (   53.36 ms per token,    18.74 tokens per second)\n",
      "llama_print_timings:       total time =    3029.38 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "75 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.72 ms /    57 runs   (    0.10 ms per token,  9963.29 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.10 ms /    23 tokens (    8.66 ms per token,   115.52 tokens per second)\n",
      "llama_print_timings:        eval time =    2968.24 ms /    56 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3247.91 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.35 ms /    58 runs   (    0.09 ms per token, 10843.15 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.12 ms /    17 tokens (   11.24 ms per token,    88.95 tokens per second)\n",
      "llama_print_timings:        eval time =    3021.54 ms /    57 runs   (   53.01 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3289.97 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "76 -- 1\n",
      "77 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.99 ms /    53 runs   (    0.09 ms per token, 10627.63 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.28 ms /    21 tokens (    9.44 ms per token,   105.91 tokens per second)\n",
      "llama_print_timings:        eval time =    2755.12 ms /    52 runs   (   52.98 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3024.98 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "78 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.39 ms /    53 runs   (    0.10 ms per token,  9827.55 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.02 ms /    17 tokens (   11.24 ms per token,    88.99 tokens per second)\n",
      "llama_print_timings:        eval time =    2751.93 ms /    52 runs   (   52.92 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    3018.77 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "79 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.63 ms /    58 runs   (    0.11 ms per token,  8746.80 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.04 ms /    20 tokens (    9.75 ms per token,   102.54 tokens per second)\n",
      "llama_print_timings:        eval time =    3048.21 ms /    57 runs   (   53.48 ms per token,    18.70 tokens per second)\n",
      "llama_print_timings:       total time =    3341.65 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "80 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.59 ms /    58 runs   (    0.10 ms per token, 10370.11 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.93 ms /    27 tokens (    7.63 ms per token,   131.11 tokens per second)\n",
      "llama_print_timings:        eval time =    3018.74 ms /    57 runs   (   52.96 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3303.32 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      11.60 ms /    57 runs   (    0.20 ms per token,  4913.37 tokens per second)\n",
      "llama_print_timings: prompt eval time =     209.00 ms /    29 tokens (    7.21 ms per token,   138.76 tokens per second)\n",
      "llama_print_timings:        eval time =    2997.27 ms /    56 runs   (   53.52 ms per token,    18.68 tokens per second)\n",
      "llama_print_timings:       total time =    3347.99 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "82 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.90 ms /    53 runs   (    0.09 ms per token, 10807.50 tokens per second)\n",
      "llama_print_timings: prompt eval time =     569.19 ms /    95 tokens (    5.99 ms per token,   166.90 tokens per second)\n",
      "llama_print_timings:        eval time =    2772.66 ms /    52 runs   (   53.32 ms per token,    18.75 tokens per second)\n",
      "llama_print_timings:       total time =    3415.75 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "83 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.45 ms /    58 runs   (    0.09 ms per token, 10638.30 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.08 ms /    21 tokens (    9.38 ms per token,   106.56 tokens per second)\n",
      "llama_print_timings:        eval time =    3019.44 ms /    57 runs   (   52.97 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3294.73 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "84 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.98 ms /    59 runs   (    0.10 ms per token,  9872.82 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.85 ms /    28 tokens (    6.78 ms per token,   147.48 tokens per second)\n",
      "llama_print_timings:        eval time =    3074.63 ms /    58 runs   (   53.01 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3347.13 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "85 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.74 ms /    51 runs   (    0.09 ms per token, 10768.58 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.31 ms /    18 tokens (   10.74 ms per token,    93.12 tokens per second)\n",
      "llama_print_timings:        eval time =    2645.16 ms /    50 runs   (   52.90 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    2907.25 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "86 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.72 ms /    58 runs   (    0.10 ms per token, 10141.63 tokens per second)\n",
      "llama_print_timings: prompt eval time =     346.42 ms /    34 tokens (   10.19 ms per token,    98.15 tokens per second)\n",
      "llama_print_timings:        eval time =    3023.03 ms /    57 runs   (   53.04 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3447.14 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "87 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.70 ms /    56 runs   (    0.10 ms per token,  9822.84 tokens per second)\n",
      "llama_print_timings: prompt eval time =     210.42 ms /    30 tokens (    7.01 ms per token,   142.57 tokens per second)\n",
      "llama_print_timings:        eval time =    2915.20 ms /    55 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3204.85 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "88 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.17 ms /    53 runs   (    0.10 ms per token, 10245.51 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.06 ms /    20 tokens (    9.75 ms per token,   102.53 tokens per second)\n",
      "llama_print_timings:        eval time =    2753.12 ms /    52 runs   (   52.94 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3019.47 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "89 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.12 ms /    53 runs   (    0.10 ms per token, 10361.68 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.58 ms /    21 tokens (    9.41 ms per token,   106.29 tokens per second)\n",
      "llama_print_timings:        eval time =    2754.14 ms /    52 runs   (   52.96 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3023.82 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "90 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.60 ms /    59 runs   (    0.09 ms per token, 10537.60 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.00 ms /    22 tokens (    9.05 ms per token,   110.56 tokens per second)\n",
      "llama_print_timings:        eval time =    3070.13 ms /    58 runs   (   52.93 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3347.67 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "91 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       8.88 ms /    59 runs   (    0.15 ms per token,  6640.41 tokens per second)\n",
      "llama_print_timings: prompt eval time =     351.92 ms /    38 tokens (    9.26 ms per token,   107.98 tokens per second)\n",
      "llama_print_timings:        eval time =    3097.27 ms /    58 runs   (   53.40 ms per token,    18.73 tokens per second)\n",
      "llama_print_timings:       total time =    3577.56 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "92 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      23.51 ms /    68 runs   (    0.35 ms per token,  2892.26 tokens per second)\n",
      "llama_print_timings: prompt eval time =     201.04 ms /    24 tokens (    8.38 ms per token,   119.38 tokens per second)\n",
      "llama_print_timings:        eval time =    3676.73 ms /    67 runs   (   54.88 ms per token,    18.22 tokens per second)\n",
      "llama_print_timings:       total time =    4195.24 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "93 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.87 ms /    52 runs   (    0.09 ms per token, 10686.40 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.97 ms /    17 tokens (   11.35 ms per token,    88.10 tokens per second)\n",
      "llama_print_timings:        eval time =    2701.80 ms /    51 runs   (   52.98 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    2965.18 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "94 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.70 ms /    58 runs   (    0.10 ms per token, 10184.37 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.75 ms /    21 tokens (    9.42 ms per token,   106.20 tokens per second)\n",
      "llama_print_timings:        eval time =    3020.05 ms /    57 runs   (   52.98 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3298.02 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.12 ms /    54 runs   (    0.09 ms per token, 10553.06 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.48 ms /    21 tokens (    9.40 ms per token,   106.34 tokens per second)\n",
      "llama_print_timings:        eval time =    2802.21 ms /    53 runs   (   52.87 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    3071.70 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "96 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      14.77 ms /    59 runs   (    0.25 ms per token,  3993.77 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.68 ms /    15 tokens (   12.65 ms per token,    79.08 tokens per second)\n",
      "llama_print_timings:        eval time =    3127.26 ms /    58 runs   (   53.92 ms per token,    18.55 tokens per second)\n",
      "llama_print_timings:       total time =    3485.97 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "97 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.13 ms /    54 runs   (    0.13 ms per token,  7576.82 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.55 ms /    20 tokens (    9.73 ms per token,   102.80 tokens per second)\n",
      "llama_print_timings:        eval time =    2849.25 ms /    53 runs   (   53.76 ms per token,    18.60 tokens per second)\n",
      "llama_print_timings:       total time =    3140.53 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "98 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.67 ms /    60 runs   (    0.09 ms per token, 10576.41 tokens per second)\n",
      "llama_print_timings: prompt eval time =     204.72 ms /    26 tokens (    7.87 ms per token,   127.00 tokens per second)\n",
      "llama_print_timings:        eval time =    3123.07 ms /    59 runs   (   52.93 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3407.50 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.09 ms /    61 runs   (    0.10 ms per token, 10021.36 tokens per second)\n",
      "llama_print_timings: prompt eval time =     357.63 ms /    43 tokens (    8.32 ms per token,   120.24 tokens per second)\n",
      "llama_print_timings:        eval time =    3182.28 ms /    60 runs   (   53.04 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3624.93 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.31 ms /    53 runs   (    0.10 ms per token,  9981.17 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.74 ms /    23 tokens (    8.73 ms per token,   114.57 tokens per second)\n",
      "llama_print_timings:        eval time =    2756.79 ms /    52 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3031.01 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "101 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.91 ms /    53 runs   (    0.09 ms per token, 10789.90 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.44 ms /    15 tokens (   12.63 ms per token,    79.18 tokens per second)\n",
      "llama_print_timings:        eval time =    2751.20 ms /    52 runs   (   52.91 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    3011.49 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "102 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      26.19 ms /    54 runs   (    0.48 ms per token,  2062.01 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.36 ms /    14 tokens (   13.45 ms per token,    74.32 tokens per second)\n",
      "llama_print_timings:        eval time =    2870.49 ms /    53 runs   (   54.16 ms per token,    18.46 tokens per second)\n",
      "llama_print_timings:       total time =    3335.35 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "103 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.86 ms /    55 runs   (    0.12 ms per token,  8021.00 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.50 ms /    18 tokens (   10.75 ms per token,    93.02 tokens per second)\n",
      "llama_print_timings:        eval time =    2864.28 ms /    54 runs   (   53.04 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3150.07 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "104 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.45 ms /    58 runs   (    0.09 ms per token, 10638.30 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.40 ms /    19 tokens (   10.23 ms per token,    97.74 tokens per second)\n",
      "llama_print_timings:        eval time =    3019.56 ms /    57 runs   (   52.97 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3291.21 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "105 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       9.64 ms /    56 runs   (    0.17 ms per token,  5806.72 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.94 ms /    19 tokens (   10.21 ms per token,    97.97 tokens per second)\n",
      "llama_print_timings:        eval time =    2945.32 ms /    55 runs   (   53.55 ms per token,    18.67 tokens per second)\n",
      "llama_print_timings:       total time =    3271.66 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.37 ms /    57 runs   (    0.09 ms per token, 10620.46 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.68 ms /    22 tokens (    9.03 ms per token,   110.73 tokens per second)\n",
      "llama_print_timings:        eval time =    2964.32 ms /    56 runs   (   52.93 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3238.99 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "107 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.53 ms /    50 runs   (    0.09 ms per token, 11027.79 tokens per second)\n",
      "llama_print_timings: prompt eval time =     190.91 ms /    17 tokens (   11.23 ms per token,    89.05 tokens per second)\n",
      "llama_print_timings:        eval time =    2590.89 ms /    49 runs   (   52.88 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    2847.99 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "108 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      15.45 ms /   135 runs   (    0.11 ms per token,  8740.69 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.90 ms /    32 tokens (    6.43 ms per token,   155.41 tokens per second)\n",
      "llama_print_timings:        eval time =    7141.27 ms /   134 runs   (   53.29 ms per token,    18.76 tokens per second)\n",
      "llama_print_timings:       total time =    7571.98 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.78 ms /    58 runs   (    0.10 ms per token, 10036.34 tokens per second)\n",
      "llama_print_timings: prompt eval time =       0.00 ms /     1 tokens (    0.00 ms per token,      inf tokens per second)\n",
      "llama_print_timings:        eval time =    3073.85 ms /    58 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3151.85 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "108 -- 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      17.00 ms /    58 runs   (    0.29 ms per token,  3410.76 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.17 ms /    15 tokens (   12.54 ms per token,    79.71 tokens per second)\n",
      "llama_print_timings:        eval time =    3095.64 ms /    57 runs   (   54.31 ms per token,    18.41 tokens per second)\n",
      "llama_print_timings:       total time =    3513.66 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "109 -- 1\n",
      "110 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      22.39 ms /    53 runs   (    0.42 ms per token,  2367.55 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.14 ms /    18 tokens (   10.79 ms per token,    92.72 tokens per second)\n",
      "llama_print_timings:        eval time =    2849.27 ms /    52 runs   (   54.79 ms per token,    18.25 tokens per second)\n",
      "llama_print_timings:       total time =    3364.85 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "111 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      13.88 ms /    51 runs   (    0.27 ms per token,  3673.03 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.24 ms /    26 tokens (    7.89 ms per token,   126.68 tokens per second)\n",
      "llama_print_timings:        eval time =    2696.85 ms /    50 runs   (   53.94 ms per token,    18.54 tokens per second)\n",
      "llama_print_timings:       total time =    3082.23 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "112 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.46 ms /    64 runs   (    0.10 ms per token,  9911.72 tokens per second)\n",
      "llama_print_timings: prompt eval time =     206.03 ms /    32 tokens (    6.44 ms per token,   155.32 tokens per second)\n",
      "llama_print_timings:        eval time =    3340.15 ms /    63 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3632.70 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "113 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.18 ms /    64 runs   (    0.10 ms per token, 10359.34 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.39 ms /    23 tokens (    8.71 ms per token,   114.78 tokens per second)\n",
      "llama_print_timings:        eval time =    3335.58 ms /    63 runs   (   52.95 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3622.55 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "114 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       9.77 ms /    50 runs   (    0.20 ms per token,  5119.80 tokens per second)\n",
      "llama_print_timings: prompt eval time =     187.83 ms /    14 tokens (   13.42 ms per token,    74.54 tokens per second)\n",
      "llama_print_timings:        eval time =    2616.65 ms /    49 runs   (   53.40 ms per token,    18.73 tokens per second)\n",
      "llama_print_timings:       total time =    2936.55 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "115 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.89 ms /    61 runs   (    0.10 ms per token, 10360.05 tokens per second)\n",
      "llama_print_timings: prompt eval time =     210.58 ms /    30 tokens (    7.02 ms per token,   142.47 tokens per second)\n",
      "llama_print_timings:        eval time =    3180.46 ms /    60 runs   (   53.01 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3472.72 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "116 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      12.61 ms /    53 runs   (    0.24 ms per token,  4204.68 tokens per second)\n",
      "llama_print_timings: prompt eval time =     186.58 ms /    13 tokens (   14.35 ms per token,    69.68 tokens per second)\n",
      "llama_print_timings:        eval time =    2796.08 ms /    52 runs   (   53.77 ms per token,    18.60 tokens per second)\n",
      "llama_print_timings:       total time =    3160.79 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "117 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.13 ms /    63 runs   (    0.10 ms per token, 10270.62 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.98 ms /    22 tokens (    9.04 ms per token,   110.56 tokens per second)\n",
      "llama_print_timings:        eval time =    3287.23 ms /    62 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3570.85 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "118 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       8.96 ms /    55 runs   (    0.16 ms per token,  6138.39 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.23 ms /    17 tokens (   11.25 ms per token,    88.90 tokens per second)\n",
      "llama_print_timings:        eval time =    2883.80 ms /    54 runs   (   53.40 ms per token,    18.73 tokens per second)\n",
      "llama_print_timings:       total time =    3211.39 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "119 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.26 ms /    55 runs   (    0.10 ms per token, 10450.31 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.64 ms /    18 tokens (   10.76 ms per token,    92.95 tokens per second)\n",
      "llama_print_timings:        eval time =    2856.73 ms /    54 runs   (   52.90 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    3124.49 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "120 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.29 ms /    56 runs   (    0.09 ms per token, 10576.02 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.64 ms /    20 tokens (    9.73 ms per token,   102.75 tokens per second)\n",
      "llama_print_timings:        eval time =    2911.18 ms /    55 runs   (   52.93 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3180.87 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "121 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.60 ms /    57 runs   (    0.10 ms per token, 10185.85 tokens per second)\n",
      "llama_print_timings: prompt eval time =     201.10 ms /    23 tokens (    8.74 ms per token,   114.37 tokens per second)\n",
      "llama_print_timings:        eval time =    2969.36 ms /    56 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3248.37 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "122 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.09 ms /    53 runs   (    0.10 ms per token, 10406.44 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.67 ms /    24 tokens (    8.36 ms per token,   119.60 tokens per second)\n",
      "llama_print_timings:        eval time =    2753.82 ms /    52 runs   (   52.96 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3024.88 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      18.53 ms /    55 runs   (    0.34 ms per token,  2968.64 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.35 ms /    19 tokens (   10.23 ms per token,    97.76 tokens per second)\n",
      "llama_print_timings:        eval time =    2935.28 ms /    54 runs   (   54.36 ms per token,    18.40 tokens per second)\n",
      "llama_print_timings:       total time =    3386.99 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "123 -- 1\n",
      "124 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      17.03 ms /    57 runs   (    0.30 ms per token,  3347.62 tokens per second)\n",
      "llama_print_timings: prompt eval time =     207.06 ms /    27 tokens (    7.67 ms per token,   130.40 tokens per second)\n",
      "llama_print_timings:        eval time =    3010.37 ms /    56 runs   (   53.76 ms per token,    18.60 tokens per second)\n",
      "llama_print_timings:       total time =    3407.17 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "125 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.32 ms /    53 runs   (    0.10 ms per token,  9966.15 tokens per second)\n",
      "llama_print_timings: prompt eval time =     202.40 ms /    25 tokens (    8.10 ms per token,   123.52 tokens per second)\n",
      "llama_print_timings:        eval time =    2754.72 ms /    52 runs   (   52.98 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3030.59 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "126 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.11 ms /    54 runs   (    0.09 ms per token, 10567.51 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.34 ms /    18 tokens (   10.74 ms per token,    93.10 tokens per second)\n",
      "llama_print_timings:        eval time =    2802.45 ms /    53 runs   (   52.88 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    3067.57 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "127 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.39 ms /    57 runs   (    0.09 ms per token, 10567.30 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.87 ms /    18 tokens (   10.71 ms per token,    93.33 tokens per second)\n",
      "llama_print_timings:        eval time =    2968.26 ms /    56 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3236.52 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "128 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      12.76 ms /    55 runs   (    0.23 ms per token,  4310.68 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.24 ms /    19 tokens (   10.22 ms per token,    97.82 tokens per second)\n",
      "llama_print_timings:        eval time =    2895.94 ms /    54 runs   (   53.63 ms per token,    18.65 tokens per second)\n",
      "llama_print_timings:       total time =    3257.42 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.60 ms /    60 runs   (    0.11 ms per token,  9096.42 tokens per second)\n",
      "llama_print_timings: prompt eval time =     202.22 ms /    25 tokens (    8.09 ms per token,   123.63 tokens per second)\n",
      "llama_print_timings:        eval time =    3132.34 ms /    59 runs   (   53.09 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3427.17 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "129 -- 1\n",
      "130 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      10.30 ms /    56 runs   (    0.18 ms per token,  5439.01 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.05 ms /    17 tokens (   11.24 ms per token,    88.98 tokens per second)\n",
      "llama_print_timings:        eval time =    2935.84 ms /    55 runs   (   53.38 ms per token,    18.73 tokens per second)\n",
      "llama_print_timings:       total time =    3266.34 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "131 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.55 ms /    59 runs   (    0.09 ms per token, 10622.97 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.03 ms /    17 tokens (   11.24 ms per token,    88.99 tokens per second)\n",
      "llama_print_timings:        eval time =    3069.32 ms /    58 runs   (   52.92 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    3339.16 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "132 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      15.00 ms /    59 runs   (    0.25 ms per token,  3932.28 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.12 ms /    28 tokens (    6.75 ms per token,   148.06 tokens per second)\n",
      "llama_print_timings:        eval time =    3132.59 ms /    58 runs   (   54.01 ms per token,    18.52 tokens per second)\n",
      "llama_print_timings:       total time =    3539.31 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "133 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.15 ms /    55 runs   (    0.09 ms per token, 10683.76 tokens per second)\n",
      "llama_print_timings: prompt eval time =     187.71 ms /    14 tokens (   13.41 ms per token,    74.58 tokens per second)\n",
      "llama_print_timings:        eval time =    2858.92 ms /    54 runs   (   52.94 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3120.41 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "134 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      24.21 ms /    54 runs   (    0.45 ms per token,  2230.11 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.34 ms /    21 tokens (    9.40 ms per token,   106.41 tokens per second)\n",
      "llama_print_timings:        eval time =    2914.88 ms /    53 runs   (   55.00 ms per token,    18.18 tokens per second)\n",
      "llama_print_timings:       total time =    3398.15 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "135 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       8.32 ms /    57 runs   (    0.15 ms per token,  6853.43 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.96 ms /    22 tokens (    9.09 ms per token,   110.02 tokens per second)\n",
      "llama_print_timings:        eval time =    2981.76 ms /    56 runs   (   53.25 ms per token,    18.78 tokens per second)\n",
      "llama_print_timings:       total time =    3284.09 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "136 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      24.17 ms /    53 runs   (    0.46 ms per token,  2192.80 tokens per second)\n",
      "llama_print_timings: prompt eval time =     204.53 ms /    26 tokens (    7.87 ms per token,   127.12 tokens per second)\n",
      "llama_print_timings:        eval time =    2880.70 ms /    52 runs   (   55.40 ms per token,    18.05 tokens per second)\n",
      "llama_print_timings:       total time =    3380.75 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "137 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.98 ms /    53 runs   (    0.09 ms per token, 10640.43 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.04 ms /    17 tokens (   11.24 ms per token,    88.98 tokens per second)\n",
      "llama_print_timings:        eval time =    2752.32 ms /    52 runs   (   52.93 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3015.02 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "138 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.28 ms /    55 runs   (    0.11 ms per token,  8764.94 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.56 ms /    19 tokens (   10.24 ms per token,    97.66 tokens per second)\n",
      "llama_print_timings:        eval time =    2864.21 ms /    54 runs   (   53.04 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3146.35 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "139 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.91 ms /    53 runs   (    0.09 ms per token, 10800.90 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.28 ms /    18 tokens (   10.74 ms per token,    93.13 tokens per second)\n",
      "llama_print_timings:        eval time =    2753.34 ms /    52 runs   (   52.95 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3016.86 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "140 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      12.44 ms /    50 runs   (    0.25 ms per token,  4019.62 tokens per second)\n",
      "llama_print_timings: prompt eval time =     184.66 ms /    12 tokens (   15.39 ms per token,    64.98 tokens per second)\n",
      "llama_print_timings:        eval time =    2631.87 ms /    49 runs   (   53.71 ms per token,    18.62 tokens per second)\n",
      "llama_print_timings:       total time =    2985.64 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "141 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.13 ms /    55 runs   (    0.09 ms per token, 10721.25 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.37 ms /    15 tokens (   12.62 ms per token,    79.21 tokens per second)\n",
      "llama_print_timings:        eval time =    2855.58 ms /    54 runs   (   52.88 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    3118.94 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "142 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.25 ms /    54 runs   (    0.10 ms per token, 10281.80 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.47 ms /    18 tokens (   10.75 ms per token,    93.04 tokens per second)\n",
      "llama_print_timings:        eval time =    2806.03 ms /    53 runs   (   52.94 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3072.73 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "143 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      20.63 ms /    58 runs   (    0.36 ms per token,  2810.89 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.57 ms /    19 tokens (   10.24 ms per token,    97.65 tokens per second)\n",
      "llama_print_timings:        eval time =    3093.84 ms /    57 runs   (   54.28 ms per token,    18.42 tokens per second)\n",
      "llama_print_timings:       total time =    3564.89 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "144 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.46 ms /    56 runs   (    0.10 ms per token, 10247.03 tokens per second)\n",
      "llama_print_timings: prompt eval time =     354.77 ms /    39 tokens (    9.10 ms per token,   109.93 tokens per second)\n",
      "llama_print_timings:        eval time =    2918.43 ms /    55 runs   (   53.06 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3350.85 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "145 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.81 ms /    57 runs   (    0.10 ms per token,  9814.05 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.08 ms /    19 tokens (   10.21 ms per token,    97.90 tokens per second)\n",
      "llama_print_timings:        eval time =    2967.82 ms /    56 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3244.00 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "146 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.37 ms /    55 runs   (    0.10 ms per token, 10251.63 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.33 ms /    14 tokens (   13.45 ms per token,    74.34 tokens per second)\n",
      "llama_print_timings:        eval time =    2856.36 ms /    54 runs   (   52.90 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    3121.84 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "147 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.59 ms /    58 runs   (    0.10 ms per token, 10366.40 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.16 ms /    16 tokens (   11.82 ms per token,    84.59 tokens per second)\n",
      "llama_print_timings:        eval time =    3015.12 ms /    57 runs   (   52.90 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    3283.60 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "148 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.17 ms /    53 runs   (    0.10 ms per token, 10245.51 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.45 ms /    22 tokens (    9.07 ms per token,   110.30 tokens per second)\n",
      "llama_print_timings:        eval time =    2756.35 ms /    52 runs   (   53.01 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3028.67 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "149 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.76 ms /    59 runs   (    0.10 ms per token, 10246.61 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.78 ms /    24 tokens (    8.37 ms per token,   119.53 tokens per second)\n",
      "llama_print_timings:        eval time =    3072.21 ms /    58 runs   (   52.97 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3351.62 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "150 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.01 ms /    51 runs   (    0.10 ms per token, 10175.58 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.19 ms /    22 tokens (    9.05 ms per token,   110.45 tokens per second)\n",
      "llama_print_timings:        eval time =    2651.30 ms /    50 runs   (   53.03 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    2920.58 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "151 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.48 ms /    55 runs   (    0.10 ms per token, 10040.16 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.94 ms /    23 tokens (    8.74 ms per token,   114.46 tokens per second)\n",
      "llama_print_timings:        eval time =    2859.35 ms /    54 runs   (   52.95 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3137.01 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "152 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      10.37 ms /    57 runs   (    0.18 ms per token,  5495.04 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.36 ms /    16 tokens (   11.83 ms per token,    84.50 tokens per second)\n",
      "llama_print_timings:        eval time =    3031.06 ms /    56 runs   (   54.13 ms per token,    18.48 tokens per second)\n",
      "llama_print_timings:       total time =    3371.16 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "153 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =      18.27 ms /    55 runs   (    0.33 ms per token,  3011.06 tokens per second)\n",
      "llama_print_timings: prompt eval time =     196.68 ms /    21 tokens (    9.37 ms per token,   106.77 tokens per second)\n",
      "llama_print_timings:        eval time =    2951.46 ms /    54 runs   (   54.66 ms per token,    18.30 tokens per second)\n",
      "llama_print_timings:       total time =    3409.33 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "154 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.13 ms /    56 runs   (    0.13 ms per token,  7854.14 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.12 ms /    19 tokens (   10.27 ms per token,    97.38 tokens per second)\n",
      "llama_print_timings:        eval time =    2924.80 ms /    55 runs   (   53.18 ms per token,    18.80 tokens per second)\n",
      "llama_print_timings:       total time =    3224.33 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "155 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.73 ms /    55 runs   (    0.10 ms per token,  9598.60 tokens per second)\n",
      "llama_print_timings: prompt eval time =     201.16 ms /    24 tokens (    8.38 ms per token,   119.31 tokens per second)\n",
      "llama_print_timings:        eval time =    2865.11 ms /    54 runs   (   53.06 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3145.77 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "156 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.36 ms /    64 runs   (    0.10 ms per token, 10056.57 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.48 ms /    18 tokens (   10.75 ms per token,    93.03 tokens per second)\n",
      "llama_print_timings:        eval time =    3350.01 ms /    63 runs   (   53.17 ms per token,    18.81 tokens per second)\n",
      "llama_print_timings:       total time =    3644.16 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "157 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.06 ms /    57 runs   (    0.12 ms per token,  8070.23 tokens per second)\n",
      "llama_print_timings: prompt eval time =     201.58 ms /    23 tokens (    8.76 ms per token,   114.10 tokens per second)\n",
      "llama_print_timings:        eval time =    2970.83 ms /    56 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3272.12 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "158 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.02 ms /    58 runs   (    0.10 ms per token,  9631.35 tokens per second)\n",
      "llama_print_timings: prompt eval time =     206.06 ms /    23 tokens (    8.96 ms per token,   111.62 tokens per second)\n",
      "llama_print_timings:        eval time =    3068.74 ms /    57 runs   (   53.84 ms per token,    18.57 tokens per second)\n",
      "llama_print_timings:       total time =    3361.14 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "159 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.49 ms /    59 runs   (    0.09 ms per token, 10744.86 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.01 ms /    18 tokens (   10.78 ms per token,    92.78 tokens per second)\n",
      "llama_print_timings:        eval time =    3074.54 ms /    58 runs   (   53.01 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3347.95 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "160 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.28 ms /    53 runs   (    0.10 ms per token, 10047.39 tokens per second)\n",
      "llama_print_timings: prompt eval time =     204.82 ms /    26 tokens (    7.88 ms per token,   126.94 tokens per second)\n",
      "llama_print_timings:        eval time =    2754.73 ms /    52 runs   (   52.98 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3033.30 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "161 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.65 ms /    56 runs   (    0.10 ms per token,  9908.00 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.75 ms /    24 tokens (    8.36 ms per token,   119.55 tokens per second)\n",
      "llama_print_timings:        eval time =    2908.36 ms /    55 runs   (   52.88 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    3188.16 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "162 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.71 ms /    56 runs   (    0.10 ms per token,  9810.79 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.98 ms /    18 tokens (   10.72 ms per token,    93.27 tokens per second)\n",
      "llama_print_timings:        eval time =    2925.14 ms /    55 runs   (   53.18 ms per token,    18.80 tokens per second)\n",
      "llama_print_timings:       total time =    3198.34 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "163 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.60 ms /    59 runs   (    0.09 ms per token, 10531.95 tokens per second)\n",
      "llama_print_timings: prompt eval time =     202.36 ms /    25 tokens (    8.09 ms per token,   123.54 tokens per second)\n",
      "llama_print_timings:        eval time =    3068.75 ms /    58 runs   (   52.91 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    3349.65 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.71 ms /    58 runs   (    0.10 ms per token, 10162.96 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.00 ms /    21 tokens (    9.43 ms per token,   106.06 tokens per second)\n",
      "llama_print_timings:        eval time =    3017.19 ms /    57 runs   (   52.93 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3294.03 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "165 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.73 ms /    52 runs   (    0.09 ms per token, 10998.31 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.89 ms /    20 tokens (    9.74 ms per token,   102.62 tokens per second)\n",
      "llama_print_timings:        eval time =    2698.53 ms /    51 runs   (   52.91 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    2961.83 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "166 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.27 ms /    58 runs   (    0.11 ms per token,  9248.92 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.82 ms /    18 tokens (   10.71 ms per token,    93.35 tokens per second)\n",
      "llama_print_timings:        eval time =    3019.66 ms /    57 runs   (   52.98 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3300.79 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "167 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.15 ms /    54 runs   (    0.10 ms per token, 10479.33 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.43 ms /    27 tokens (    7.61 ms per token,   131.43 tokens per second)\n",
      "llama_print_timings:        eval time =    2807.64 ms /    53 runs   (   52.97 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3084.31 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "168 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.31 ms /    54 runs   (    0.10 ms per token, 10175.24 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.74 ms /    20 tokens (    9.74 ms per token,   102.70 tokens per second)\n",
      "llama_print_timings:        eval time =    2808.80 ms /    53 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3076.34 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "169 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.51 ms /    58 runs   (    0.09 ms per token, 10532.05 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.76 ms /    20 tokens (    9.74 ms per token,   102.69 tokens per second)\n",
      "llama_print_timings:        eval time =    3017.81 ms /    57 runs   (   52.94 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3288.97 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "170 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.17 ms /    53 runs   (    0.10 ms per token, 10253.43 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.70 ms /    23 tokens (    8.73 ms per token,   114.60 tokens per second)\n",
      "llama_print_timings:        eval time =    2754.05 ms /    52 runs   (   52.96 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3026.37 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "171 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.98 ms /    60 runs   (    0.10 ms per token, 10038.48 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.03 ms /    21 tokens (    9.38 ms per token,   106.58 tokens per second)\n",
      "llama_print_timings:        eval time =    3125.07 ms /    59 runs   (   52.97 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3406.09 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "172 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.66 ms /    54 runs   (    0.10 ms per token,  9544.01 tokens per second)\n",
      "llama_print_timings: prompt eval time =     201.08 ms /    23 tokens (    8.74 ms per token,   114.38 tokens per second)\n",
      "llama_print_timings:        eval time =    2814.00 ms /    53 runs   (   53.09 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    3089.95 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "173 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.27 ms /    56 runs   (    0.09 ms per token, 10622.15 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.81 ms /    19 tokens (   10.25 ms per token,    97.53 tokens per second)\n",
      "llama_print_timings:        eval time =    2915.86 ms /    55 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3182.82 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "174 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.46 ms /    59 runs   (    0.09 ms per token, 10797.95 tokens per second)\n",
      "llama_print_timings: prompt eval time =     351.27 ms /    38 tokens (    9.24 ms per token,   108.18 tokens per second)\n",
      "llama_print_timings:        eval time =    3083.19 ms /    58 runs   (   53.16 ms per token,    18.81 tokens per second)\n",
      "llama_print_timings:       total time =    3510.25 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "175 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.30 ms /    56 runs   (    0.09 ms per token, 10570.03 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.15 ms /    21 tokens (    9.39 ms per token,   106.52 tokens per second)\n",
      "llama_print_timings:        eval time =    2919.11 ms /    55 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3189.83 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "176 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.69 ms /    52 runs   (    0.09 ms per token, 11082.69 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.67 ms /    19 tokens (   10.25 ms per token,    97.60 tokens per second)\n",
      "llama_print_timings:        eval time =    2703.43 ms /    51 runs   (   53.01 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    2964.41 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "177 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.14 ms /    54 runs   (    0.10 ms per token, 10509.93 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.19 ms /    21 tokens (    9.44 ms per token,   105.96 tokens per second)\n",
      "llama_print_timings:        eval time =    2811.76 ms /    53 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3080.17 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "178 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.67 ms /    50 runs   (    0.09 ms per token, 10718.11 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.35 ms /    21 tokens (    9.40 ms per token,   106.41 tokens per second)\n",
      "llama_print_timings:        eval time =    2597.69 ms /    49 runs   (   53.01 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    2858.77 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "179 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.57 ms /    57 runs   (    0.10 ms per token, 10233.39 tokens per second)\n",
      "llama_print_timings: prompt eval time =     202.77 ms /    25 tokens (    8.11 ms per token,   123.29 tokens per second)\n",
      "llama_print_timings:        eval time =    2971.16 ms /    56 runs   (   53.06 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3249.89 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "180 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.18 ms /    57 runs   (    0.09 ms per token, 11005.99 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.63 ms /    20 tokens (    9.78 ms per token,   102.23 tokens per second)\n",
      "llama_print_timings:        eval time =    2972.99 ms /    56 runs   (   53.09 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3241.12 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "181 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.05 ms /    55 runs   (    0.09 ms per token, 10884.62 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.36 ms /    22 tokens (    9.06 ms per token,   110.35 tokens per second)\n",
      "llama_print_timings:        eval time =    2865.34 ms /    54 runs   (   53.06 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3134.60 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "182 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.70 ms /    63 runs   (    0.09 ms per token, 11044.88 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.50 ms /    26 tokens (    7.90 ms per token,   126.52 tokens per second)\n",
      "llama_print_timings:        eval time =    3290.38 ms /    62 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3576.76 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "183 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.21 ms /    55 runs   (    0.09 ms per token, 10550.55 tokens per second)\n",
      "llama_print_timings: prompt eval time =     190.07 ms /    15 tokens (   12.67 ms per token,    78.92 tokens per second)\n",
      "llama_print_timings:        eval time =    2859.98 ms /    54 runs   (   52.96 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3122.34 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "184 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.40 ms /    61 runs   (    0.09 ms per token, 11302.58 tokens per second)\n",
      "llama_print_timings: prompt eval time =     360.21 ms /    44 tokens (    8.19 ms per token,   122.15 tokens per second)\n",
      "llama_print_timings:        eval time =    3189.26 ms /    60 runs   (   53.15 ms per token,    18.81 tokens per second)\n",
      "llama_print_timings:       total time =    3628.17 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "185 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.00 ms /    53 runs   (    0.09 ms per token, 10591.53 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.30 ms /    22 tokens (    9.01 ms per token,   110.94 tokens per second)\n",
      "llama_print_timings:        eval time =    2757.87 ms /    52 runs   (   53.04 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3025.10 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "186 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.13 ms /    56 runs   (    0.09 ms per token, 10918.31 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.64 ms /    20 tokens (    9.78 ms per token,   102.23 tokens per second)\n",
      "llama_print_timings:        eval time =    2917.89 ms /    55 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3185.31 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "187 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.05 ms /    53 runs   (    0.10 ms per token, 10495.05 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.28 ms /    17 tokens (   11.25 ms per token,    88.88 tokens per second)\n",
      "llama_print_timings:        eval time =    2755.36 ms /    52 runs   (   52.99 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3016.00 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "188 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.92 ms /    55 runs   (    0.09 ms per token, 11185.68 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.17 ms /    17 tokens (   11.25 ms per token,    88.93 tokens per second)\n",
      "llama_print_timings:        eval time =    2861.90 ms /    54 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3123.47 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "189 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.77 ms /    61 runs   (    0.09 ms per token, 10566.43 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.71 ms /    27 tokens (    7.62 ms per token,   131.26 tokens per second)\n",
      "llama_print_timings:        eval time =    3182.18 ms /    60 runs   (   53.04 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3466.36 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "190 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.97 ms /    52 runs   (    0.10 ms per token, 10456.46 tokens per second)\n",
      "llama_print_timings: prompt eval time =     209.57 ms /    29 tokens (    7.23 ms per token,   138.38 tokens per second)\n",
      "llama_print_timings:        eval time =    2702.91 ms /    51 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    2979.14 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "191 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.13 ms /    55 runs   (    0.09 ms per token, 10721.25 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.60 ms /    22 tokens (    9.07 ms per token,   110.22 tokens per second)\n",
      "llama_print_timings:        eval time =    2866.73 ms /    54 runs   (   53.09 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3139.10 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "192 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.18 ms /    55 runs   (    0.09 ms per token, 10621.86 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.58 ms /    19 tokens (   10.24 ms per token,    97.65 tokens per second)\n",
      "llama_print_timings:        eval time =    2864.30 ms /    54 runs   (   53.04 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3129.72 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "193 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.68 ms /    52 runs   (    0.09 ms per token, 11113.49 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.54 ms /    18 tokens (   10.75 ms per token,    93.00 tokens per second)\n",
      "llama_print_timings:        eval time =    2705.13 ms /    51 runs   (   53.04 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    2964.99 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "194 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.26 ms /    62 runs   (    0.10 ms per token,  9897.83 tokens per second)\n",
      "llama_print_timings: prompt eval time =     213.20 ms /    31 tokens (    6.88 ms per token,   145.40 tokens per second)\n",
      "llama_print_timings:        eval time =    3242.13 ms /    61 runs   (   53.15 ms per token,    18.81 tokens per second)\n",
      "llama_print_timings:       total time =    3539.40 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "195 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.44 ms /    59 runs   (    0.09 ms per token, 10853.57 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.52 ms /    19 tokens (   10.24 ms per token,    97.68 tokens per second)\n",
      "llama_print_timings:        eval time =    3076.32 ms /    58 runs   (   53.04 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3346.34 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "196 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.10 ms /    56 runs   (    0.09 ms per token, 10989.01 tokens per second)\n",
      "llama_print_timings: prompt eval time =     201.00 ms /    23 tokens (    8.74 ms per token,   114.43 tokens per second)\n",
      "llama_print_timings:        eval time =    2917.43 ms /    55 runs   (   53.04 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3190.38 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "197 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.26 ms /    56 runs   (    0.09 ms per token, 10646.39 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.45 ms /    16 tokens (   11.84 ms per token,    84.45 tokens per second)\n",
      "llama_print_timings:        eval time =    2914.20 ms /    55 runs   (   52.99 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3177.45 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.12 ms /    54 runs   (    0.09 ms per token, 10546.88 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.03 ms /    26 tokens (    7.89 ms per token,   126.81 tokens per second)\n",
      "llama_print_timings:        eval time =    2810.74 ms /    53 runs   (   53.03 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3084.20 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "198 -- 1\n",
      "199 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.30 ms /    58 runs   (    0.09 ms per token, 10935.14 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.93 ms /    20 tokens (    9.80 ms per token,   102.08 tokens per second)\n",
      "llama_print_timings:        eval time =    3025.87 ms /    57 runs   (   53.09 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3296.33 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.08 ms /    55 runs   (    0.09 ms per token, 10833.17 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.12 ms /    21 tokens (    9.43 ms per token,   106.00 tokens per second)\n",
      "llama_print_timings:        eval time =    2864.08 ms /    54 runs   (   53.04 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3133.06 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "201 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.23 ms /    56 runs   (    0.09 ms per token, 10715.65 tokens per second)\n",
      "llama_print_timings: prompt eval time =     186.80 ms /    13 tokens (   14.37 ms per token,    69.59 tokens per second)\n",
      "llama_print_timings:        eval time =    2908.77 ms /    55 runs   (   52.89 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    3168.85 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "202 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.14 ms /    55 runs   (    0.09 ms per token, 10694.15 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.00 ms /    20 tokens (    9.75 ms per token,   102.57 tokens per second)\n",
      "llama_print_timings:        eval time =    2857.91 ms /    54 runs   (   52.92 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3125.77 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "203 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.81 ms /    53 runs   (    0.09 ms per token, 11011.84 tokens per second)\n",
      "llama_print_timings: prompt eval time =     186.36 ms /    13 tokens (   14.34 ms per token,    69.76 tokens per second)\n",
      "llama_print_timings:        eval time =    2753.96 ms /    52 runs   (   52.96 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3007.99 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "204 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.18 ms /    56 runs   (    0.09 ms per token, 10812.90 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.66 ms /    21 tokens (    9.41 ms per token,   106.24 tokens per second)\n",
      "llama_print_timings:        eval time =    2919.41 ms /    55 runs   (   53.08 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3189.74 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "205 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.93 ms /    53 runs   (    0.09 ms per token, 10750.51 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.90 ms /    21 tokens (    9.42 ms per token,   106.11 tokens per second)\n",
      "llama_print_timings:        eval time =    2756.25 ms /    52 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3022.78 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "206 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.85 ms /    63 runs   (    0.09 ms per token, 10761.87 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.20 ms /    28 tokens (    6.76 ms per token,   147.99 tokens per second)\n",
      "llama_print_timings:        eval time =    3290.67 ms /    62 runs   (   53.08 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3560.96 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "207 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.22 ms /    64 runs   (    0.10 ms per token, 10282.78 tokens per second)\n",
      "llama_print_timings: prompt eval time =     206.10 ms /    27 tokens (    7.63 ms per token,   131.00 tokens per second)\n",
      "llama_print_timings:        eval time =    3343.53 ms /    63 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3635.05 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "208 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.53 ms /    61 runs   (    0.09 ms per token, 11026.75 tokens per second)\n",
      "llama_print_timings: prompt eval time =     204.69 ms /    26 tokens (    7.87 ms per token,   127.02 tokens per second)\n",
      "llama_print_timings:        eval time =    3183.20 ms /    60 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3466.19 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "209 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.20 ms /    57 runs   (    0.09 ms per token, 10967.87 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.37 ms /    17 tokens (   11.26 ms per token,    88.84 tokens per second)\n",
      "llama_print_timings:        eval time =    2967.05 ms /    56 runs   (   52.98 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3231.72 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "210 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.53 ms /    58 runs   (    0.10 ms per token, 10482.56 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.19 ms /    21 tokens (    9.44 ms per token,   105.96 tokens per second)\n",
      "llama_print_timings:        eval time =    3025.08 ms /    57 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3300.25 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "211 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.41 ms /    55 runs   (    0.10 ms per token, 10172.00 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.91 ms /    21 tokens (    9.42 ms per token,   106.11 tokens per second)\n",
      "llama_print_timings:        eval time =    2865.91 ms /    54 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3137.21 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "212 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.23 ms /    57 runs   (    0.09 ms per token, 10900.75 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.12 ms /    18 tokens (   10.73 ms per token,    93.21 tokens per second)\n",
      "llama_print_timings:        eval time =    2970.00 ms /    56 runs   (   53.04 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3236.33 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "213 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.59 ms /    57 runs   (    0.10 ms per token, 10193.13 tokens per second)\n",
      "llama_print_timings: prompt eval time =     342.03 ms /    33 tokens (   10.36 ms per token,    96.48 tokens per second)\n",
      "llama_print_timings:        eval time =    2973.19 ms /    56 runs   (   53.09 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    3390.99 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "214 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.39 ms /    56 runs   (    0.10 ms per token, 10391.54 tokens per second)\n",
      "llama_print_timings: prompt eval time =     201.04 ms /    24 tokens (    8.38 ms per token,   119.38 tokens per second)\n",
      "llama_print_timings:        eval time =    2916.30 ms /    55 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3189.45 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "215 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.29 ms /    57 runs   (    0.09 ms per token, 10764.87 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.91 ms /    23 tokens (    8.74 ms per token,   114.48 tokens per second)\n",
      "llama_print_timings:        eval time =    2969.63 ms /    56 runs   (   53.03 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3243.99 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.96 ms /    56 runs   (    0.09 ms per token, 11294.88 tokens per second)\n",
      "llama_print_timings: prompt eval time =     371.81 ms /    52 tokens (    7.15 ms per token,   139.85 tokens per second)\n",
      "llama_print_timings:        eval time =    2932.25 ms /    55 runs   (   53.31 ms per token,    18.76 tokens per second)\n",
      "llama_print_timings:       total time =    3378.64 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "217 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.99 ms /    62 runs   (    0.10 ms per token, 10348.86 tokens per second)\n",
      "llama_print_timings: prompt eval time =     341.99 ms /    33 tokens (   10.36 ms per token,    96.49 tokens per second)\n",
      "llama_print_timings:        eval time =    3242.95 ms /    61 runs   (   53.16 ms per token,    18.81 tokens per second)\n",
      "llama_print_timings:       total time =    3664.86 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "218 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.00 ms /    53 runs   (    0.09 ms per token, 10604.24 tokens per second)\n",
      "llama_print_timings: prompt eval time =     209.40 ms /    29 tokens (    7.22 ms per token,   138.49 tokens per second)\n",
      "llama_print_timings:        eval time =    2758.48 ms /    52 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3036.12 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "219 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.85 ms /    53 runs   (    0.09 ms per token, 10918.83 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.58 ms /    16 tokens (   11.85 ms per token,    84.40 tokens per second)\n",
      "llama_print_timings:        eval time =    2754.98 ms /    52 runs   (   52.98 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3013.87 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "220 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.87 ms /    52 runs   (    0.09 ms per token, 10675.43 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.76 ms /    21 tokens (    9.42 ms per token,   106.19 tokens per second)\n",
      "llama_print_timings:        eval time =    2702.51 ms /    51 runs   (   52.99 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    2967.91 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "221 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.49 ms /    49 runs   (    0.09 ms per token, 10922.87 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.84 ms /    15 tokens (   12.66 ms per token,    79.01 tokens per second)\n",
      "llama_print_timings:        eval time =    2544.71 ms /    48 runs   (   53.01 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    2797.87 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "222 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.42 ms /    55 runs   (    0.10 ms per token, 10147.60 tokens per second)\n",
      "llama_print_timings: prompt eval time =     347.55 ms /    36 tokens (    9.65 ms per token,   103.58 tokens per second)\n",
      "llama_print_timings:        eval time =    2868.82 ms /    54 runs   (   53.13 ms per token,    18.82 tokens per second)\n",
      "llama_print_timings:       total time =    3289.36 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "223 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.92 ms /    54 runs   (    0.09 ms per token, 10980.07 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.87 ms /    19 tokens (   10.26 ms per token,    97.50 tokens per second)\n",
      "llama_print_timings:        eval time =    2809.31 ms /    53 runs   (   53.01 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3072.93 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "224 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.85 ms /    53 runs   (    0.09 ms per token, 10936.86 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.78 ms /    15 tokens (   12.65 ms per token,    79.04 tokens per second)\n",
      "llama_print_timings:        eval time =    2754.11 ms /    52 runs   (   52.96 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3011.59 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "225 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.16 ms /    54 runs   (    0.10 ms per token, 10473.24 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.94 ms /    20 tokens (    9.75 ms per token,   102.60 tokens per second)\n",
      "llama_print_timings:        eval time =    2810.15 ms /    53 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3076.93 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "226 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.04 ms /    55 runs   (    0.09 ms per token, 10901.88 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.42 ms /    17 tokens (   11.26 ms per token,    88.81 tokens per second)\n",
      "llama_print_timings:        eval time =    2862.72 ms /    54 runs   (   53.01 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3125.32 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "227 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.32 ms /    67 runs   (    0.09 ms per token, 10609.66 tokens per second)\n",
      "llama_print_timings: prompt eval time =     369.56 ms /    50 tokens (    7.39 ms per token,   135.29 tokens per second)\n",
      "llama_print_timings:        eval time =    3513.89 ms /    66 runs   (   53.24 ms per token,    18.78 tokens per second)\n",
      "llama_print_timings:       total time =    3974.19 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "228 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.33 ms /    58 runs   (    0.09 ms per token, 10887.93 tokens per second)\n",
      "llama_print_timings: prompt eval time =     191.77 ms /    17 tokens (   11.28 ms per token,    88.65 tokens per second)\n",
      "llama_print_timings:        eval time =    3021.71 ms /    57 runs   (   53.01 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3289.80 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "229 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.24 ms /    55 runs   (    0.10 ms per token, 10496.18 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.78 ms /    24 tokens (    8.37 ms per token,   119.53 tokens per second)\n",
      "llama_print_timings:        eval time =    2864.91 ms /    54 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3136.01 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "230 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.21 ms /    55 runs   (    0.09 ms per token, 10564.73 tokens per second)\n",
      "llama_print_timings: prompt eval time =     345.40 ms /    34 tokens (   10.16 ms per token,    98.44 tokens per second)\n",
      "llama_print_timings:        eval time =    2867.03 ms /    54 runs   (   53.09 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    3283.57 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "231 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.46 ms /    59 runs   (    0.09 ms per token, 10813.78 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.47 ms /    23 tokens (    8.72 ms per token,   114.73 tokens per second)\n",
      "llama_print_timings:        eval time =    3080.00 ms /    58 runs   (   53.10 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    3358.46 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "232 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       9.76 ms /   109 runs   (    0.09 ms per token, 11164.60 tokens per second)\n",
      "llama_print_timings: prompt eval time =     517.44 ms /    67 tokens (    7.72 ms per token,   129.48 tokens per second)\n",
      "llama_print_timings:        eval time =    5766.69 ms /   108 runs   (   53.40 ms per token,    18.73 tokens per second)\n",
      "llama_print_timings:       total time =    6429.11 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "233 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.07 ms /    53 runs   (    0.10 ms per token, 10453.65 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.08 ms /    18 tokens (   10.73 ms per token,    93.22 tokens per second)\n",
      "llama_print_timings:        eval time =    2752.57 ms /    52 runs   (   52.93 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3014.62 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "234 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.31 ms /    56 runs   (    0.09 ms per token, 10550.11 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.26 ms /    19 tokens (   10.22 ms per token,    97.81 tokens per second)\n",
      "llama_print_timings:        eval time =    2920.46 ms /    55 runs   (   53.10 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    3189.35 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "235 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.36 ms /    57 runs   (    0.09 ms per token, 10626.40 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.30 ms /    20 tokens (    9.77 ms per token,   102.41 tokens per second)\n",
      "llama_print_timings:        eval time =    2972.09 ms /    56 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3240.97 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "236 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.04 ms /    56 runs   (    0.09 ms per token, 11119.94 tokens per second)\n",
      "llama_print_timings: prompt eval time =     187.85 ms /    14 tokens (   13.42 ms per token,    74.53 tokens per second)\n",
      "llama_print_timings:        eval time =    2910.73 ms /    55 runs   (   52.92 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    3170.98 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "237 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.85 ms /    52 runs   (    0.09 ms per token, 10723.86 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.18 ms /    19 tokens (   10.27 ms per token,    97.34 tokens per second)\n",
      "llama_print_timings:        eval time =    2700.50 ms /    51 runs   (   52.95 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    2965.03 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "238 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.97 ms /    52 runs   (    0.10 ms per token, 10471.20 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.08 ms /    19 tokens (   10.21 ms per token,    97.90 tokens per second)\n",
      "llama_print_timings:        eval time =    2700.26 ms /    51 runs   (   52.95 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    2964.07 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "239 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.66 ms /    50 runs   (    0.09 ms per token, 10738.83 tokens per second)\n",
      "llama_print_timings: prompt eval time =     213.04 ms /    31 tokens (    6.87 ms per token,   145.52 tokens per second)\n",
      "llama_print_timings:        eval time =    2601.15 ms /    49 runs   (   53.08 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    2879.43 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "240 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.60 ms /    58 runs   (    0.10 ms per token, 10357.14 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.69 ms /    27 tokens (    7.62 ms per token,   131.26 tokens per second)\n",
      "llama_print_timings:        eval time =    3021.11 ms /    57 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3302.07 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "241 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.95 ms /    51 runs   (    0.10 ms per token, 10298.87 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.74 ms /    18 tokens (   10.76 ms per token,    92.91 tokens per second)\n",
      "llama_print_timings:        eval time =    2650.61 ms /    50 runs   (   53.01 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    2913.99 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "242 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.83 ms /    50 runs   (    0.10 ms per token, 10362.69 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.29 ms /    19 tokens (   10.23 ms per token,    97.79 tokens per second)\n",
      "llama_print_timings:        eval time =    2597.05 ms /    49 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    2858.87 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "243 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.29 ms /    57 runs   (    0.09 ms per token, 10777.08 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.73 ms /    18 tokens (   10.76 ms per token,    92.91 tokens per second)\n",
      "llama_print_timings:        eval time =    2968.14 ms /    56 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3235.20 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "244 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.11 ms /    54 runs   (    0.09 ms per token, 10565.45 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.84 ms /    19 tokens (   10.20 ms per token,    98.02 tokens per second)\n",
      "llama_print_timings:        eval time =    2807.76 ms /    53 runs   (   52.98 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3073.89 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "245 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.66 ms /    51 runs   (    0.09 ms per token, 10953.61 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.37 ms /    16 tokens (   11.84 ms per token,    84.49 tokens per second)\n",
      "llama_print_timings:        eval time =    2648.00 ms /    50 runs   (   52.96 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    2902.90 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "246 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.88 ms /    51 runs   (    0.10 ms per token, 10448.68 tokens per second)\n",
      "llama_print_timings: prompt eval time =     201.00 ms /    24 tokens (    8.38 ms per token,   119.40 tokens per second)\n",
      "llama_print_timings:        eval time =    2648.64 ms /    50 runs   (   52.97 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    2915.67 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "247 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.16 ms /    57 runs   (    0.09 ms per token, 11044.37 tokens per second)\n",
      "llama_print_timings: prompt eval time =     353.00 ms /    41 tokens (    8.61 ms per token,   116.15 tokens per second)\n",
      "llama_print_timings:        eval time =    2974.18 ms /    56 runs   (   53.11 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    3402.10 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "248 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.67 ms /    61 runs   (    0.09 ms per token, 10765.97 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.73 ms /    23 tokens (    8.73 ms per token,   114.58 tokens per second)\n",
      "llama_print_timings:        eval time =    3183.15 ms /    60 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3462.03 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "249 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.30 ms /    57 runs   (    0.09 ms per token, 10760.81 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.97 ms /    19 tokens (   10.26 ms per token,    97.45 tokens per second)\n",
      "llama_print_timings:        eval time =    2969.60 ms /    56 runs   (   53.03 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3237.09 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "250 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.22 ms /    54 runs   (    0.10 ms per token, 10350.78 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.83 ms /    19 tokens (   10.25 ms per token,    97.52 tokens per second)\n",
      "llama_print_timings:        eval time =    2810.03 ms /    53 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3075.52 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "251 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.26 ms /    58 runs   (    0.09 ms per token, 11026.62 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.96 ms /    22 tokens (    9.04 ms per token,   110.58 tokens per second)\n",
      "llama_print_timings:        eval time =    3021.18 ms /    57 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3295.54 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "252 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.00 ms /    53 runs   (    0.09 ms per token, 10610.61 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.03 ms /    22 tokens (    9.05 ms per token,   110.53 tokens per second)\n",
      "llama_print_timings:        eval time =    2757.50 ms /    52 runs   (   53.03 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3024.97 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "253 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.25 ms /    53 runs   (    0.10 ms per token, 10087.55 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.57 ms /    22 tokens (    9.07 ms per token,   110.24 tokens per second)\n",
      "llama_print_timings:        eval time =    2757.09 ms /    52 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3028.10 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "254 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.92 ms /    53 runs   (    0.09 ms per token, 10781.12 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.50 ms /    18 tokens (   10.75 ms per token,    93.03 tokens per second)\n",
      "llama_print_timings:        eval time =    2757.07 ms /    52 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3019.01 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "255 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.89 ms /    54 runs   (    0.09 ms per token, 11042.94 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.94 ms /    15 tokens (   12.66 ms per token,    78.97 tokens per second)\n",
      "llama_print_timings:        eval time =    2806.97 ms /    53 runs   (   52.96 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3067.51 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "256 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.20 ms /    55 runs   (    0.09 ms per token, 10574.89 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.58 ms /    22 tokens (    9.07 ms per token,   110.23 tokens per second)\n",
      "llama_print_timings:        eval time =    2867.16 ms /    54 runs   (   53.10 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    3138.93 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "257 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.75 ms /    51 runs   (    0.09 ms per token, 10732.32 tokens per second)\n",
      "llama_print_timings: prompt eval time =     205.10 ms /    27 tokens (    7.60 ms per token,   131.64 tokens per second)\n",
      "llama_print_timings:        eval time =    2654.19 ms /    50 runs   (   53.08 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    2925.30 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "258 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.11 ms /    55 runs   (    0.09 ms per token, 10763.21 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.56 ms /    22 tokens (    9.07 ms per token,   110.24 tokens per second)\n",
      "llama_print_timings:        eval time =    2866.85 ms /    54 runs   (   53.09 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3137.94 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.17 ms /    55 runs   (    0.09 ms per token, 10648.60 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.79 ms /    18 tokens (   10.77 ms per token,    92.89 tokens per second)\n",
      "llama_print_timings:        eval time =    2865.35 ms /    54 runs   (   53.06 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3131.16 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "259 -- 1\n",
      "260 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.99 ms /    52 runs   (    0.10 ms per token, 10425.02 tokens per second)\n",
      "llama_print_timings: prompt eval time =     206.35 ms /    32 tokens (    6.45 ms per token,   155.08 tokens per second)\n",
      "llama_print_timings:        eval time =    2710.14 ms /    51 runs   (   53.14 ms per token,    18.82 tokens per second)\n",
      "llama_print_timings:       total time =    2983.51 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "261 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.24 ms /    56 runs   (    0.09 ms per token, 10697.23 tokens per second)\n",
      "llama_print_timings: prompt eval time =     195.15 ms /    20 tokens (    9.76 ms per token,   102.48 tokens per second)\n",
      "llama_print_timings:        eval time =    2919.28 ms /    55 runs   (   53.08 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3186.65 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "262 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.51 ms /    57 runs   (    0.10 ms per token, 10344.83 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.46 ms /    28 tokens (    6.77 ms per token,   147.79 tokens per second)\n",
      "llama_print_timings:        eval time =    2971.65 ms /    56 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3233.98 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "263 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.16 ms /    56 runs   (    0.09 ms per token, 10863.24 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.38 ms /    16 tokens (   11.84 ms per token,    84.49 tokens per second)\n",
      "llama_print_timings:        eval time =    2916.59 ms /    55 runs   (   53.03 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3178.94 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "264 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.98 ms /    53 runs   (    0.09 ms per token, 10653.27 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.65 ms /    24 tokens (    8.36 ms per token,   119.61 tokens per second)\n",
      "llama_print_timings:        eval time =    2761.07 ms /    52 runs   (   53.10 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    3029.74 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "265 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.39 ms /    58 runs   (    0.09 ms per token, 10762.66 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.44 ms /    22 tokens (    9.07 ms per token,   110.31 tokens per second)\n",
      "llama_print_timings:        eval time =    3023.77 ms /    57 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3298.61 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "266 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.27 ms /    55 runs   (    0.10 ms per token, 10428.52 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.61 ms /    19 tokens (   10.24 ms per token,    97.63 tokens per second)\n",
      "llama_print_timings:        eval time =    2864.20 ms /    54 runs   (   53.04 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3132.23 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "267 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.86 ms /    56 runs   (    0.09 ms per token, 11534.50 tokens per second)\n",
      "llama_print_timings: prompt eval time =     357.56 ms /    43 tokens (    8.32 ms per token,   120.26 tokens per second)\n",
      "llama_print_timings:        eval time =    2923.80 ms /    55 runs   (   53.16 ms per token,    18.81 tokens per second)\n",
      "llama_print_timings:       total time =    3353.69 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "268 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.25 ms /    57 runs   (    0.09 ms per token, 10859.21 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.73 ms /    19 tokens (   10.25 ms per token,    97.57 tokens per second)\n",
      "llama_print_timings:        eval time =    2973.28 ms /    56 runs   (   53.09 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    3241.38 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "269 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.74 ms /    63 runs   (    0.09 ms per token, 10971.79 tokens per second)\n",
      "llama_print_timings: prompt eval time =     345.49 ms /    35 tokens (    9.87 ms per token,   101.31 tokens per second)\n",
      "llama_print_timings:        eval time =    3297.26 ms /    62 runs   (   53.18 ms per token,    18.80 tokens per second)\n",
      "llama_print_timings:       total time =    3725.37 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "270 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.95 ms /    61 runs   (    0.10 ms per token, 10248.66 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.94 ms /    23 tokens (    8.74 ms per token,   114.46 tokens per second)\n",
      "llama_print_timings:        eval time =    3188.26 ms /    60 runs   (   53.14 ms per token,    18.82 tokens per second)\n",
      "llama_print_timings:       total time =    3471.20 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "271 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.88 ms /    54 runs   (    0.09 ms per token, 11072.38 tokens per second)\n",
      "llama_print_timings: prompt eval time =     349.90 ms /    37 tokens (    9.46 ms per token,   105.75 tokens per second)\n",
      "llama_print_timings:        eval time =    2817.58 ms /    53 runs   (   53.16 ms per token,    18.81 tokens per second)\n",
      "llama_print_timings:       total time =    3236.74 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "272 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.05 ms /    52 runs   (    0.10 ms per token, 10301.11 tokens per second)\n",
      "llama_print_timings: prompt eval time =     206.01 ms /    32 tokens (    6.44 ms per token,   155.33 tokens per second)\n",
      "llama_print_timings:        eval time =    2705.35 ms /    51 runs   (   53.05 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    2978.21 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "273 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.29 ms /    56 runs   (    0.09 ms per token, 10596.03 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.96 ms /    20 tokens (    9.75 ms per token,   102.59 tokens per second)\n",
      "llama_print_timings:        eval time =    2915.98 ms /    55 runs   (   53.02 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3185.34 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "274 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.19 ms /    58 runs   (    0.09 ms per token, 11166.73 tokens per second)\n",
      "llama_print_timings: prompt eval time =     356.87 ms /    43 tokens (    8.30 ms per token,   120.49 tokens per second)\n",
      "llama_print_timings:        eval time =    3026.27 ms /    57 runs   (   53.09 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3461.29 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "275 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.43 ms /    57 runs   (    0.10 ms per token, 10503.04 tokens per second)\n",
      "llama_print_timings: prompt eval time =     209.40 ms /    29 tokens (    7.22 ms per token,   138.49 tokens per second)\n",
      "llama_print_timings:        eval time =    2971.52 ms /    56 runs   (   53.06 ms per token,    18.85 tokens per second)\n",
      "llama_print_timings:       total time =    3255.11 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "276 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.02 ms /    54 runs   (    0.09 ms per token, 10759.12 tokens per second)\n",
      "llama_print_timings: prompt eval time =     201.12 ms /    21 tokens (    9.58 ms per token,   104.42 tokens per second)\n",
      "llama_print_timings:        eval time =    2818.12 ms /    53 runs   (   53.17 ms per token,    18.81 tokens per second)\n",
      "llama_print_timings:       total time =    3092.55 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "277 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.12 ms /    53 runs   (    0.12 ms per token,  8660.13 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.94 ms /    22 tokens (    9.04 ms per token,   110.59 tokens per second)\n",
      "llama_print_timings:        eval time =    2804.61 ms /    52 runs   (   53.93 ms per token,    18.54 tokens per second)\n",
      "llama_print_timings:       total time =    3091.51 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "278 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       8.71 ms /    65 runs   (    0.13 ms per token,  7460.97 tokens per second)\n",
      "llama_print_timings: prompt eval time =     202.90 ms /    24 tokens (    8.45 ms per token,   118.29 tokens per second)\n",
      "llama_print_timings:        eval time =    3425.91 ms /    64 runs   (   53.53 ms per token,    18.68 tokens per second)\n",
      "llama_print_timings:       total time =    3750.46 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "279 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.09 ms /    58 runs   (    0.12 ms per token,  8179.38 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.74 ms /    19 tokens (   10.20 ms per token,    98.07 tokens per second)\n",
      "llama_print_timings:        eval time =    3031.85 ms /    57 runs   (   53.19 ms per token,    18.80 tokens per second)\n",
      "llama_print_timings:       total time =    3324.22 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "280 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.42 ms /    56 runs   (    0.10 ms per token, 10339.73 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.56 ms /    16 tokens (   11.79 ms per token,    84.85 tokens per second)\n",
      "llama_print_timings:        eval time =    2920.76 ms /    55 runs   (   53.10 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    3187.44 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "281 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.01 ms /    53 runs   (    0.09 ms per token, 10585.18 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.90 ms /    15 tokens (   12.66 ms per token,    78.99 tokens per second)\n",
      "llama_print_timings:        eval time =    2755.00 ms /    52 runs   (   52.98 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3017.76 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "282 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.08 ms /    56 runs   (    0.09 ms per token, 11012.78 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.29 ms /    15 tokens (   12.62 ms per token,    79.24 tokens per second)\n",
      "llama_print_timings:        eval time =    2913.97 ms /    55 runs   (   52.98 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3176.33 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "283 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.23 ms /    55 runs   (    0.10 ms per token, 10508.22 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.13 ms /    22 tokens (    9.05 ms per token,   110.48 tokens per second)\n",
      "llama_print_timings:        eval time =    2866.11 ms /    54 runs   (   53.08 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3140.27 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "284 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.09 ms /    53 runs   (    0.10 ms per token, 10414.62 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.46 ms /    21 tokens (    9.40 ms per token,   106.35 tokens per second)\n",
      "llama_print_timings:        eval time =    2753.89 ms /    52 runs   (   52.96 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3022.75 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "285 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.16 ms /    54 runs   (    0.10 ms per token, 10471.20 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.35 ms /    22 tokens (    9.06 ms per token,   110.36 tokens per second)\n",
      "llama_print_timings:        eval time =    2807.80 ms /    53 runs   (   52.98 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3079.27 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "286 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.89 ms /    55 runs   (    0.09 ms per token, 11238.25 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.42 ms /    15 tokens (   12.63 ms per token,    79.19 tokens per second)\n",
      "llama_print_timings:        eval time =    2857.38 ms /    54 runs   (   52.91 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    3118.83 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "287 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.27 ms /    55 runs   (    0.10 ms per token, 10444.36 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.39 ms /    18 tokens (   10.74 ms per token,    93.08 tokens per second)\n",
      "llama_print_timings:        eval time =    2861.84 ms /    54 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3129.49 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "288 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.95 ms /    53 runs   (    0.09 ms per token, 10709.23 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.09 ms /    19 tokens (   10.22 ms per token,    97.89 tokens per second)\n",
      "llama_print_timings:        eval time =    2752.73 ms /    52 runs   (   52.94 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3015.77 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "289 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.48 ms /    58 runs   (    0.09 ms per token, 10589.74 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.63 ms /    18 tokens (   10.76 ms per token,    92.96 tokens per second)\n",
      "llama_print_timings:        eval time =    3019.67 ms /    57 runs   (   52.98 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3289.63 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "290 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.16 ms /    55 runs   (    0.09 ms per token, 10656.85 tokens per second)\n",
      "llama_print_timings: prompt eval time =     192.90 ms /    18 tokens (   10.72 ms per token,    93.31 tokens per second)\n",
      "llama_print_timings:        eval time =    2858.64 ms /    54 runs   (   52.94 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3124.79 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "291 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.43 ms /    55 runs   (    0.10 ms per token, 10136.38 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.96 ms /    22 tokens (    9.04 ms per token,   110.57 tokens per second)\n",
      "llama_print_timings:        eval time =    2861.74 ms /    54 runs   (   53.00 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3136.47 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "292 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.05 ms /    50 runs   (    0.10 ms per token,  9900.99 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.27 ms /    23 tokens (    8.71 ms per token,   114.84 tokens per second)\n",
      "llama_print_timings:        eval time =    2609.46 ms /    49 runs   (   53.25 ms per token,    18.78 tokens per second)\n",
      "llama_print_timings:       total time =    2879.17 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.28 ms /    56 runs   (    0.11 ms per token,  8921.46 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.18 ms /    19 tokens (   10.22 ms per token,    97.85 tokens per second)\n",
      "llama_print_timings:        eval time =    2944.25 ms /    55 runs   (   53.53 ms per token,    18.68 tokens per second)\n",
      "llama_print_timings:       total time =    3237.18 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "293 -- 1\n",
      "294 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.29 ms /    53 runs   (    0.14 ms per token,  7271.23 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.71 ms /    16 tokens (   11.86 ms per token,    84.34 tokens per second)\n",
      "llama_print_timings:        eval time =    2794.32 ms /    52 runs   (   53.74 ms per token,    18.61 tokens per second)\n",
      "llama_print_timings:       total time =    3079.27 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.76 ms /    56 runs   (    0.12 ms per token,  8284.02 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.18 ms /    16 tokens (   12.07 ms per token,    82.82 tokens per second)\n",
      "llama_print_timings:        eval time =    2929.04 ms /    55 runs   (   53.26 ms per token,    18.78 tokens per second)\n",
      "llama_print_timings:       total time =    3218.08 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "295 -- 1\n",
      "296 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.87 ms /    56 runs   (    0.10 ms per token,  9538.41 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.33 ms /    19 tokens (   10.23 ms per token,    97.77 tokens per second)\n",
      "llama_print_timings:        eval time =    2918.71 ms /    55 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    3194.73 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "297 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.68 ms /    56 runs   (    0.10 ms per token,  9855.68 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.63 ms /    22 tokens (    9.03 ms per token,   110.76 tokens per second)\n",
      "llama_print_timings:        eval time =    2921.29 ms /    55 runs   (   53.11 ms per token,    18.83 tokens per second)\n",
      "llama_print_timings:       total time =    3201.33 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "298 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.75 ms /    59 runs   (    0.13 ms per token,  7609.96 tokens per second)\n",
      "llama_print_timings: prompt eval time =     201.53 ms /    23 tokens (    8.76 ms per token,   114.13 tokens per second)\n",
      "llama_print_timings:        eval time =    3089.22 ms /    58 runs   (   53.26 ms per token,    18.77 tokens per second)\n",
      "llama_print_timings:       total time =    3395.35 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "299 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       8.19 ms /    61 runs   (    0.13 ms per token,  7444.47 tokens per second)\n",
      "llama_print_timings: prompt eval time =     202.78 ms /    23 tokens (    8.82 ms per token,   113.43 tokens per second)\n",
      "llama_print_timings:        eval time =    3209.24 ms /    60 runs   (   53.49 ms per token,    18.70 tokens per second)\n",
      "llama_print_timings:       total time =    3522.84 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.00 ms /    51 runs   (    0.12 ms per token,  8495.75 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.40 ms /    16 tokens (   11.84 ms per token,    84.48 tokens per second)\n",
      "llama_print_timings:        eval time =    2653.73 ms /    50 runs   (   53.07 ms per token,    18.84 tokens per second)\n",
      "llama_print_timings:       total time =    2926.56 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.66 ms /    57 runs   (    0.13 ms per token,  7442.22 tokens per second)\n",
      "llama_print_timings: prompt eval time =     342.41 ms /    33 tokens (   10.38 ms per token,    96.38 tokens per second)\n",
      "llama_print_timings:        eval time =    2975.87 ms /    56 runs   (   53.14 ms per token,    18.82 tokens per second)\n",
      "llama_print_timings:       total time =    3420.62 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "301 -- 1\n",
      "302 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.60 ms /    54 runs   (    0.12 ms per token,  8176.86 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.26 ms /    21 tokens (    9.39 ms per token,   106.46 tokens per second)\n",
      "llama_print_timings:        eval time =    2816.03 ms /    53 runs   (   53.13 ms per token,    18.82 tokens per second)\n",
      "llama_print_timings:       total time =    3105.35 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.68 ms /    52 runs   (    0.11 ms per token,  9159.77 tokens per second)\n",
      "llama_print_timings: prompt eval time =     204.76 ms /    26 tokens (    7.88 ms per token,   126.98 tokens per second)\n",
      "llama_print_timings:        eval time =    2709.87 ms /    51 runs   (   53.13 ms per token,    18.82 tokens per second)\n",
      "llama_print_timings:       total time =    2997.38 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "303 -- 1\n",
      "304 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.21 ms /    58 runs   (    0.12 ms per token,  8047.73 tokens per second)\n",
      "llama_print_timings: prompt eval time =     201.69 ms /    24 tokens (    8.40 ms per token,   119.00 tokens per second)\n",
      "llama_print_timings:        eval time =    3028.88 ms /    57 runs   (   53.14 ms per token,    18.82 tokens per second)\n",
      "llama_print_timings:       total time =    3328.68 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "305 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.67 ms /    57 runs   (    0.13 ms per token,  7431.55 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.77 ms /    18 tokens (   10.77 ms per token,    92.89 tokens per second)\n",
      "llama_print_timings:        eval time =    2978.41 ms /    56 runs   (   53.19 ms per token,    18.80 tokens per second)\n",
      "llama_print_timings:       total time =    3278.45 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.12 ms /    54 runs   (    0.13 ms per token,  7587.47 tokens per second)\n",
      "llama_print_timings: prompt eval time =     197.81 ms /    21 tokens (    9.42 ms per token,   106.16 tokens per second)\n",
      "llama_print_timings:        eval time =    2841.66 ms /    53 runs   (   53.62 ms per token,    18.65 tokens per second)\n",
      "llama_print_timings:       total time =    3138.54 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "306 -- 1\n",
      "307 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.20 ms /    55 runs   (    0.11 ms per token,  8869.54 tokens per second)\n",
      "llama_print_timings: prompt eval time =     199.56 ms /    22 tokens (    9.07 ms per token,   110.25 tokens per second)\n",
      "llama_print_timings:        eval time =    2855.32 ms /    54 runs   (   52.88 ms per token,    18.91 tokens per second)\n",
      "llama_print_timings:       total time =    3139.95 ms\n",
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       7.73 ms /    56 runs   (    0.14 ms per token,  7249.19 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.66 ms /    18 tokens (   10.76 ms per token,    92.95 tokens per second)\n",
      "llama_print_timings:        eval time =    2915.47 ms /    55 runs   (   53.01 ms per token,    18.86 tokens per second)\n",
      "llama_print_timings:       total time =    3219.87 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "308 -- 1\n",
      "309 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.22 ms /    56 runs   (    0.11 ms per token,  9003.22 tokens per second)\n",
      "llama_print_timings: prompt eval time =     216.89 ms /    31 tokens (    7.00 ms per token,   142.93 tokens per second)\n",
      "llama_print_timings:        eval time =    2911.45 ms /    55 runs   (   52.94 ms per token,    18.89 tokens per second)\n",
      "llama_print_timings:       total time =    3214.05 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "310 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.60 ms /    56 runs   (    0.10 ms per token,  9992.86 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.62 ms /    22 tokens (    9.03 ms per token,   110.76 tokens per second)\n",
      "llama_print_timings:        eval time =    2907.43 ms /    55 runs   (   52.86 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    3184.71 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "311 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.17 ms /    53 runs   (    0.10 ms per token, 10241.55 tokens per second)\n",
      "llama_print_timings: prompt eval time =     200.94 ms /    23 tokens (    8.74 ms per token,   114.46 tokens per second)\n",
      "llama_print_timings:        eval time =    2775.97 ms /    52 runs   (   53.38 ms per token,    18.73 tokens per second)\n",
      "llama_print_timings:       total time =    3054.41 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "312 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.36 ms /    60 runs   (    0.11 ms per token,  9436.93 tokens per second)\n",
      "llama_print_timings: prompt eval time =     194.51 ms /    20 tokens (    9.73 ms per token,   102.82 tokens per second)\n",
      "llama_print_timings:        eval time =    3127.33 ms /    59 runs   (   53.01 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3415.35 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "313 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.14 ms /    53 runs   (    0.10 ms per token, 10313.29 tokens per second)\n",
      "llama_print_timings: prompt eval time =     193.98 ms /    19 tokens (   10.21 ms per token,    97.95 tokens per second)\n",
      "llama_print_timings:        eval time =    2748.71 ms /    52 runs   (   52.86 ms per token,    18.92 tokens per second)\n",
      "llama_print_timings:       total time =    3013.74 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "314 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       4.92 ms /    49 runs   (    0.10 ms per token,  9967.45 tokens per second)\n",
      "llama_print_timings: prompt eval time =     182.61 ms /    10 tokens (   18.26 ms per token,    54.76 tokens per second)\n",
      "llama_print_timings:        eval time =    2534.94 ms /    48 runs   (   52.81 ms per token,    18.94 tokens per second)\n",
      "llama_print_timings:       total time =    2789.01 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "315 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       5.30 ms /    55 runs   (    0.10 ms per token, 10375.40 tokens per second)\n",
      "llama_print_timings: prompt eval time =     198.78 ms /    22 tokens (    9.04 ms per token,   110.67 tokens per second)\n",
      "llama_print_timings:        eval time =    2857.47 ms /    54 runs   (   52.92 ms per token,    18.90 tokens per second)\n",
      "llama_print_timings:       total time =    3130.26 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "316 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.41 ms /    62 runs   (    0.10 ms per token,  9675.41 tokens per second)\n",
      "llama_print_timings: prompt eval time =     210.69 ms /    30 tokens (    7.02 ms per token,   142.39 tokens per second)\n",
      "llama_print_timings:        eval time =    3232.03 ms /    61 runs   (   52.98 ms per token,    18.87 tokens per second)\n",
      "llama_print_timings:       total time =    3531.06 ms\n",
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "317 -- 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    5882.46 ms\n",
      "llama_print_timings:      sample time =       6.08 ms /    62 runs   (    0.10 ms per token, 10197.37 tokens per second)\n",
      "llama_print_timings: prompt eval time =     188.53 ms /    28 tokens (    6.73 ms per token,   148.52 tokens per second)\n",
      "llama_print_timings:        eval time =    3231.61 ms /    61 runs   (   52.98 ms per token,    18.88 tokens per second)\n",
      "llama_print_timings:       total time =    3504.93 ms\n"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "i = 0\n",
    "for _, row in test.iterrows():\n",
    "  prepared_prompt = prompt + row['text'] + \"\\n### OUTPUT:\\n\"\n",
    "  json_out = ''\n",
    "  attempt = 0\n",
    "  while True:\n",
    "    attempt += 1\n",
    "    json_out = generate(prepared_prompt, attempt)\n",
    "    parsedJson = parseResult(json_out, row)\n",
    "    print(f\"{i} -- {attempt}\")\n",
    "    if parsedJson is not None:\n",
    "      parsedJson['text'] = row['text']\n",
    "      results.append(parsedJson)\n",
    "      i+=1\n",
    "      break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "318"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df = pd.DataFrame(results, columns=['text', 'pun_word', 'target_word', 'source_sense', 'target_sense'])\n",
    "results_df.to_json('outputs-task2-hetero-openocra7b.json', orient=\"records\")\n",
    "len(results_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cprf(true, pred):\n",
    "  correct = (true['pun_word'].str.strip().str.lower() == pred['pun_word'].str.strip().str.lower()).sum()\n",
    "  coverage = len(pred) / len(true)\n",
    "  precision = correct / len(pred)\n",
    "  recall = correct / len(true)\n",
    "  f1 = (2 * precision * recall) / (precision + recall)\n",
    "  return coverage, precision, recall, f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coverage: 1.0000\n",
      "Precision: 0.6258\n",
      "Recall: 0.6258\n",
      "F1-Score: 0.6258\n"
     ]
    }
   ],
   "source": [
    "c, p, r, f1 = cprf(test, results_df)\n",
    "print(f\"Coverage: {c:.4f}\")\n",
    "print(f\"Precision: {p:.4f}\")\n",
    "print(f\"Recall: {r:.4f}\")\n",
    "print(f\"F1-Score: {f1:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "llama",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}