draw.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import plotly\n",
    "import json\n",
    "import pathlib\n",
    "import polars as pl \n",
    "import plotly.express as px"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>batch</th>\n",
       "      <th>total_token</th>\n",
       "      <th>token_per_s</th>\n",
       "      <th>avg_first_token_latency</th>\n",
       "      <th>min_token_latency</th>\n",
       "      <th>max_token_latency</th>\n",
       "      <th>median_token_latency</th>\n",
       "      <th>avg_token_latency</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1980</td>\n",
       "      <td>33.000000</td>\n",
       "      <td>0.058836</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.051738</td>\n",
       "      <td>0.030188</td>\n",
       "      <td>0.030079</td>\n",
       "      <td>llama-7b@vllm-0.2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3717</td>\n",
       "      <td>61.950000</td>\n",
       "      <td>0.038135</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.033544</td>\n",
       "      <td>0.032201</td>\n",
       "      <td>0.032077</td>\n",
       "      <td>llama-7b@vllm-0.2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>5522</td>\n",
       "      <td>92.033333</td>\n",
       "      <td>0.045159</td>\n",
       "      <td>0.000020</td>\n",
       "      <td>0.066875</td>\n",
       "      <td>0.032459</td>\n",
       "      <td>0.032353</td>\n",
       "      <td>llama-7b@vllm-0.2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>7276</td>\n",
       "      <td>121.266667</td>\n",
       "      <td>0.064571</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.070608</td>\n",
       "      <td>0.032745</td>\n",
       "      <td>0.032674</td>\n",
       "      <td>llama-7b@vllm-0.2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>8985</td>\n",
       "      <td>149.750000</td>\n",
       "      <td>0.068814</td>\n",
       "      <td>0.000021</td>\n",
       "      <td>0.070883</td>\n",
       "      <td>0.033126</td>\n",
       "      <td>0.033049</td>\n",
       "      <td>llama-7b@vllm-0.2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>891</th>\n",
       "      <td>124</td>\n",
       "      <td>33645</td>\n",
       "      <td>560.750000</td>\n",
       "      <td>11.422629</td>\n",
       "      <td>0.000680</td>\n",
       "      <td>15.003589</td>\n",
       "      <td>0.070776</td>\n",
       "      <td>0.158934</td>\n",
       "      <td>llama-7b-gptq@tgi-1.3.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>892</th>\n",
       "      <td>125</td>\n",
       "      <td>33540</td>\n",
       "      <td>559.000000</td>\n",
       "      <td>11.397340</td>\n",
       "      <td>0.000767</td>\n",
       "      <td>15.002494</td>\n",
       "      <td>0.071026</td>\n",
       "      <td>0.160884</td>\n",
       "      <td>llama-7b-gptq@tgi-1.3.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>893</th>\n",
       "      <td>126</td>\n",
       "      <td>33340</td>\n",
       "      <td>555.666667</td>\n",
       "      <td>11.419920</td>\n",
       "      <td>0.000732</td>\n",
       "      <td>15.001899</td>\n",
       "      <td>0.071503</td>\n",
       "      <td>0.162987</td>\n",
       "      <td>llama-7b-gptq@tgi-1.3.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>894</th>\n",
       "      <td>127</td>\n",
       "      <td>33109</td>\n",
       "      <td>551.816667</td>\n",
       "      <td>11.557860</td>\n",
       "      <td>0.000592</td>\n",
       "      <td>15.004105</td>\n",
       "      <td>0.071722</td>\n",
       "      <td>0.164875</td>\n",
       "      <td>llama-7b-gptq@tgi-1.3.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>895</th>\n",
       "      <td>128</td>\n",
       "      <td>32953</td>\n",
       "      <td>549.216667</td>\n",
       "      <td>11.464229</td>\n",
       "      <td>0.001169</td>\n",
       "      <td>15.002849</td>\n",
       "      <td>0.072315</td>\n",
       "      <td>0.167376</td>\n",
       "      <td>llama-7b-gptq@tgi-1.3.3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>896 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     batch  total_token  token_per_s  avg_first_token_latency  \\\n",
       "0        1         1980    33.000000                 0.058836   \n",
       "1        2         3717    61.950000                 0.038135   \n",
       "2        3         5522    92.033333                 0.045159   \n",
       "3        4         7276   121.266667                 0.064571   \n",
       "4        5         8985   149.750000                 0.068814   \n",
       "..     ...          ...          ...                      ...   \n",
       "891    124        33645   560.750000                11.422629   \n",
       "892    125        33540   559.000000                11.397340   \n",
       "893    126        33340   555.666667                11.419920   \n",
       "894    127        33109   551.816667                11.557860   \n",
       "895    128        32953   549.216667                11.464229   \n",
       "\n",
       "     min_token_latency  max_token_latency  median_token_latency  \\\n",
       "0             0.000022           0.051738              0.030188   \n",
       "1             0.000022           0.033544              0.032201   \n",
       "2             0.000020           0.066875              0.032459   \n",
       "3             0.000022           0.070608              0.032745   \n",
       "4             0.000021           0.070883              0.033126   \n",
       "..                 ...                ...                   ...   \n",
       "891           0.000680          15.003589              0.070776   \n",
       "892           0.000767          15.002494              0.071026   \n",
       "893           0.000732          15.001899              0.071503   \n",
       "894           0.000592          15.004105              0.071722   \n",
       "895           0.001169          15.002849              0.072315   \n",
       "\n",
       "     avg_token_latency                     type  \n",
       "0             0.030079      llama-7b@vllm-0.2.5  \n",
       "1             0.032077      llama-7b@vllm-0.2.5  \n",
       "2             0.032353      llama-7b@vllm-0.2.5  \n",
       "3             0.032674      llama-7b@vllm-0.2.5  \n",
       "4             0.033049      llama-7b@vllm-0.2.5  \n",
       "..                 ...                      ...  \n",
       "891           0.158934  llama-7b-gptq@tgi-1.3.3  \n",
       "892           0.160884  llama-7b-gptq@tgi-1.3.3  \n",
       "893           0.162987  llama-7b-gptq@tgi-1.3.3  \n",
       "894           0.164875  llama-7b-gptq@tgi-1.3.3  \n",
       "895           0.167376  llama-7b-gptq@tgi-1.3.3  \n",
       "\n",
       "[896 rows x 9 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llamas = pl.concat([\n",
    "    pl.from_records(json.load(open('result/2023_12_15_12_05_57_llama_vllm.json'))['results']).drop('raw_ticks').sort('batch').with_columns(type=pl.lit('llama-7b@vllm-0.2.5')),\n",
    "    pl.from_records(json.load(open('result/2023_12_15_12_32_55_llama_vllm.json'))['results']).drop('raw_ticks').sort('batch').with_columns(type=pl.lit('llama-7b-awq@vllm-0.2.5')),\n",
    "    pl.from_records(json.load(open('result/2023_12_15_13_10_47_llama_vllm.json'))['results']).drop('raw_ticks').sort('batch').with_columns(type=pl.lit('llama-7b-gptq@vllm-gptq branch')),\n",
    "    pl.from_records(json.load(open('result/2023_12_15_14_14_12_llama_tgi.json'))['results']).drop('raw_ticks').sort('batch').with_columns(type=pl.lit('llama-7b@tgi-1.3.3')),\n",
    "    pl.from_records(json.load(open('result/2023_12_15_14_56_18_llama_tgi.json'))['results']).drop('raw_ticks').sort('batch').with_columns(type=pl.lit('llama-7b-eetq@tgi-1.3.3')),\n",
    "    pl.from_records(json.load(open('result/2023_12_15_15_17_00_llama_tgi.json'))['results']).drop('raw_ticks').sort('batch').with_columns(type=pl.lit('llama-7b-awq@tgi-1.3.3')),\n",
    "    pl.from_records(json.load(open('result/2023_12_15_15_39_43_llama_tgi.json'))['results']).drop('raw_ticks').sort('batch').with_columns(type=pl.lit('llama-7b-gptq@tgi-1.3.3')),\n",
    "]).to_pandas()\n",
    "llamas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "hovertemplate": "type=llama-7b@vllm-0.2.5<br>batch=%{x}<br>token_per_s=%{y}<extra></extra>",
         "legendgroup": "llama-7b@vllm-0.2.5",
         "line": {
          "color": "#636efa",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b@vllm-0.2.5",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          33,
          61.95,
          92.03333333333333,
          121.26666666666667,
          149.75,
          177.66666666666666,
          204.23333333333332,
          231.05,
          256.93333333333334,
          281.31666666666666,
          306.18333333333334,
          329.9166666666667,
          354.4,
          377.25,
          400.8,
          423.03333333333336,
          429.8333333333333,
          449.98333333333335,
          470.68333333333334,
          492.3666666666667,
          510.71666666666664,
          549.8666666666667,
          567,
          586.8666666666667,
          606.55,
          623.3,
          646.7833333333333,
          665.7333333333333,
          683.8333333333334,
          694.1666666666666,
          702.7833333333333,
          705.0833333333334,
          719.9333333333333,
          721.35,
          713.3666666666667,
          726.0333333333333,
          736.7333333333333,
          748.2333333333333,
          757.5333333333333,
          750.3,
          758.1,
          763.6333333333333,
          772.8,
          782.25,
          787.3833333333333,
          797.45,
          801.8,
          806.6166666666667,
          815.55,
          818.7166666666667,
          825.85,
          831.85,
          840.5666666666667,
          843.2333333333333,
          848.1333333333333,
          852.5666666666667,
          866.8333333333334,
          869.3666666666667,
          870.4166666666666,
          877.2666666666667,
          872.7166666666667,
          874.6333333333333,
          871.4166666666666,
          870.9166666666666,
          871.85,
          859.1666666666666,
          851.2833333333333,
          844.2333333333333,
          841.0833333333334,
          843.9833333333333,
          842.3666666666667,
          842.2166666666667,
          849.2,
          848.8,
          850.4333333333333,
          849.05,
          855.7166666666667,
          862.3666666666667,
          865.3833333333333,
          862.7833333333333,
          867.55,
          863.65,
          871.0666666666667,
          871.65,
          875.1333333333333,
          879.9,
          880.45,
          880.85,
          883.6333333333333,
          878.5666666666667,
          879.6333333333333,
          882.3333333333334,
          874.2166666666667,
          882.7166666666667,
          878.7833333333333,
          880.7166666666667,
          875.05,
          869.7166666666667,
          867.7666666666667,
          885.2833333333333,
          868.1,
          878.7833333333333,
          866.8833333333333,
          875.6333333333333,
          874.75,
          864.3333333333334,
          872.5666666666667,
          871.2833333333333,
          871,
          871.0833333333334,
          864.2833333333333,
          862.55,
          860.0166666666667,
          847.5333333333333,
          855.6166666666667,
          864.95,
          861.4,
          859.1833333333333,
          866.6833333333333,
          858.9833333333333,
          858.8166666666667,
          850.3666666666667,
          841.7666666666667,
          843.2833333333333,
          860.4833333333333,
          859.9666666666667,
          854.6833333333333,
          845.8833333333333
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=llama-7b-awq@vllm-0.2.5<br>batch=%{x}<br>token_per_s=%{y}<extra></extra>",
         "legendgroup": "llama-7b-awq@vllm-0.2.5",
         "line": {
          "color": "#EF553B",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b-awq@vllm-0.2.5",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          77.11666666666666,
          146.38333333333333,
          214.55,
          278.93333333333334,
          336.7,
          389.8333333333333,
          439.1,
          484.8,
          530.8166666666667,
          560.15,
          606.4666666666667,
          646.6333333333333,
          683.3,
          713.8666666666667,
          738.4833333333333,
          767.0666666666667,
          684.9833333333333,
          687.55,
          721.8333333333334,
          762.9,
          783.75,
          806.1666666666666,
          806.0833333333334,
          825.4833333333333,
          856.3166666666667,
          848.4,
          879.35,
          916.35,
          931.4666666666667,
          938.3833333333333,
          949.7833333333333,
          954.15,
          843.85,
          820.75,
          851.7666666666667,
          889.9,
          897.9,
          907.7166666666667,
          912.7333333333333,
          918.85,
          943.2833333333333,
          914.5833333333334,
          936.9833333333333,
          974.6666666666666,
          981.3666666666667,
          979.85,
          982.55,
          987,
          933.5,
          904.6166666666667,
          923.6666666666666,
          961.8666666666667,
          966.5,
          972.2,
          972.9166666666666,
          975.5166666666667,
          1000.3833333333333,
          969,
          994.6,
          1036.45,
          1030.85,
          1032.0833333333333,
          1038.6,
          1024.5333333333333,
          947.6833333333333,
          909.8333333333334,
          927.05,
          940.0166666666667,
          948.7833333333333,
          949.7333333333333,
          948.4833333333333,
          955.45,
          968.6833333333333,
          934.3166666666667,
          956.8833333333333,
          980.9166666666666,
          980.95,
          978.05,
          973.4,
          972.2666666666667,
          931.65,
          897.0166666666667,
          913.05,
          944.3333333333334,
          932.9,
          934.3666666666667,
          925.95,
          927.1166666666667,
          940.55,
          896.4333333333333,
          915.9833333333333,
          943.2833333333333,
          929.5,
          936.7833333333333,
          925.2,
          930.15,
          902.2333333333333,
          855.7333333333333,
          887.8833333333333,
          905.8,
          888.65,
          888.7,
          892.3166666666667,
          879.8333333333334,
          898.0166666666667,
          847.0166666666667,
          869.15,
          892.15,
          880.65,
          880.5666666666667,
          872.5833333333334,
          880.9,
          860.0666666666667,
          808.3833333333333,
          838.1833333333333,
          861.1,
          852,
          846.8833333333333,
          838.7666666666667,
          837.1,
          856.2,
          793.2333333333333,
          820.0166666666667,
          844.1,
          830.8833333333333,
          841.2166666666667,
          835.8166666666667,
          835.1333333333333
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=llama-7b-gptq@vllm-gptq branch<br>batch=%{x}<br>token_per_s=%{y}<extra></extra>",
         "legendgroup": "llama-7b-gptq@vllm-gptq branch",
         "line": {
          "color": "#00cc96",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b-gptq@vllm-gptq branch",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          98.23333333333333,
          178.15,
          252.76666666666668,
          313.28333333333336,
          369.2,
          427.95,
          442.0833333333333,
          466.8333333333333,
          365.18333333333334,
          378.78333333333336,
          410.8833333333333,
          443.15,
          466.5833333333333,
          482.48333333333335,
          486.71666666666664,
          537.5333333333333,
          459.1666666666667,
          462.55,
          482.9,
          510.1,
          524.6333333333333,
          533.95,
          533.9833333333333,
          576.9333333333333,
          510.71666666666664,
          507.5,
          524.4166666666666,
          545.1166666666667,
          556.3666666666667,
          563.2333333333333,
          561.9666666666667,
          591.75,
          544.15,
          533.1666666666666,
          548.2333333333333,
          570.0666666666667,
          575.0333333333333,
          579.45,
          575.4666666666667,
          602.7333333333333,
          558.6333333333333,
          549.2833333333333,
          563.0833333333334,
          581.9166666666666,
          587.8333333333334,
          591.2666666666667,
          590.2666666666667,
          608.9,
          579.5666666666667,
          567.2333333333333,
          592.6833333333333,
          605.1333333333333,
          615.1166666666667,
          623.0666666666667,
          628.5833333333334,
          638.05,
          648.5166666666667,
          656.5333333333333,
          666.05,
          675.8166666666667,
          685.5,
          692.5833333333334,
          697.8666666666667,
          702.5833333333334,
          703.4166666666666,
          708.8666666666667,
          714.6,
          727.8833333333333,
          731.5,
          739.15,
          746.0333333333333,
          748.85,
          755.8166666666667,
          755.7,
          759.9,
          769.1166666666667,
          774.2333333333333,
          779.7166666666667,
          782.05,
          790.3666666666667,
          794.2833333333333,
          797.3,
          801.65,
          808.2333333333333,
          812.85,
          816.9666666666667,
          815.7833333333333,
          820.0666666666667,
          826.2333333333333,
          825.2833333333333,
          829.2833333333333,
          836,
          838.2666666666667,
          840.8,
          839.5666666666667,
          842.1833333333333,
          851.9666666666667,
          850.0166666666667,
          851.6,
          863.1166666666667,
          863.45,
          854.7833333333333,
          862.0666666666667,
          862.5166666666667,
          873.75,
          854.7666666666667,
          857.8,
          864.0666666666667,
          867.1666666666666,
          869.7833333333333,
          865.5833333333334,
          868.2666666666667,
          873.35,
          864.9,
          871.0666666666667,
          877.5666666666667,
          879.5833333333334,
          880.35,
          882.3166666666667,
          877.25,
          890.3166666666667,
          880.2166666666667,
          899.6333333333333,
          900.2333333333333,
          902.2333333333333,
          897.7,
          902.4,
          900.5333333333333
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=llama-7b@tgi-1.3.3<br>batch=%{x}<br>token_per_s=%{y}<extra></extra>",
         "legendgroup": "llama-7b@tgi-1.3.3",
         "line": {
          "color": "#ab63fa",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b@tgi-1.3.3",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          32.85,
          61.13333333333333,
          90.61666666666666,
          119.53333333333333,
          146.51666666666668,
          174.76666666666668,
          201.16666666666666,
          226.6,
          252.26666666666668,
          275.6666666666667,
          300.56666666666666,
          324.65,
          348.8666666666667,
          371.55,
          393.3833333333333,
          414.1666666666667,
          421.76666666666665,
          440.05,
          460.2,
          461.68333333333334,
          461.3666666666667,
          461.45,
          459.31666666666666,
          461.05,
          462.15,
          458.8666666666667,
          459.98333333333335,
          460.6333333333333,
          458.95,
          459.15,
          458.98333333333335,
          458.71666666666664,
          459.78333333333336,
          457.78333333333336,
          460.53333333333336,
          460.28333333333336,
          460.3666666666667,
          462.81666666666666,
          462.28333333333336,
          461.6666666666667,
          462.3666666666667,
          460.3666666666667,
          460.4,
          461.98333333333335,
          461.1166666666667,
          461.06666666666666,
          461.03333333333336,
          458.53333333333336,
          459.75,
          459.25,
          459.4,
          460.21666666666664,
          458.6333333333333,
          457.78333333333336,
          459.05,
          458.21666666666664,
          459.4166666666667,
          457.46666666666664,
          457.6,
          458.9166666666667,
          458.48333333333335,
          457.21666666666664,
          458.23333333333335,
          456.35,
          458.1666666666667,
          456.21666666666664,
          454.3666666666667,
          455.75,
          456.6666666666667,
          457.93333333333334,
          458.1,
          459.18333333333334,
          460.2,
          459.65,
          461.05,
          464.55,
          463.8666666666667,
          463.95,
          463.8833333333333,
          462.7,
          464.1666666666667,
          462.1166666666667,
          462.35,
          463.8,
          463.45,
          463.25,
          462.5,
          462.65,
          463.55,
          461.5,
          460.0833333333333,
          463.18333333333334,
          463.03333333333336,
          461.23333333333335,
          463,
          461.93333333333334,
          463.51666666666665,
          461.1666666666667,
          461.6,
          462.55,
          462.2,
          461.98333333333335,
          462.03333333333336,
          460.8,
          461.3833333333333,
          460.43333333333334,
          459.95,
          461.9,
          460.2,
          461.46666666666664,
          460.35,
          460.98333333333335,
          461.81666666666666,
          459.8666666666667,
          460.43333333333334,
          461.45,
          461.21666666666664,
          461.2,
          461.01666666666665,
          460.15,
          461.23333333333335,
          458.6166666666667,
          459.51666666666665,
          461.06666666666666,
          460.5,
          460.78333333333336,
          460.8333333333333,
          457.5833333333333
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=llama-7b-eetq@tgi-1.3.3<br>batch=%{x}<br>token_per_s=%{y}<extra></extra>",
         "legendgroup": "llama-7b-eetq@tgi-1.3.3",
         "line": {
          "color": "#FFA15A",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b-eetq@tgi-1.3.3",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          55.233333333333334,
          106.23333333333333,
          156.38333333333333,
          206.08333333333334,
          251.53333333333333,
          295.6,
          337.01666666666665,
          379,
          419.15,
          453.53333333333336,
          492.31666666666666,
          528.8333333333334,
          561.7,
          594.6666666666666,
          626.4,
          655.1666666666666,
          690.3166666666667,
          714.6833333333333,
          745.4166666666666,
          777.3,
          800.65,
          826.55,
          845.1166666666667,
          868.9666666666667,
          892.6166666666667,
          905.3,
          929.15,
          959.15,
          978.6,
          1000.3666666666667,
          1019.4666666666667,
          965.6666666666666,
          981.2,
          962.2,
          972.0166666666667,
          984.4166666666666,
          976.9666666666667,
          978.05,
          977.3833333333333,
          972.6166666666667,
          976.2,
          960.3,
          971.6666666666666,
          984.2333333333333,
          978.2,
          979.1666666666666,
          974.65,
          972.75,
          973.6,
          957.0166666666667,
          965.0833333333334,
          981.3333333333334,
          971.8666666666667,
          975.75,
          971.7666666666667,
          971.2166666666667,
          977,
          958.1333333333333,
          968.75,
          983.85,
          980.1,
          972.25,
          974.8,
          965.7333333333333,
          979.4166666666666,
          958.6333333333333,
          968.9666666666667,
          980.1333333333333,
          976.3166666666667,
          974.85,
          971.7333333333333,
          969.4166666666666,
          977.3166666666667,
          951.7666666666667,
          965.7833333333333,
          979.1,
          972.6666666666666,
          971.5166666666667,
          970.4833333333333,
          970.0166666666667,
          979.5,
          960.0333333333333,
          968.9333333333333,
          980.1333333333333,
          978.1,
          978.4,
          974.5666666666667,
          973.4666666666667,
          975.0333333333333,
          960.2666666666667,
          968.15,
          981.95,
          976.3833333333333,
          974.3833333333333,
          973.8,
          971.8333333333334,
          977.6666666666666,
          956.8666666666667,
          968.6166666666667,
          979.6833333333333,
          971.9666666666667,
          976.8833333333333,
          974.2,
          971.05,
          975.4166666666666,
          963.25,
          968.7166666666667,
          981.4,
          974.8666666666667,
          975.6166666666667,
          971.0666666666667,
          973.1,
          976.3666666666667,
          960.05,
          970.4166666666666,
          979.1833333333333,
          977.9166666666666,
          975.8666666666667,
          971.6,
          963.2833333333333,
          976.25,
          951.0666666666667,
          964.2166666666667,
          971.2333333333333,
          977.5333333333333,
          968.9666666666667,
          971.0666666666667,
          965.05
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=llama-7b-awq@tgi-1.3.3<br>batch=%{x}<br>token_per_s=%{y}<extra></extra>",
         "legendgroup": "llama-7b-awq@tgi-1.3.3",
         "line": {
          "color": "#19d3f3",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b-awq@tgi-1.3.3",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          75.56666666666666,
          141.5,
          207.66666666666666,
          270.8333333333333,
          325.9166666666667,
          376.8833333333333,
          425.3666666666667,
          469.6666666666667,
          512.6333333333333,
          542.8,
          585.55,
          626.3166666666667,
          656.5333333333333,
          685.3166666666667,
          715.1,
          740.8,
          665.5833333333334,
          667.45,
          698.9666666666667,
          741.2666666666667,
          756.8666666666667,
          771.3,
          787.1,
          802.2666666666667,
          832.0833333333334,
          826.7166666666667,
          853.4333333333333,
          888.45,
          895.9333333333333,
          907.5666666666667,
          917.65,
          918.0666666666667,
          823.35,
          803.6666666666666,
          827.7666666666667,
          868.5833333333334,
          872.5833333333334,
          883.3666666666667,
          880.5166666666667,
          869.8,
          883.95,
          853.3666666666667,
          864.8333333333334,
          890.4166666666666,
          883.5333333333333,
          881.1833333333333,
          877.0333333333333,
          868.85,
          886.0833333333334,
          851.6833333333333,
          865.7833333333333,
          889.2333333333333,
          883.95,
          882.05,
          879.9166666666666,
          866.3333333333334,
          883.5666666666667,
          850.2833333333333,
          861.0666666666667,
          891.8666666666667,
          876.85,
          879.3,
          876.9333333333333,
          866.35,
          884.3833333333333,
          848.3,
          860.7,
          887.0333333333333,
          881.8833333333333,
          879.2333333333333,
          873.6833333333333,
          864.8833333333333,
          885.1333333333333,
          844.5166666666667,
          859.6833333333333,
          887,
          881.6666666666666,
          876.05,
          876.2666666666667,
          862.65,
          885.7,
          848.8333333333334,
          859.9166666666666,
          887.7666666666667,
          880.5166666666667,
          878.2666666666667,
          875.15,
          862.3833333333333,
          883.8833333333333,
          846.9333333333333,
          858.35,
          882.75,
          880.7,
          875.15,
          869.6333333333333,
          864.2,
          882.0333333333333,
          842.8166666666667,
          857.1833333333333,
          881.95,
          876.5333333333333,
          877.0333333333333,
          873.2166666666667,
          864.35,
          880.8,
          840.65,
          855.8666666666667,
          884.2666666666667,
          876.65,
          875.05,
          871.9,
          858.1333333333333,
          877.3833333333333,
          841.8166666666667,
          855.3166666666667,
          882.7,
          880.9833333333333,
          875.5,
          868.2666666666667,
          861.0833333333334,
          873.4166666666666,
          838.4333333333333,
          852.9833333333333,
          877.4666666666667,
          871.6833333333333,
          866.1333333333333,
          865.7833333333333,
          853.7166666666667
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=llama-7b-gptq@tgi-1.3.3<br>batch=%{x}<br>token_per_s=%{y}<extra></extra>",
         "legendgroup": "llama-7b-gptq@tgi-1.3.3",
         "line": {
          "color": "#FF6692",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b-gptq@tgi-1.3.3",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          95.66666666666667,
          172.21666666666667,
          243.3,
          303.26666666666665,
          357.71666666666664,
          418.8666666666667,
          431.8333333333333,
          456.45,
          358.35,
          372.45,
          403.6,
          430.8333333333333,
          455,
          475.45,
          478.6333333333333,
          529.6833333333333,
          454.3833333333333,
          455.5,
          476.85,
          502.1333333333333,
          517.25,
          527.4666666666667,
          523.1833333333333,
          565.3166666666667,
          505.9166666666667,
          500.48333333333335,
          517.2666666666667,
          540.5833333333334,
          549.8833333333333,
          557.7333333333333,
          555.2333333333333,
          586.25,
          534.6,
          523.85,
          539.75,
          562.9166666666666,
          567.0833333333334,
          571.75,
          567.7666666666667,
          591.8166666666667,
          554.4833333333333,
          539.5833333333334,
          544.75,
          556.8166666666667,
          554.8,
          552.4333333333333,
          550.3333333333334,
          545.5666666666667,
          554.0333333333333,
          539.8666666666667,
          543.8833333333333,
          557.5,
          555.1,
          551.9,
          548.9833333333333,
          545.8,
          553.1666666666666,
          538.6,
          544.05,
          555.6166666666667,
          553.7166666666667,
          550.9166666666666,
          548.0166666666667,
          543.9166666666666,
          551.1166666666667,
          534.5333333333333,
          540.5666666666667,
          554.7,
          552.9666666666667,
          549.2666666666667,
          548.7833333333333,
          544.6666666666666,
          552.5666666666667,
          537.7833333333333,
          543.4,
          555.15,
          553.9333333333333,
          551.9166666666666,
          549.9833333333333,
          546.5833333333334,
          555.8166666666667,
          538.9666666666667,
          545.3,
          557.4166666666666,
          554.35,
          552.75,
          551.0833333333334,
          546.9333333333333,
          554.85,
          539.2833333333333,
          545.6333333333333,
          557.7833333333333,
          556.1333333333333,
          553.7166666666667,
          550.1,
          546.8166666666667,
          554.8166666666667,
          539.8333333333334,
          546.4333333333333,
          559.15,
          557.4333333333333,
          554.8,
          551.6666666666666,
          548,
          555.6666666666666,
          540.75,
          546.9,
          557.6166666666667,
          558,
          554.3666666666667,
          551.3,
          547.7333333333333,
          555.7666666666667,
          540.3166666666667,
          547.8666666666667,
          560.1,
          558.1333333333333,
          555.4666666666667,
          552.1166666666667,
          547.75,
          555.5666666666667,
          541.0333333333333,
          547.3,
          560.75,
          559,
          555.6666666666666,
          551.8166666666667,
          549.2166666666667
         ],
         "yaxis": "y"
        }
       ],
       "layout": {
        "legend": {
         "title": {
          "text": "type"
         },
         "tracegroupgap": 0
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "Llama Throughput"
        },
        "xaxis": {
         "anchor": "y",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "batch"
         }
        },
        "yaxis": {
         "anchor": "x",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "token_per_s"
         }
        }
       }
      },
      "text/html": [
       "<div>                            <div id=\"af9664d9-344f-4fe5-b415-10a6a3631713\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"af9664d9-344f-4fe5-b415-10a6a3631713\")) {                    Plotly.newPlot(                        \"af9664d9-344f-4fe5-b415-10a6a3631713\",                        [{\"hovertemplate\":\"type=llama-7b@vllm-0.2.5\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003etoken_per_s=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b@vllm-0.2.5\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b@vllm-0.2.5\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[33.0,61.95,92.03333333333333,121.26666666666667,149.75,177.66666666666666,204.23333333333332,231.05,256.93333333333334,281.31666666666666,306.18333333333334,329.9166666666667,354.4,377.25,400.8,423.03333333333336,429.8333333333333,449.98333333333335,470.68333333333334,492.3666666666667,510.71666666666664,549.8666666666667,567.0,586.8666666666667,606.55,623.3,646.7833333333333,665.7333333333333,683.8333333333334,694.1666666666666,702.7833333333333,705.0833333333334,719.9333333333333,721.35,713.3666666666667,726.0333333333333,736.7333333333333,748.2333333333333,757.5333333333333,750.3,758.1,763.6333333333333,772.8,782.25,787.3833333333333,797.45,801.8,806.6166666666667,815.55,818.7166666666667,825.85,831.85,840.5666666666667,843.2333333333333,848.1333333333333,852.5666666666667,866.8333333333334,869.3666666666667,870.4166666666666,877.2666666666667,872.7166666666667,874.6333333333333,871.4166666666666,870.9166666666666,871.85,859.1666666666666,851.2833333333333,844.2333333333333,841.0833333333334,843.9833333333333,842.3666666666667,842.2166666666667,849.2,848.8,850.4333333333333,849.05,855.7166666666667,862.3666666666667,865.3833333333333,862.7833333333333,867.55,863.65,871.0666666666667,871.65,875.1333333333333,879.9,880.45,880.85,883.6333333333333,878.5666666666667,879.6333333333333,882.3333333333334,874.2166666666667,882.7166666666667,878.7833333333333,880.7166666666667,875.05,869.7166666666667,867.7666666666667,885.2833333333333,868.1,878.7833333333333,866.8833333333333,875.6333333333333,874.75,864.3333333333334,872.5666666666667,871.2833333333333,871.0,871.0833333333334,864.2833333333333,862.55,860.0166666666667,847.5333333333333,855.6166666666667,864.95,861.4,859.1833333333333,866.6833333333333,858.9833333333333,858.8166666666667,850.3666666666667,841.7666666666667,843.2833333333333,860.4833333333333,859.9666666666667,854.6833333333333,845.8833333333333],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=llama-7b-awq@vllm-0.2.5\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003etoken_per_s=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b-awq@vllm-0.2.5\",\"line\":{\"color\":\"#EF553B\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b-awq@vllm-0.2.5\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[77.11666666666666,146.38333333333333,214.55,278.93333333333334,336.7,389.8333333333333,439.1,484.8,530.8166666666667,560.15,606.4666666666667,646.6333333333333,683.3,713.8666666666667,738.4833333333333,767.0666666666667,684.9833333333333,687.55,721.8333333333334,762.9,783.75,806.1666666666666,806.0833333333334,825.4833333333333,856.3166666666667,848.4,879.35,916.35,931.4666666666667,938.3833333333333,949.7833333333333,954.15,843.85,820.75,851.7666666666667,889.9,897.9,907.7166666666667,912.7333333333333,918.85,943.2833333333333,914.5833333333334,936.9833333333333,974.6666666666666,981.3666666666667,979.85,982.55,987.0,933.5,904.6166666666667,923.6666666666666,961.8666666666667,966.5,972.2,972.9166666666666,975.5166666666667,1000.3833333333333,969.0,994.6,1036.45,1030.85,1032.0833333333333,1038.6,1024.5333333333333,947.6833333333333,909.8333333333334,927.05,940.0166666666667,948.7833333333333,949.7333333333333,948.4833333333333,955.45,968.6833333333333,934.3166666666667,956.8833333333333,980.9166666666666,980.95,978.05,973.4,972.2666666666667,931.65,897.0166666666667,913.05,944.3333333333334,932.9,934.3666666666667,925.95,927.1166666666667,940.55,896.4333333333333,915.9833333333333,943.2833333333333,929.5,936.7833333333333,925.2,930.15,902.2333333333333,855.7333333333333,887.8833333333333,905.8,888.65,888.7,892.3166666666667,879.8333333333334,898.0166666666667,847.0166666666667,869.15,892.15,880.65,880.5666666666667,872.5833333333334,880.9,860.0666666666667,808.3833333333333,838.1833333333333,861.1,852.0,846.8833333333333,838.7666666666667,837.1,856.2,793.2333333333333,820.0166666666667,844.1,830.8833333333333,841.2166666666667,835.8166666666667,835.1333333333333],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=llama-7b-gptq@vllm-gptq branch\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003etoken_per_s=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b-gptq@vllm-gptq branch\",\"line\":{\"color\":\"#00cc96\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b-gptq@vllm-gptq branch\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[98.23333333333333,178.15,252.76666666666668,313.28333333333336,369.2,427.95,442.0833333333333,466.8333333333333,365.18333333333334,378.78333333333336,410.8833333333333,443.15,466.5833333333333,482.48333333333335,486.71666666666664,537.5333333333333,459.1666666666667,462.55,482.9,510.1,524.6333333333333,533.95,533.9833333333333,576.9333333333333,510.71666666666664,507.5,524.4166666666666,545.1166666666667,556.3666666666667,563.2333333333333,561.9666666666667,591.75,544.15,533.1666666666666,548.2333333333333,570.0666666666667,575.0333333333333,579.45,575.4666666666667,602.7333333333333,558.6333333333333,549.2833333333333,563.0833333333334,581.9166666666666,587.8333333333334,591.2666666666667,590.2666666666667,608.9,579.5666666666667,567.2333333333333,592.6833333333333,605.1333333333333,615.1166666666667,623.0666666666667,628.5833333333334,638.05,648.5166666666667,656.5333333333333,666.05,675.8166666666667,685.5,692.5833333333334,697.8666666666667,702.5833333333334,703.4166666666666,708.8666666666667,714.6,727.8833333333333,731.5,739.15,746.0333333333333,748.85,755.8166666666667,755.7,759.9,769.1166666666667,774.2333333333333,779.7166666666667,782.05,790.3666666666667,794.2833333333333,797.3,801.65,808.2333333333333,812.85,816.9666666666667,815.7833333333333,820.0666666666667,826.2333333333333,825.2833333333333,829.2833333333333,836.0,838.2666666666667,840.8,839.5666666666667,842.1833333333333,851.9666666666667,850.0166666666667,851.6,863.1166666666667,863.45,854.7833333333333,862.0666666666667,862.5166666666667,873.75,854.7666666666667,857.8,864.0666666666667,867.1666666666666,869.7833333333333,865.5833333333334,868.2666666666667,873.35,864.9,871.0666666666667,877.5666666666667,879.5833333333334,880.35,882.3166666666667,877.25,890.3166666666667,880.2166666666667,899.6333333333333,900.2333333333333,902.2333333333333,897.7,902.4,900.5333333333333],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=llama-7b@tgi-1.3.3\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003etoken_per_s=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b@tgi-1.3.3\",\"line\":{\"color\":\"#ab63fa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b@tgi-1.3.3\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[32.85,61.13333333333333,90.61666666666666,119.53333333333333,146.51666666666668,174.76666666666668,201.16666666666666,226.6,252.26666666666668,275.6666666666667,300.56666666666666,324.65,348.8666666666667,371.55,393.3833333333333,414.1666666666667,421.76666666666665,440.05,460.2,461.68333333333334,461.3666666666667,461.45,459.31666666666666,461.05,462.15,458.8666666666667,459.98333333333335,460.6333333333333,458.95,459.15,458.98333333333335,458.71666666666664,459.78333333333336,457.78333333333336,460.53333333333336,460.28333333333336,460.3666666666667,462.81666666666666,462.28333333333336,461.6666666666667,462.3666666666667,460.3666666666667,460.4,461.98333333333335,461.1166666666667,461.06666666666666,461.03333333333336,458.53333333333336,459.75,459.25,459.4,460.21666666666664,458.6333333333333,457.78333333333336,459.05,458.21666666666664,459.4166666666667,457.46666666666664,457.6,458.9166666666667,458.48333333333335,457.21666666666664,458.23333333333335,456.35,458.1666666666667,456.21666666666664,454.3666666666667,455.75,456.6666666666667,457.93333333333334,458.1,459.18333333333334,460.2,459.65,461.05,464.55,463.8666666666667,463.95,463.8833333333333,462.7,464.1666666666667,462.1166666666667,462.35,463.8,463.45,463.25,462.5,462.65,463.55,461.5,460.0833333333333,463.18333333333334,463.03333333333336,461.23333333333335,463.0,461.93333333333334,463.51666666666665,461.1666666666667,461.6,462.55,462.2,461.98333333333335,462.03333333333336,460.8,461.3833333333333,460.43333333333334,459.95,461.9,460.2,461.46666666666664,460.35,460.98333333333335,461.81666666666666,459.8666666666667,460.43333333333334,461.45,461.21666666666664,461.2,461.01666666666665,460.15,461.23333333333335,458.6166666666667,459.51666666666665,461.06666666666666,460.5,460.78333333333336,460.8333333333333,457.5833333333333],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=llama-7b-eetq@tgi-1.3.3\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003etoken_per_s=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b-eetq@tgi-1.3.3\",\"line\":{\"color\":\"#FFA15A\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b-eetq@tgi-1.3.3\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[55.233333333333334,106.23333333333333,156.38333333333333,206.08333333333334,251.53333333333333,295.6,337.01666666666665,379.0,419.15,453.53333333333336,492.31666666666666,528.8333333333334,561.7,594.6666666666666,626.4,655.1666666666666,690.3166666666667,714.6833333333333,745.4166666666666,777.3,800.65,826.55,845.1166666666667,868.9666666666667,892.6166666666667,905.3,929.15,959.15,978.6,1000.3666666666667,1019.4666666666667,965.6666666666666,981.2,962.2,972.0166666666667,984.4166666666666,976.9666666666667,978.05,977.3833333333333,972.6166666666667,976.2,960.3,971.6666666666666,984.2333333333333,978.2,979.1666666666666,974.65,972.75,973.6,957.0166666666667,965.0833333333334,981.3333333333334,971.8666666666667,975.75,971.7666666666667,971.2166666666667,977.0,958.1333333333333,968.75,983.85,980.1,972.25,974.8,965.7333333333333,979.4166666666666,958.6333333333333,968.9666666666667,980.1333333333333,976.3166666666667,974.85,971.7333333333333,969.4166666666666,977.3166666666667,951.7666666666667,965.7833333333333,979.1,972.6666666666666,971.5166666666667,970.4833333333333,970.0166666666667,979.5,960.0333333333333,968.9333333333333,980.1333333333333,978.1,978.4,974.5666666666667,973.4666666666667,975.0333333333333,960.2666666666667,968.15,981.95,976.3833333333333,974.3833333333333,973.8,971.8333333333334,977.6666666666666,956.8666666666667,968.6166666666667,979.6833333333333,971.9666666666667,976.8833333333333,974.2,971.05,975.4166666666666,963.25,968.7166666666667,981.4,974.8666666666667,975.6166666666667,971.0666666666667,973.1,976.3666666666667,960.05,970.4166666666666,979.1833333333333,977.9166666666666,975.8666666666667,971.6,963.2833333333333,976.25,951.0666666666667,964.2166666666667,971.2333333333333,977.5333333333333,968.9666666666667,971.0666666666667,965.05],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=llama-7b-awq@tgi-1.3.3\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003etoken_per_s=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b-awq@tgi-1.3.3\",\"line\":{\"color\":\"#19d3f3\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b-awq@tgi-1.3.3\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[75.56666666666666,141.5,207.66666666666666,270.8333333333333,325.9166666666667,376.8833333333333,425.3666666666667,469.6666666666667,512.6333333333333,542.8,585.55,626.3166666666667,656.5333333333333,685.3166666666667,715.1,740.8,665.5833333333334,667.45,698.9666666666667,741.2666666666667,756.8666666666667,771.3,787.1,802.2666666666667,832.0833333333334,826.7166666666667,853.4333333333333,888.45,895.9333333333333,907.5666666666667,917.65,918.0666666666667,823.35,803.6666666666666,827.7666666666667,868.5833333333334,872.5833333333334,883.3666666666667,880.5166666666667,869.8,883.95,853.3666666666667,864.8333333333334,890.4166666666666,883.5333333333333,881.1833333333333,877.0333333333333,868.85,886.0833333333334,851.6833333333333,865.7833333333333,889.2333333333333,883.95,882.05,879.9166666666666,866.3333333333334,883.5666666666667,850.2833333333333,861.0666666666667,891.8666666666667,876.85,879.3,876.9333333333333,866.35,884.3833333333333,848.3,860.7,887.0333333333333,881.8833333333333,879.2333333333333,873.6833333333333,864.8833333333333,885.1333333333333,844.5166666666667,859.6833333333333,887.0,881.6666666666666,876.05,876.2666666666667,862.65,885.7,848.8333333333334,859.9166666666666,887.7666666666667,880.5166666666667,878.2666666666667,875.15,862.3833333333333,883.8833333333333,846.9333333333333,858.35,882.75,880.7,875.15,869.6333333333333,864.2,882.0333333333333,842.8166666666667,857.1833333333333,881.95,876.5333333333333,877.0333333333333,873.2166666666667,864.35,880.8,840.65,855.8666666666667,884.2666666666667,876.65,875.05,871.9,858.1333333333333,877.3833333333333,841.8166666666667,855.3166666666667,882.7,880.9833333333333,875.5,868.2666666666667,861.0833333333334,873.4166666666666,838.4333333333333,852.9833333333333,877.4666666666667,871.6833333333333,866.1333333333333,865.7833333333333,853.7166666666667],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=llama-7b-gptq@tgi-1.3.3\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003etoken_per_s=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b-gptq@tgi-1.3.3\",\"line\":{\"color\":\"#FF6692\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b-gptq@tgi-1.3.3\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[95.66666666666667,172.21666666666667,243.3,303.26666666666665,357.71666666666664,418.8666666666667,431.8333333333333,456.45,358.35,372.45,403.6,430.8333333333333,455.0,475.45,478.6333333333333,529.6833333333333,454.3833333333333,455.5,476.85,502.1333333333333,517.25,527.4666666666667,523.1833333333333,565.3166666666667,505.9166666666667,500.48333333333335,517.2666666666667,540.5833333333334,549.8833333333333,557.7333333333333,555.2333333333333,586.25,534.6,523.85,539.75,562.9166666666666,567.0833333333334,571.75,567.7666666666667,591.8166666666667,554.4833333333333,539.5833333333334,544.75,556.8166666666667,554.8,552.4333333333333,550.3333333333334,545.5666666666667,554.0333333333333,539.8666666666667,543.8833333333333,557.5,555.1,551.9,548.9833333333333,545.8,553.1666666666666,538.6,544.05,555.6166666666667,553.7166666666667,550.9166666666666,548.0166666666667,543.9166666666666,551.1166666666667,534.5333333333333,540.5666666666667,554.7,552.9666666666667,549.2666666666667,548.7833333333333,544.6666666666666,552.5666666666667,537.7833333333333,543.4,555.15,553.9333333333333,551.9166666666666,549.9833333333333,546.5833333333334,555.8166666666667,538.9666666666667,545.3,557.4166666666666,554.35,552.75,551.0833333333334,546.9333333333333,554.85,539.2833333333333,545.6333333333333,557.7833333333333,556.1333333333333,553.7166666666667,550.1,546.8166666666667,554.8166666666667,539.8333333333334,546.4333333333333,559.15,557.4333333333333,554.8,551.6666666666666,548.0,555.6666666666666,540.75,546.9,557.6166666666667,558.0,554.3666666666667,551.3,547.7333333333333,555.7666666666667,540.3166666666667,547.8666666666667,560.1,558.1333333333333,555.4666666666667,552.1166666666667,547.75,555.5666666666667,541.0333333333333,547.3,560.75,559.0,555.6666666666666,551.8166666666667,549.2166666666667],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"batch\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"token_per_s\"}},\"legend\":{\"title\":{\"text\":\"type\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Llama Throughput\"}},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('af9664d9-344f-4fe5-b415-10a6a3631713');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                });            </script>        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig_llama_throughput = px.line(llamas, x='batch', y='token_per_s', color='type', title='Llama Throughput')\n",
    "fig_llama_throughput.write_image('images/llama2-7b-throughput.png', scale=2, width=1440)\n",
    "fig_llama_throughput.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "hovertemplate": "type=llama-7b@vllm-0.2.5<br>batch=%{x}<br>avg_token_latency=%{y}<extra></extra>",
         "legendgroup": "llama-7b@vllm-0.2.5",
         "line": {
          "color": "#636efa",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b@vllm-0.2.5",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.03007858482358873,
          0.03207707125476514,
          0.0323533020789664,
          0.03267394882021391,
          0.03304903025030531,
          0.03340385220282668,
          0.033859011067985625,
          0.034218594539114855,
          0.03454925697178362,
          0.035064215658726916,
          0.035407597203095276,
          0.03581848951317848,
          0.03615010674797014,
          0.03643930761366144,
          0.03684701166762131,
          0.03719535783322842,
          0.03886163556418092,
          0.039276941607363385,
          0.03954313909951233,
          0.039851449723194356,
          0.040222825313945144,
          0.03920103513034665,
          0.03968972756120742,
          0.03998617663411178,
          0.040287723173851035,
          0.040720077831198666,
          0.04072772049958858,
          0.04092843239937044,
          0.041330091751100514,
          0.04192336122228074,
          0.042607181290653086,
          0.04362942570465321,
          0.04398686061682026,
          0.044962008219945095,
          0.04668968065858602,
          0.047574268842546556,
          0.04812648455733703,
          0.04855909999635346,
          0.04918698061306827,
          0.05078542489886473,
          0.05143190688511677,
          0.052216750236133515,
          0.05278438782352433,
          0.053363330705229334,
          0.05410469904578199,
          0.05456016454919794,
          0.05534657998844423,
          0.056144431860345234,
          0.05666953314658126,
          0.057499872148488354,
          0.05809411860900043,
          0.05887817397686277,
          0.05934283795675575,
          0.060221988480517664,
          0.06083957507414378,
          0.061621780002302555,
          0.06171612391072384,
          0.06229385855587446,
          0.06342702853773674,
          0.06410262775049308,
          0.0652298474808708,
          0.0663677600428142,
          0.06759450482035811,
          0.06866878032301815,
          0.06965669905188335,
          0.0711802730099721,
          0.07275971888205984,
          0.07417257042765647,
          0.07384937958910207,
          0.07488908872921345,
          0.0762307852360214,
          0.07768915196779863,
          0.07869696870685162,
          0.07972455211012663,
          0.07954948576726668,
          0.07985161430346506,
          0.07971122324266876,
          0.08037896860742888,
          0.08092562803363333,
          0.07951001410876231,
          0.08005480104434727,
          0.0810104503308799,
          0.08092851471704206,
          0.08270324305755686,
          0.08189597876095682,
          0.0821109481650718,
          0.08301389453208978,
          0.08366909573973819,
          0.0841659623314629,
          0.08535332933881903,
          0.08615026561402714,
          0.08676943507325763,
          0.08812839456584709,
          0.08843035736438172,
          0.08934323757924731,
          0.09051560315860961,
          0.09269618048233841,
          0.0932650945901188,
          0.09555077477615033,
          0.09353226257330273,
          0.0975730524860209,
          0.09609726037962701,
          0.09930123387614442,
          0.09826311732665084,
          0.09939466093568819,
          0.10089193954148536,
          0.10122461521422171,
          0.10241202606457156,
          0.10252932281781547,
          0.10363220939619505,
          0.10539135115321834,
          0.10671311445118725,
          0.10813328372250813,
          0.1115291778835087,
          0.1094894534215286,
          0.10933474740470925,
          0.11164625192184964,
          0.1123497546511484,
          0.11246754982645675,
          0.1147517686624116,
          0.11534351634657858,
          0.11677329726122176,
          0.12064228606994905,
          0.12075971471604158,
          0.11659851077625447,
          0.11661943959875003,
          0.11807136001154414,
          0.12055337170941918
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=llama-7b-awq@vllm-0.2.5<br>batch=%{x}<br>avg_token_latency=%{y}<extra></extra>",
         "legendgroup": "llama-7b-awq@vllm-0.2.5",
         "line": {
          "color": "#EF553B",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b-awq@vllm-0.2.5",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.012875345429280082,
          0.013545030326820298,
          0.013834350669379138,
          0.014155496834704276,
          0.014633094292784248,
          0.015129552140206436,
          0.01563000607611045,
          0.016171942800939882,
          0.01655166926546783,
          0.017381473245688547,
          0.017645205798250903,
          0.018060405890540584,
          0.0184361536375186,
          0.01899888156149546,
          0.019636692044332854,
          0.02017689913041123,
          0.024099142929292865,
          0.025355368003168576,
          0.025456740434258697,
          0.02526488609859611,
          0.025845270237598174,
          0.02628219269614279,
          0.027417113666109844,
          0.02800239458017076,
          0.02807496879207292,
          0.029432772378234842,
          0.029521243152584687,
          0.029342735873377716,
          0.029838997417952266,
          0.03058607216803436,
          0.031183343758575368,
          0.03203827441767611,
          0.03756106772469934,
          0.03966623068190571,
          0.0392361759903536,
          0.038684321260382785,
          0.03935448945312587,
          0.03995957405110211,
          0.04072233441205517,
          0.041451404252665046,
          0.0412161672889104,
          0.043563686204978164,
          0.043309016236423636,
          0.042766898169265634,
          0.04337113984014774,
          0.044377949213036,
          0.04520864972125848,
          0.046227001110515704,
          0.049983159195177145,
          0.05239857692425104,
          0.05234737904252367,
          0.05126956836316933,
          0.05190517543369591,
          0.052622662046423636,
          0.0534172040369668,
          0.0542745281688863,
          0.05367204079185855,
          0.056212059413399545,
          0.05588250802809644,
          0.05424837039181799,
          0.055560669574863275,
          0.05640733228521011,
          0.056953800148601935,
          0.05808126194883115,
          0.06327974156003578,
          0.06743824309462075,
          0.06693475316520725,
          0.0672923182641686,
          0.06726225731960297,
          0.06768404315304567,
          0.06854288806007222,
          0.06865744442732985,
          0.06844500723400265,
          0.0715127394228125,
          0.07084333680511351,
          0.06974523116725197,
          0.07043009568473205,
          0.07156199318391901,
          0.072455711734047,
          0.0731940804382019,
          0.07698565447583065,
          0.0805481924811049,
          0.08030349405433236,
          0.0782835835326378,
          0.0798932183054638,
          0.08028723210630838,
          0.08173794874707066,
          0.08286474693858029,
          0.08241487627861716,
          0.08666626773464993,
          0.08615858089703905,
          0.08438068201605249,
          0.08603813200227346,
          0.08647908171066235,
          0.08750227739655077,
          0.08872316377273934,
          0.0915428299772823,
          0.09691491512607167,
          0.09519509450601432,
          0.09411506501619891,
          0.09599445190323676,
          0.09706009156581312,
          0.0976177796734089,
          0.10016179991433811,
          0.09864064404991114,
          0.10417185295494909,
          0.10335306245856776,
          0.1017102857167982,
          0.10312293992404749,
          0.10462502353588096,
          0.1055254194935456,
          0.106661982395825,
          0.10898089339300153,
          0.11605297857749,
          0.11388610038946033,
          0.1117029624542117,
          0.11419465381287482,
          0.11585596691090538,
          0.11652547914996626,
          0.11879252029403106,
          0.11725814024898454,
          0.12581174696383085,
          0.12345308672254676,
          0.1210681983568218,
          0.12340291989046852,
          0.12326619389887415,
          0.12443451139330594,
          0.12505606114522005
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=llama-7b-gptq@vllm-gptq branch<br>batch=%{x}<br>avg_token_latency=%{y}<extra></extra>",
         "legendgroup": "llama-7b-gptq@vllm-gptq branch",
         "line": {
          "color": "#00cc96",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b-gptq@vllm-gptq branch",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.010094541407193632,
          0.011049684704368908,
          0.011669617031649817,
          0.012527654541948105,
          0.013256456141973608,
          0.013731403697822874,
          0.015515876613459082,
          0.016801542274723285,
          0.024280908997398553,
          0.025981479733312596,
          0.026301942650062773,
          0.02652412245345831,
          0.027369550338524732,
          0.028462242845175866,
          0.030273243629318786,
          0.029189585955785812,
          0.03626081001350268,
          0.03822427857651408,
          0.038617229902291716,
          0.03846473698363036,
          0.03924724294494559,
          0.04037979331198285,
          0.042121074411623396,
          0.040667155900941734,
          0.0479557079035996,
          0.050186601973858894,
          0.05040196932153174,
          0.050286666586021184,
          0.05098273567164931,
          0.052024939216310635,
          0.05388052560403937,
          0.05274557422436279,
          0.05910657347833004,
          0.06213400244695208,
          0.062203267367605064,
          0.061350312048006325,
          0.06260839903454869,
          0.063755004701556,
          0.06577416020206789,
          0.06437758919953557,
          0.07144602524697168,
          0.07439628409712665,
          0.07433149288805672,
          0.07352493027257624,
          0.07429014340561393,
          0.07559487198297425,
          0.07728595342684097,
          0.07650323659112057,
          0.0820079582574049,
          0.08544982983272599,
          0.08307496334400047,
          0.08317302311655825,
          0.0833621034525156,
          0.08350133829908517,
          0.08407573238761767,
          0.08479498954596038,
          0.08488768067803455,
          0.08523463896735375,
          0.08541414704141335,
          0.08530989450341579,
          0.08572114585222469,
          0.08615028705250871,
          0.08687531265150916,
          0.08725300623022174,
          0.08888659857836732,
          0.08931082229951583,
          0.08967493760926863,
          0.08957915096968366,
          0.09005869477997269,
          0.09042471815648592,
          0.09107592644977885,
          0.09174723629538621,
          0.09216584371620092,
          0.09298206006223803,
          0.09380067273680748,
          0.09374324173276705,
          0.09365351463852409,
          0.09461717501023865,
          0.09511995023696261,
          0.0954118100491544,
          0.09568157145483328,
          0.09650075866353114,
          0.09684716486103667,
          0.0973561107476594,
          0.09782800784693775,
          0.09825865196493123,
          0.09942725487766359,
          0.0998780457922121,
          0.10030327244969134,
          0.10095072533580388,
          0.10180480744407826,
          0.10193881503391142,
          0.1025978901229719,
          0.10292544638935128,
          0.10430284029001077,
          0.1049056438595713,
          0.10460929344585077,
          0.10600029789092363,
          0.10660676074260221,
          0.10653806329787409,
          0.10724297510614386,
          0.1087813466716127,
          0.10871006590603963,
          0.10958659691319983,
          0.10988863522065806,
          0.11273441807415273,
          0.11361863095158155,
          0.11352306493981933,
          0.11412765189794336,
          0.11483691095025583,
          0.11594504018794648,
          0.11707218210449938,
          0.11743371929804763,
          0.11928749115121202,
          0.11912270184155453,
          0.11978593360468347,
          0.12039018688106107,
          0.12103726291721105,
          0.12155846537877554,
          0.12295238954444744,
          0.12241979158368739,
          0.12436584439965534,
          0.12250129874978716,
          0.1239056527965897,
          0.12461948392971156,
          0.1253775541290335,
          0.12556658145988486,
          0.12643535392722835
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=llama-7b@tgi-1.3.3<br>batch=%{x}<br>avg_token_latency=%{y}<extra></extra>",
         "legendgroup": "llama-7b@tgi-1.3.3",
         "line": {
          "color": "#ab63fa",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b@tgi-1.3.3",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.030305246997971106,
          0.03253334405670573,
          0.032889617089572123,
          0.03320942346806995,
          0.03371695439813467,
          0.034028316981785205,
          0.03447434066793664,
          0.03492873368225945,
          0.035294029233153626,
          0.03585953612230959,
          0.036194619667627874,
          0.03655305328173916,
          0.03681373007887452,
          0.037175870828735746,
          0.03759796399110306,
          0.038023545778551146,
          0.03967216197328627,
          0.040208880225012164,
          0.04057111312070854,
          0.04067760940193478,
          0.04090378694960594,
          0.04110037339939331,
          0.041390334940382405,
          0.04160457596233253,
          0.04170445783288237,
          0.0421126174004433,
          0.04227074279995647,
          0.04232680646707437,
          0.04215346516752022,
          0.042452557713419874,
          0.042718797754312005,
          0.043016277658976115,
          0.04314723611501743,
          0.04365179230946253,
          0.04382054469010575,
          0.04399890148397709,
          0.044334716103866544,
          0.044396973172566814,
          0.046039020407037404,
          0.04767788959620896,
          0.04909123584316214,
          0.051021117929983435,
          0.05259908649033458,
          0.05378689687957696,
          0.05548317109520334,
          0.05699158074454161,
          0.05855478295687231,
          0.060449081315498546,
          0.06163252062863862,
          0.0634328282940663,
          0.06489905115531136,
          0.06626292742965267,
          0.06785837307235236,
          0.0695200481234212,
          0.07081512481007902,
          0.07252821618433782,
          0.07372405014801416,
          0.07571175742094126,
          0.07695313186429191,
          0.07847125246722324,
          0.08010217200599831,
          0.08154505499759378,
          0.0831250822891177,
          0.08487853801212523,
          0.08616165670931286,
          0.08809411181781292,
          0.0903930695519409,
          0.09161968274093933,
          0.09350789517025555,
          0.09493763197849042,
          0.09661439198592008,
          0.09848383215981223,
          0.09989139578136313,
          0.10233072691493515,
          0.10390053827600053,
          0.10509427282147656,
          0.10692895278575035,
          0.10841656244616861,
          0.1098648026604388,
          0.11178263631064059,
          0.11297130561127496,
          0.11520332385847748,
          0.1167012804113403,
          0.1178289020687792,
          0.11950531749491147,
          0.1211112675809322,
          0.1226055189723968,
          0.12441938951507618,
          0.125643152753316,
          0.12790616769873572,
          0.1292757246269154,
          0.13041907657507812,
          0.1319798358218996,
          0.13354796500278268,
          0.13518184436490263,
          0.1371500382681846,
          0.13812113319963354,
          0.14042799107097895,
          0.14181661087500014,
          0.1429637857660071,
          0.14459832658427096,
          0.14621730641448127,
          0.14774003828850726,
          0.14947600656405505,
          0.15062273298688694,
          0.1530775651221675,
          0.15444085278718697,
          0.15547675582600914,
          0.15704992153864258,
          0.15861709395406012,
          0.1601501408282824,
          0.16188451463292203,
          0.16309698734306513,
          0.16545161072325307,
          0.16676308864106326,
          0.167825451497923,
          0.1695012614936937,
          0.1709411659680543,
          0.17246242193980652,
          0.17437293546818064,
          0.1753249285964443,
          0.17746482036435587,
          0.17911259683645875,
          0.17982387166418415,
          0.1817500236732532,
          0.18308286735776058,
          0.18464697996734142,
          0.18647258516995074
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=llama-7b-eetq@tgi-1.3.3<br>batch=%{x}<br>avg_token_latency=%{y}<extra></extra>",
         "legendgroup": "llama-7b-eetq@tgi-1.3.3",
         "line": {
          "color": "#FFA15A",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b-eetq@tgi-1.3.3",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.01797777643391615,
          0.018697935631158274,
          0.019009534588617937,
          0.01925713715584346,
          0.019712125942618473,
          0.020115998279544318,
          0.020570936734286265,
          0.020910998519349494,
          0.021235132986600423,
          0.021778782228502328,
          0.022051343028538745,
          0.022380669778852308,
          0.022809555117559013,
          0.023181700525616473,
          0.023570517218257637,
          0.024019501607351793,
          0.024214363599797374,
          0.02472138320211746,
          0.02500125580804914,
          0.025229615519468902,
          0.025695467731551628,
          0.02606446715847009,
          0.026586123891947616,
          0.026966830629310597,
          0.027338735789682062,
          0.027962333967727175,
          0.028333372308238833,
          0.028509210274583526,
          0.028919853548998502,
          0.029219187420778978,
          0.029604169987771434,
          0.032244055597911776,
          0.0327125129997791,
          0.033372244314782064,
          0.03303424069386682,
          0.032633235173537466,
          0.03286801521155636,
          0.03280415308145471,
          0.03289615886000039,
          0.033033614674743375,
          0.03281582429160685,
          0.033446229617185304,
          0.03305578777710124,
          0.032643824992159724,
          0.032807727547025274,
          0.032829413871710345,
          0.032977325853075466,
          0.033006790559188114,
          0.032847721729423504,
          0.03349223884037201,
          0.0332009964998709,
          0.032683017438128556,
          0.032843254061822674,
          0.032937142920437104,
          0.033085051215706984,
          0.03309502350409307,
          0.032898046976373434,
          0.03350129126621213,
          0.03305817009128601,
          0.03263592244576581,
          0.03274628685368834,
          0.032837103531567796,
          0.03284651597768801,
          0.03301480867078565,
          0.03267129163866228,
          0.033291789319065526,
          0.03362607941752378,
          0.033890183733182346,
          0.03466393380559619,
          0.035372799679027435,
          0.036131371056840035,
          0.03690181622403523,
          0.03724278409614721,
          0.03888642815231324,
          0.03903211224111707,
          0.03893128077366963,
          0.04000847636432477,
          0.04067958649101947,
          0.04139971060722305,
          0.042151736818920864,
          0.04245794082006129,
          0.04436261150439231,
          0.04436402784187061,
          0.04424693343065559,
          0.04532193989522181,
          0.04600940109982293,
          0.04697955586282799,
          0.047751044823568754,
          0.04807485550880312,
          0.05006375601662748,
          0.0502013395351616,
          0.049869604225843196,
          0.05097866494110913,
          0.051581757907308574,
          0.05250593345538061,
          0.053251227067775735,
          0.05353798160637578,
          0.05505290673935786,
          0.05541262285028482,
          0.055365505143617925,
          0.05616834175688442,
          0.0569392433731427,
          0.05775303321677622,
          0.0586455719675611,
          0.05899994179654836,
          0.06024156728997324,
          0.060340173220394554,
          0.060480993732202025,
          0.061262232652960644,
          0.06217905788234786,
          0.0628870206831491,
          0.06352111976653659,
          0.0639148112508788,
          0.06586583336202993,
          0.06565604370289409,
          0.06533686713157194,
          0.06630940860630605,
          0.06713530390982343,
          0.0678862652076297,
          0.06904584415028515,
          0.06879935696332887,
          0.07162956430061009,
          0.07141127556282305,
          0.07074661842325904,
          0.07128620510040584,
          0.0726637912620147,
          0.0732752854356206,
          0.07454642683724537
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=llama-7b-awq@tgi-1.3.3<br>batch=%{x}<br>avg_token_latency=%{y}<extra></extra>",
         "legendgroup": "llama-7b-awq@tgi-1.3.3",
         "line": {
          "color": "#19d3f3",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b-awq@tgi-1.3.3",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.01311240498478183,
          0.013997631243698612,
          0.014304282149117479,
          0.014617797310830568,
          0.015151251855075315,
          0.01569423599752383,
          0.016211829332145225,
          0.016749457373705066,
          0.017227408964724755,
          0.018048556143023992,
          0.01837754821096957,
          0.018710168822613702,
          0.0192987819697403,
          0.01987774702271479,
          0.02042364338474772,
          0.02100243098895319,
          0.024881237871315697,
          0.02619360604951701,
          0.02637773248795817,
          0.026142380918585385,
          0.026844601205562493,
          0.027576034897942876,
          0.028288216759857036,
          0.028960002649576305,
          0.029032343232027923,
          0.03036448408392617,
          0.03047127389313726,
          0.030319916190425164,
          0.031083992569387656,
          0.03173408353125599,
          0.03237159531306428,
          0.03337122783824719,
          0.038562508921803115,
          0.04059102829382007,
          0.04055030871547938,
          0.039716558476849406,
          0.04057048984948896,
          0.04114182913820231,
          0.04143234612812553,
          0.04214312271491515,
          0.04123750257124276,
          0.04335558260493716,
          0.04293253531289263,
          0.04174590588076746,
          0.042258319878819874,
          0.04260953156499781,
          0.04221213330827393,
          0.04364237846571136,
          0.04280416785766947,
          0.045145954926419596,
          0.043816402720754044,
          0.04312405245591702,
          0.043609140958538295,
          0.04389258457343728,
          0.04413386099568148,
          0.044531400423816696,
          0.04351707105762637,
          0.04607390678469927,
          0.04549308360883791,
          0.04344336943827549,
          0.04435521707931802,
          0.04474899211795356,
          0.04510503371746213,
          0.04614688018185456,
          0.044946911178184536,
          0.04796592917580267,
          0.047209773114461354,
          0.04523017914445152,
          0.045806444015685996,
          0.046259674410130056,
          0.04694598875521257,
          0.04805863870429548,
          0.04654678293422847,
          0.050540364089854,
          0.04940064720365999,
          0.047172790624576896,
          0.04834852455139916,
          0.04952765301704636,
          0.05059374995596863,
          0.05235411652242136,
          0.05125667630939777,
          0.05577691705564459,
          0.05544715972196933,
          0.05336389284665653,
          0.05484764021216713,
          0.055857997825717064,
          0.05694525241484283,
          0.05878027320070365,
          0.057582160216582065,
          0.06229088829166027,
          0.06183860307768764,
          0.05959013972702727,
          0.06087827142191823,
          0.062039419287798006,
          0.06322503805515743,
          0.06503222115054377,
          0.06348118857938115,
          0.06904204657819966,
          0.0680192852280146,
          0.0655959270562393,
          0.06698855023497884,
          0.0679753406613774,
          0.06914795627799496,
          0.0709289528698635,
          0.06964499053354889,
          0.07487200402973886,
          0.07413910199963016,
          0.07143100927542566,
          0.07298391149061755,
          0.07393978503317884,
          0.07498285466855906,
          0.07697604052223801,
          0.07542352567908363,
          0.08119086856506813,
          0.08007534660698741,
          0.07720341561549822,
          0.07796063342363636,
          0.07939266120972792,
          0.08121910977506927,
          0.08281787117365753,
          0.08154541313086514,
          0.0879217934092356,
          0.08634090338300841,
          0.08361600125854723,
          0.08512898111906109,
          0.08602390934213361,
          0.08733408778381818,
          0.08962615275581338
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=llama-7b-gptq@tgi-1.3.3<br>batch=%{x}<br>avg_token_latency=%{y}<extra></extra>",
         "legendgroup": "llama-7b-gptq@tgi-1.3.3",
         "line": {
          "color": "#FF6692",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b-gptq@tgi-1.3.3",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.01033642817299659,
          0.011467812645601982,
          0.01216348892307124,
          0.012938801651736762,
          0.01371040416386347,
          0.014036010707822325,
          0.015892558405480633,
          0.017200275898821074,
          0.02473053627208075,
          0.02642786063829629,
          0.026814037919764135,
          0.027320809018762018,
          0.02806652534835165,
          0.02887925894510479,
          0.030758657176439854,
          0.02961642730836032,
          0.03678246650097187,
          0.03881029601390453,
          0.03900561386036849,
          0.03909099929065962,
          0.03980709043876281,
          0.04088535724630305,
          0.04302048314921472,
          0.041494453696217196,
          0.04844769051563202,
          0.05089119238895739,
          0.05098368610871269,
          0.05068871373090826,
          0.05130967167241569,
          0.052540475659962685,
          0.05450919785327489,
          0.0532552875208247,
          0.059992325887984616,
          0.06309985272963026,
          0.06311047592860929,
          0.062280337965092435,
          0.06331401685951502,
          0.0645735051023757,
          0.06671640106337706,
          0.06562111871024227,
          0.07202484649388287,
          0.07486496400312019,
          0.0750411940057821,
          0.07424263978305276,
          0.07530879661187231,
          0.07621382846156399,
          0.0776232103206542,
          0.07911964328994321,
          0.07878796842392143,
          0.0815155468117436,
          0.08195750472233647,
          0.08076758755381154,
          0.08193623378949716,
          0.08323288238702704,
          0.08453980601782066,
          0.08584187726652034,
          0.08550411201241345,
          0.08867574445472784,
          0.088594350554943,
          0.08752760984083728,
          0.08825826108945595,
          0.08957012806477087,
          0.09092132514990434,
          0.09246045908417366,
          0.09174569023153031,
          0.09546269531819071,
          0.09537963386036254,
          0.09400272346400428,
          0.09528569991289419,
          0.09652886270891488,
          0.09806848137297665,
          0.09978441329883728,
          0.09924015970293126,
          0.10308637176902098,
          0.10314005659302575,
          0.10174266281425479,
          0.1030394136054886,
          0.10453539467655147,
          0.105820085687142,
          0.10753752401396546,
          0.10668715739986209,
          0.11125041948645586,
          0.1115409056822239,
          0.11027636236959167,
          0.11163883416072869,
          0.11367879562023261,
          0.11532920824418579,
          0.11747325247598754,
          0.11689858279474939,
          0.12172445398637081,
          0.1216301658279978,
          0.12020511482009141,
          0.12178508736340227,
          0.1236934308453182,
          0.1254847610654423,
          0.12774529344393348,
          0.12725902480203624,
          0.1320780015573633,
          0.1317717848306592,
          0.13001075972666828,
          0.13170376378054216,
          0.13356457240987707,
          0.13565752954610283,
          0.13786563273433697,
          0.13721130897126582,
          0.142260087558391,
          0.1418688175540624,
          0.13955066421771223,
          0.14140641659498254,
          0.14360100969740208,
          0.14543303318522843,
          0.14781203005154503,
          0.1469389408188092,
          0.1524650999536381,
          0.15130661299531517,
          0.14946590574272414,
          0.15127262129482202,
          0.15277653713244133,
          0.15532773233133082,
          0.15757293981977807,
          0.1563622383755623,
          0.1623868286051068,
          0.16180990838531562,
          0.15893379894979856,
          0.16088446514016114,
          0.16298682224682098,
          0.16487461541194215,
          0.16737590481704265
         ],
         "yaxis": "y"
        }
       ],
       "layout": {
        "legend": {
         "title": {
          "text": "type"
         },
         "tracegroupgap": 0
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "Llama Average none-first Token Latency"
        },
        "xaxis": {
         "anchor": "y",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "batch"
         }
        },
        "yaxis": {
         "anchor": "x",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "avg_token_latency"
         }
        }
       }
      },
      "text/html": [
       "<div>                            <div id=\"01e79e18-fc83-47c2-878b-0bf1aadef388\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"01e79e18-fc83-47c2-878b-0bf1aadef388\")) {                    Plotly.newPlot(                        \"01e79e18-fc83-47c2-878b-0bf1aadef388\",                        [{\"hovertemplate\":\"type=llama-7b@vllm-0.2.5\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b@vllm-0.2.5\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b@vllm-0.2.5\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.03007858482358873,0.03207707125476514,0.0323533020789664,0.03267394882021391,0.03304903025030531,0.03340385220282668,0.033859011067985625,0.034218594539114855,0.03454925697178362,0.035064215658726916,0.035407597203095276,0.03581848951317848,0.03615010674797014,0.03643930761366144,0.03684701166762131,0.03719535783322842,0.03886163556418092,0.039276941607363385,0.03954313909951233,0.039851449723194356,0.040222825313945144,0.03920103513034665,0.03968972756120742,0.03998617663411178,0.040287723173851035,0.040720077831198666,0.04072772049958858,0.04092843239937044,0.041330091751100514,0.04192336122228074,0.042607181290653086,0.04362942570465321,0.04398686061682026,0.044962008219945095,0.04668968065858602,0.047574268842546556,0.04812648455733703,0.04855909999635346,0.04918698061306827,0.05078542489886473,0.05143190688511677,0.052216750236133515,0.05278438782352433,0.053363330705229334,0.05410469904578199,0.05456016454919794,0.05534657998844423,0.056144431860345234,0.05666953314658126,0.057499872148488354,0.05809411860900043,0.05887817397686277,0.05934283795675575,0.060221988480517664,0.06083957507414378,0.061621780002302555,0.06171612391072384,0.06229385855587446,0.06342702853773674,0.06410262775049308,0.0652298474808708,0.0663677600428142,0.06759450482035811,0.06866878032301815,0.06965669905188335,0.0711802730099721,0.07275971888205984,0.07417257042765647,0.07384937958910207,0.07488908872921345,0.0762307852360214,0.07768915196779863,0.07869696870685162,0.07972455211012663,0.07954948576726668,0.07985161430346506,0.07971122324266876,0.08037896860742888,0.08092562803363333,0.07951001410876231,0.08005480104434727,0.0810104503308799,0.08092851471704206,0.08270324305755686,0.08189597876095682,0.0821109481650718,0.08301389453208978,0.08366909573973819,0.0841659623314629,0.08535332933881903,0.08615026561402714,0.08676943507325763,0.08812839456584709,0.08843035736438172,0.08934323757924731,0.09051560315860961,0.09269618048233841,0.0932650945901188,0.09555077477615033,0.09353226257330273,0.0975730524860209,0.09609726037962701,0.09930123387614442,0.09826311732665084,0.09939466093568819,0.10089193954148536,0.10122461521422171,0.10241202606457156,0.10252932281781547,0.10363220939619505,0.10539135115321834,0.10671311445118725,0.10813328372250813,0.1115291778835087,0.1094894534215286,0.10933474740470925,0.11164625192184964,0.1123497546511484,0.11246754982645675,0.1147517686624116,0.11534351634657858,0.11677329726122176,0.12064228606994905,0.12075971471604158,0.11659851077625447,0.11661943959875003,0.11807136001154414,0.12055337170941918],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=llama-7b-awq@vllm-0.2.5\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b-awq@vllm-0.2.5\",\"line\":{\"color\":\"#EF553B\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b-awq@vllm-0.2.5\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.012875345429280082,0.013545030326820298,0.013834350669379138,0.014155496834704276,0.014633094292784248,0.015129552140206436,0.01563000607611045,0.016171942800939882,0.01655166926546783,0.017381473245688547,0.017645205798250903,0.018060405890540584,0.0184361536375186,0.01899888156149546,0.019636692044332854,0.02017689913041123,0.024099142929292865,0.025355368003168576,0.025456740434258697,0.02526488609859611,0.025845270237598174,0.02628219269614279,0.027417113666109844,0.02800239458017076,0.02807496879207292,0.029432772378234842,0.029521243152584687,0.029342735873377716,0.029838997417952266,0.03058607216803436,0.031183343758575368,0.03203827441767611,0.03756106772469934,0.03966623068190571,0.0392361759903536,0.038684321260382785,0.03935448945312587,0.03995957405110211,0.04072233441205517,0.041451404252665046,0.0412161672889104,0.043563686204978164,0.043309016236423636,0.042766898169265634,0.04337113984014774,0.044377949213036,0.04520864972125848,0.046227001110515704,0.049983159195177145,0.05239857692425104,0.05234737904252367,0.05126956836316933,0.05190517543369591,0.052622662046423636,0.0534172040369668,0.0542745281688863,0.05367204079185855,0.056212059413399545,0.05588250802809644,0.05424837039181799,0.055560669574863275,0.05640733228521011,0.056953800148601935,0.05808126194883115,0.06327974156003578,0.06743824309462075,0.06693475316520725,0.0672923182641686,0.06726225731960297,0.06768404315304567,0.06854288806007222,0.06865744442732985,0.06844500723400265,0.0715127394228125,0.07084333680511351,0.06974523116725197,0.07043009568473205,0.07156199318391901,0.072455711734047,0.0731940804382019,0.07698565447583065,0.0805481924811049,0.08030349405433236,0.0782835835326378,0.0798932183054638,0.08028723210630838,0.08173794874707066,0.08286474693858029,0.08241487627861716,0.08666626773464993,0.08615858089703905,0.08438068201605249,0.08603813200227346,0.08647908171066235,0.08750227739655077,0.08872316377273934,0.0915428299772823,0.09691491512607167,0.09519509450601432,0.09411506501619891,0.09599445190323676,0.09706009156581312,0.0976177796734089,0.10016179991433811,0.09864064404991114,0.10417185295494909,0.10335306245856776,0.1017102857167982,0.10312293992404749,0.10462502353588096,0.1055254194935456,0.106661982395825,0.10898089339300153,0.11605297857749,0.11388610038946033,0.1117029624542117,0.11419465381287482,0.11585596691090538,0.11652547914996626,0.11879252029403106,0.11725814024898454,0.12581174696383085,0.12345308672254676,0.1210681983568218,0.12340291989046852,0.12326619389887415,0.12443451139330594,0.12505606114522005],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=llama-7b-gptq@vllm-gptq branch\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b-gptq@vllm-gptq branch\",\"line\":{\"color\":\"#00cc96\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b-gptq@vllm-gptq branch\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.010094541407193632,0.011049684704368908,0.011669617031649817,0.012527654541948105,0.013256456141973608,0.013731403697822874,0.015515876613459082,0.016801542274723285,0.024280908997398553,0.025981479733312596,0.026301942650062773,0.02652412245345831,0.027369550338524732,0.028462242845175866,0.030273243629318786,0.029189585955785812,0.03626081001350268,0.03822427857651408,0.038617229902291716,0.03846473698363036,0.03924724294494559,0.04037979331198285,0.042121074411623396,0.040667155900941734,0.0479557079035996,0.050186601973858894,0.05040196932153174,0.050286666586021184,0.05098273567164931,0.052024939216310635,0.05388052560403937,0.05274557422436279,0.05910657347833004,0.06213400244695208,0.062203267367605064,0.061350312048006325,0.06260839903454869,0.063755004701556,0.06577416020206789,0.06437758919953557,0.07144602524697168,0.07439628409712665,0.07433149288805672,0.07352493027257624,0.07429014340561393,0.07559487198297425,0.07728595342684097,0.07650323659112057,0.0820079582574049,0.08544982983272599,0.08307496334400047,0.08317302311655825,0.0833621034525156,0.08350133829908517,0.08407573238761767,0.08479498954596038,0.08488768067803455,0.08523463896735375,0.08541414704141335,0.08530989450341579,0.08572114585222469,0.08615028705250871,0.08687531265150916,0.08725300623022174,0.08888659857836732,0.08931082229951583,0.08967493760926863,0.08957915096968366,0.09005869477997269,0.09042471815648592,0.09107592644977885,0.09174723629538621,0.09216584371620092,0.09298206006223803,0.09380067273680748,0.09374324173276705,0.09365351463852409,0.09461717501023865,0.09511995023696261,0.0954118100491544,0.09568157145483328,0.09650075866353114,0.09684716486103667,0.0973561107476594,0.09782800784693775,0.09825865196493123,0.09942725487766359,0.0998780457922121,0.10030327244969134,0.10095072533580388,0.10180480744407826,0.10193881503391142,0.1025978901229719,0.10292544638935128,0.10430284029001077,0.1049056438595713,0.10460929344585077,0.10600029789092363,0.10660676074260221,0.10653806329787409,0.10724297510614386,0.1087813466716127,0.10871006590603963,0.10958659691319983,0.10988863522065806,0.11273441807415273,0.11361863095158155,0.11352306493981933,0.11412765189794336,0.11483691095025583,0.11594504018794648,0.11707218210449938,0.11743371929804763,0.11928749115121202,0.11912270184155453,0.11978593360468347,0.12039018688106107,0.12103726291721105,0.12155846537877554,0.12295238954444744,0.12241979158368739,0.12436584439965534,0.12250129874978716,0.1239056527965897,0.12461948392971156,0.1253775541290335,0.12556658145988486,0.12643535392722835],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=llama-7b@tgi-1.3.3\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b@tgi-1.3.3\",\"line\":{\"color\":\"#ab63fa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b@tgi-1.3.3\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.030305246997971106,0.03253334405670573,0.032889617089572123,0.03320942346806995,0.03371695439813467,0.034028316981785205,0.03447434066793664,0.03492873368225945,0.035294029233153626,0.03585953612230959,0.036194619667627874,0.03655305328173916,0.03681373007887452,0.037175870828735746,0.03759796399110306,0.038023545778551146,0.03967216197328627,0.040208880225012164,0.04057111312070854,0.04067760940193478,0.04090378694960594,0.04110037339939331,0.041390334940382405,0.04160457596233253,0.04170445783288237,0.0421126174004433,0.04227074279995647,0.04232680646707437,0.04215346516752022,0.042452557713419874,0.042718797754312005,0.043016277658976115,0.04314723611501743,0.04365179230946253,0.04382054469010575,0.04399890148397709,0.044334716103866544,0.044396973172566814,0.046039020407037404,0.04767788959620896,0.04909123584316214,0.051021117929983435,0.05259908649033458,0.05378689687957696,0.05548317109520334,0.05699158074454161,0.05855478295687231,0.060449081315498546,0.06163252062863862,0.0634328282940663,0.06489905115531136,0.06626292742965267,0.06785837307235236,0.0695200481234212,0.07081512481007902,0.07252821618433782,0.07372405014801416,0.07571175742094126,0.07695313186429191,0.07847125246722324,0.08010217200599831,0.08154505499759378,0.0831250822891177,0.08487853801212523,0.08616165670931286,0.08809411181781292,0.0903930695519409,0.09161968274093933,0.09350789517025555,0.09493763197849042,0.09661439198592008,0.09848383215981223,0.09989139578136313,0.10233072691493515,0.10390053827600053,0.10509427282147656,0.10692895278575035,0.10841656244616861,0.1098648026604388,0.11178263631064059,0.11297130561127496,0.11520332385847748,0.1167012804113403,0.1178289020687792,0.11950531749491147,0.1211112675809322,0.1226055189723968,0.12441938951507618,0.125643152753316,0.12790616769873572,0.1292757246269154,0.13041907657507812,0.1319798358218996,0.13354796500278268,0.13518184436490263,0.1371500382681846,0.13812113319963354,0.14042799107097895,0.14181661087500014,0.1429637857660071,0.14459832658427096,0.14621730641448127,0.14774003828850726,0.14947600656405505,0.15062273298688694,0.1530775651221675,0.15444085278718697,0.15547675582600914,0.15704992153864258,0.15861709395406012,0.1601501408282824,0.16188451463292203,0.16309698734306513,0.16545161072325307,0.16676308864106326,0.167825451497923,0.1695012614936937,0.1709411659680543,0.17246242193980652,0.17437293546818064,0.1753249285964443,0.17746482036435587,0.17911259683645875,0.17982387166418415,0.1817500236732532,0.18308286735776058,0.18464697996734142,0.18647258516995074],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=llama-7b-eetq@tgi-1.3.3\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b-eetq@tgi-1.3.3\",\"line\":{\"color\":\"#FFA15A\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b-eetq@tgi-1.3.3\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.01797777643391615,0.018697935631158274,0.019009534588617937,0.01925713715584346,0.019712125942618473,0.020115998279544318,0.020570936734286265,0.020910998519349494,0.021235132986600423,0.021778782228502328,0.022051343028538745,0.022380669778852308,0.022809555117559013,0.023181700525616473,0.023570517218257637,0.024019501607351793,0.024214363599797374,0.02472138320211746,0.02500125580804914,0.025229615519468902,0.025695467731551628,0.02606446715847009,0.026586123891947616,0.026966830629310597,0.027338735789682062,0.027962333967727175,0.028333372308238833,0.028509210274583526,0.028919853548998502,0.029219187420778978,0.029604169987771434,0.032244055597911776,0.0327125129997791,0.033372244314782064,0.03303424069386682,0.032633235173537466,0.03286801521155636,0.03280415308145471,0.03289615886000039,0.033033614674743375,0.03281582429160685,0.033446229617185304,0.03305578777710124,0.032643824992159724,0.032807727547025274,0.032829413871710345,0.032977325853075466,0.033006790559188114,0.032847721729423504,0.03349223884037201,0.0332009964998709,0.032683017438128556,0.032843254061822674,0.032937142920437104,0.033085051215706984,0.03309502350409307,0.032898046976373434,0.03350129126621213,0.03305817009128601,0.03263592244576581,0.03274628685368834,0.032837103531567796,0.03284651597768801,0.03301480867078565,0.03267129163866228,0.033291789319065526,0.03362607941752378,0.033890183733182346,0.03466393380559619,0.035372799679027435,0.036131371056840035,0.03690181622403523,0.03724278409614721,0.03888642815231324,0.03903211224111707,0.03893128077366963,0.04000847636432477,0.04067958649101947,0.04139971060722305,0.042151736818920864,0.04245794082006129,0.04436261150439231,0.04436402784187061,0.04424693343065559,0.04532193989522181,0.04600940109982293,0.04697955586282799,0.047751044823568754,0.04807485550880312,0.05006375601662748,0.0502013395351616,0.049869604225843196,0.05097866494110913,0.051581757907308574,0.05250593345538061,0.053251227067775735,0.05353798160637578,0.05505290673935786,0.05541262285028482,0.055365505143617925,0.05616834175688442,0.0569392433731427,0.05775303321677622,0.0586455719675611,0.05899994179654836,0.06024156728997324,0.060340173220394554,0.060480993732202025,0.061262232652960644,0.06217905788234786,0.0628870206831491,0.06352111976653659,0.0639148112508788,0.06586583336202993,0.06565604370289409,0.06533686713157194,0.06630940860630605,0.06713530390982343,0.0678862652076297,0.06904584415028515,0.06879935696332887,0.07162956430061009,0.07141127556282305,0.07074661842325904,0.07128620510040584,0.0726637912620147,0.0732752854356206,0.07454642683724537],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=llama-7b-awq@tgi-1.3.3\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b-awq@tgi-1.3.3\",\"line\":{\"color\":\"#19d3f3\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b-awq@tgi-1.3.3\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.01311240498478183,0.013997631243698612,0.014304282149117479,0.014617797310830568,0.015151251855075315,0.01569423599752383,0.016211829332145225,0.016749457373705066,0.017227408964724755,0.018048556143023992,0.01837754821096957,0.018710168822613702,0.0192987819697403,0.01987774702271479,0.02042364338474772,0.02100243098895319,0.024881237871315697,0.02619360604951701,0.02637773248795817,0.026142380918585385,0.026844601205562493,0.027576034897942876,0.028288216759857036,0.028960002649576305,0.029032343232027923,0.03036448408392617,0.03047127389313726,0.030319916190425164,0.031083992569387656,0.03173408353125599,0.03237159531306428,0.03337122783824719,0.038562508921803115,0.04059102829382007,0.04055030871547938,0.039716558476849406,0.04057048984948896,0.04114182913820231,0.04143234612812553,0.04214312271491515,0.04123750257124276,0.04335558260493716,0.04293253531289263,0.04174590588076746,0.042258319878819874,0.04260953156499781,0.04221213330827393,0.04364237846571136,0.04280416785766947,0.045145954926419596,0.043816402720754044,0.04312405245591702,0.043609140958538295,0.04389258457343728,0.04413386099568148,0.044531400423816696,0.04351707105762637,0.04607390678469927,0.04549308360883791,0.04344336943827549,0.04435521707931802,0.04474899211795356,0.04510503371746213,0.04614688018185456,0.044946911178184536,0.04796592917580267,0.047209773114461354,0.04523017914445152,0.045806444015685996,0.046259674410130056,0.04694598875521257,0.04805863870429548,0.04654678293422847,0.050540364089854,0.04940064720365999,0.047172790624576896,0.04834852455139916,0.04952765301704636,0.05059374995596863,0.05235411652242136,0.05125667630939777,0.05577691705564459,0.05544715972196933,0.05336389284665653,0.05484764021216713,0.055857997825717064,0.05694525241484283,0.05878027320070365,0.057582160216582065,0.06229088829166027,0.06183860307768764,0.05959013972702727,0.06087827142191823,0.062039419287798006,0.06322503805515743,0.06503222115054377,0.06348118857938115,0.06904204657819966,0.0680192852280146,0.0655959270562393,0.06698855023497884,0.0679753406613774,0.06914795627799496,0.0709289528698635,0.06964499053354889,0.07487200402973886,0.07413910199963016,0.07143100927542566,0.07298391149061755,0.07393978503317884,0.07498285466855906,0.07697604052223801,0.07542352567908363,0.08119086856506813,0.08007534660698741,0.07720341561549822,0.07796063342363636,0.07939266120972792,0.08121910977506927,0.08281787117365753,0.08154541313086514,0.0879217934092356,0.08634090338300841,0.08361600125854723,0.08512898111906109,0.08602390934213361,0.08733408778381818,0.08962615275581338],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=llama-7b-gptq@tgi-1.3.3\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b-gptq@tgi-1.3.3\",\"line\":{\"color\":\"#FF6692\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b-gptq@tgi-1.3.3\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.01033642817299659,0.011467812645601982,0.01216348892307124,0.012938801651736762,0.01371040416386347,0.014036010707822325,0.015892558405480633,0.017200275898821074,0.02473053627208075,0.02642786063829629,0.026814037919764135,0.027320809018762018,0.02806652534835165,0.02887925894510479,0.030758657176439854,0.02961642730836032,0.03678246650097187,0.03881029601390453,0.03900561386036849,0.03909099929065962,0.03980709043876281,0.04088535724630305,0.04302048314921472,0.041494453696217196,0.04844769051563202,0.05089119238895739,0.05098368610871269,0.05068871373090826,0.05130967167241569,0.052540475659962685,0.05450919785327489,0.0532552875208247,0.059992325887984616,0.06309985272963026,0.06311047592860929,0.062280337965092435,0.06331401685951502,0.0645735051023757,0.06671640106337706,0.06562111871024227,0.07202484649388287,0.07486496400312019,0.0750411940057821,0.07424263978305276,0.07530879661187231,0.07621382846156399,0.0776232103206542,0.07911964328994321,0.07878796842392143,0.0815155468117436,0.08195750472233647,0.08076758755381154,0.08193623378949716,0.08323288238702704,0.08453980601782066,0.08584187726652034,0.08550411201241345,0.08867574445472784,0.088594350554943,0.08752760984083728,0.08825826108945595,0.08957012806477087,0.09092132514990434,0.09246045908417366,0.09174569023153031,0.09546269531819071,0.09537963386036254,0.09400272346400428,0.09528569991289419,0.09652886270891488,0.09806848137297665,0.09978441329883728,0.09924015970293126,0.10308637176902098,0.10314005659302575,0.10174266281425479,0.1030394136054886,0.10453539467655147,0.105820085687142,0.10753752401396546,0.10668715739986209,0.11125041948645586,0.1115409056822239,0.11027636236959167,0.11163883416072869,0.11367879562023261,0.11532920824418579,0.11747325247598754,0.11689858279474939,0.12172445398637081,0.1216301658279978,0.12020511482009141,0.12178508736340227,0.1236934308453182,0.1254847610654423,0.12774529344393348,0.12725902480203624,0.1320780015573633,0.1317717848306592,0.13001075972666828,0.13170376378054216,0.13356457240987707,0.13565752954610283,0.13786563273433697,0.13721130897126582,0.142260087558391,0.1418688175540624,0.13955066421771223,0.14140641659498254,0.14360100969740208,0.14543303318522843,0.14781203005154503,0.1469389408188092,0.1524650999536381,0.15130661299531517,0.14946590574272414,0.15127262129482202,0.15277653713244133,0.15532773233133082,0.15757293981977807,0.1563622383755623,0.1623868286051068,0.16180990838531562,0.15893379894979856,0.16088446514016114,0.16298682224682098,0.16487461541194215,0.16737590481704265],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"batch\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"avg_token_latency\"}},\"legend\":{\"title\":{\"text\":\"type\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Llama Average none-first Token Latency\"}},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('01e79e18-fc83-47c2-878b-0bf1aadef388');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                });            </script>        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig_llama_avg_latency = px.line(llamas, x='batch', y='avg_token_latency', color='type', title='Llama Average none-first Token Latency')\n",
    "fig_llama_avg_latency.write_image('images/llama2-7b-avglat.png', scale=2,width=1440)\n",
    "fig_llama_avg_latency.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "hovertemplate": "type=llama-7b@vllm-0.2.5<br>batch=%{x}<br>avg_first_token_latency=%{y}<extra></extra>",
         "legendgroup": "llama-7b@vllm-0.2.5",
         "line": {
          "color": "#636efa",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b@vllm-0.2.5",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.058835917399983374,
          0.03813487970001006,
          0.04515944600000997,
          0.06457146570000986,
          0.06881393540000318,
          0.07118065423332307,
          0.08099228248571698,
          0.07043429429999719,
          0.0890274482888799,
          0.0824064050599918,
          0.08683974941817393,
          0.09100914358331996,
          0.09623095466071975,
          0.152705304482138,
          0.10651397461667406,
          0.11617754334375263,
          0.12281072564705736,
          0.12561062927776825,
          0.1522523864342253,
          0.1239577232125015,
          0.15813189348810802,
          0.13228031764772005,
          0.1441537042826044,
          0.14865968271874172,
          0.14663487336000344,
          0.15628160583652026,
          0.16105728835185573,
          0.19232931160713815,
          0.16459665726724856,
          0.22584658345834327,
          0.27790370029032374,
          0.341290981007802,
          0.3673320560538601,
          0.45375632925000636,
          0.488757730985078,
          0.415489102399999,
          0.4827771898087042,
          0.5134150172881237,
          0.5374975214871849,
          0.572639630733344,
          0.5941705762032502,
          0.6185319853174666,
          0.6244504822170517,
          0.6159056330227227,
          0.6480805961111181,
          0.653802473304345,
          0.681398825421419,
          0.6927900375774612,
          0.7181051535857201,
          0.7412455269580253,
          0.7713191131408461,
          0.7281089725101992,
          0.7516023347397444,
          0.7733061755850285,
          0.8127115837919564,
          0.822599544543618,
          0.7855772629483938,
          0.8952047110128208,
          0.838701966044598,
          0.8265143136025688,
          0.9165684656815255,
          0.8251735096687483,
          0.8624744839625009,
          0.8868440780188608,
          0.8954305323687549,
          1.0859695029000036,
          1.1480340123375057,
          1.2719484162784949,
          1.7461908953170717,
          1.7227909286335454,
          1.7219322154250059,
          1.6400776847405054,
          1.5058526064187405,
          1.5231446459750004,
          1.9264487348553447,
          2.1793019840445838,
          2.4209151446898765,
          2.8207020632173947,
          3.2124879346455675,
          3.2671099679374933,
          3.2732624823024734,
          3.3913553901829303,
          3.4870632870060203,
          3.709676402880946,
          3.64121875749411,
          3.653081999319778,
          3.6814276891149498,
          3.7334131204602374,
          3.7840215137921365,
          3.8350843456388892,
          3.8378120671538407,
          3.866392940157599,
          3.9421263278387055,
          3.868324142170219,
          3.9703712516578853,
          3.934384187234048,
          3.968551261788665,
          3.885136304719378,
          4.183554407803099,
          4.1764065050855566,
          4.061296817380004,
          4.198322738373694,
          4.358730496722504,
          4.192881267790579,
          4.337823682510419,
          4.3782982340989705,
          4.2556043054690615,
          4.011422309985512,
          4.193590529750001,
          4.393772548595955,
          4.277771191616502,
          4.47920499473501,
          4.752323044494999,
          4.521386473328576,
          4.797423604489889,
          4.733539008777219,
          4.623601991682927,
          4.736413511882918,
          4.717419656610832,
          4.661943101281553,
          4.806857630879812,
          5.165636263864075,
          5.176727761630329,
          5.378880631051402,
          5.416820673071096,
          5.720703710712921,
          5.869616206355135,
          6.2781982552844
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=llama-7b-awq@vllm-0.2.5<br>batch=%{x}<br>avg_first_token_latency=%{y}<extra></extra>",
         "legendgroup": "llama-7b-awq@vllm-0.2.5",
         "line": {
          "color": "#EF553B",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b-awq@vllm-0.2.5",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.019464123916729175,
          0.028330771818197965,
          0.03693724678784995,
          0.04627445359092648,
          0.059786729919942445,
          0.07112013513333296,
          0.08328931511429931,
          0.09456613612499293,
          0.11634796440740154,
          0.13000515760001893,
          0.1321420646464788,
          0.14450953397916066,
          0.1725918869422965,
          0.17033594299999372,
          0.18118380924167543,
          0.18339910298374623,
          0.1943392884608124,
          0.21407508840739967,
          0.2202303563596312,
          0.22888459619166496,
          0.24085585133334894,
          0.25036819127269677,
          0.26568878051448075,
          0.253648733316564,
          0.2732106541142587,
          0.3159157242384446,
          0.2975587275777966,
          0.30700427774286904,
          0.32411109178622705,
          0.3387297986933269,
          0.34693886986451616,
          0.34459731340624844,
          0.3499319261742495,
          0.38714591216176375,
          0.4205047729214162,
          0.3936521469305553,
          0.4055380039999732,
          0.40647260206579977,
          0.422014811955128,
          0.4251858914500076,
          0.48043195120124815,
          0.46664048510117917,
          0.5336639283198005,
          0.4714959863466003,
          0.4901327111722139,
          0.489886999184794,
          0.5182116830552251,
          0.46393970722619404,
          0.4952953167687069,
          0.5678290082066633,
          0.5666209950784185,
          0.550370814314093,
          0.5816298372578785,
          0.5548178023641945,
          0.589810875103015,
          0.5753721321905151,
          0.6351488054386166,
          0.6788872733390805,
          0.6172264209434986,
          0.7087209888111071,
          0.6636188552896003,
          0.6508907918010806,
          0.6448743967671964,
          0.8153880382448051,
          0.9482114074205198,
          0.891104534095245,
          1.0206405475172184,
          1.040138912496982,
          1.1160943871597722,
          1.2871025774730658,
          1.4194971100687552,
          1.4819661633889076,
          1.5718770252909247,
          1.8916952020926965,
          1.8054935044203613,
          1.7883764543703644,
          2.064612102318187,
          1.9937426055062588,
          2.1566053665785963,
          2.2553405110749707,
          2.3625584340185033,
          2.4491004099268445,
          2.3960533000662707,
          2.47636313065475,
          2.566872421517653,
          2.6890977974709225,
          2.7497805594655085,
          2.643020788232948,
          2.690156817691017,
          2.9047642076388884,
          2.779194836021986,
          2.8290884049891187,
          2.9729399760430026,
          2.896174527563826,
          3.1618803087105287,
          2.9303740617604555,
          3.175039309649494,
          3.36584153876166,
          3.0679363714040497,
          3.1031345803950057,
          3.344709210731339,
          3.2751552187990134,
          3.21513302886408,
          3.2041069732692313,
          3.4740757647999976,
          4.217111779180344,
          3.8839343641170334,
          3.6799189250151434,
          3.688991141402829,
          3.5027921130744266,
          4.039075035059705,
          3.9080685277253906,
          3.983902567034003,
          4.714611416437826,
          4.4505953837150605,
          4.450314861409558,
          4.159244008787882,
          4.02175758427672,
          4.74125285589005,
          4.180763937268301,
          4.218288550103947,
          4.8887274778912015,
          4.418879322849554,
          4.581166031570699,
          4.427966952589612,
          4.765414865793826,
          4.771548837228849,
          4.956309427125618
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=llama-7b-gptq@vllm-gptq branch<br>batch=%{x}<br>avg_first_token_latency=%{y}<extra></extra>",
         "legendgroup": "llama-7b-gptq@vllm-gptq branch",
         "line": {
          "color": "#00cc96",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b-gptq@vllm-gptq branch",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.03033113406660656,
          0.06207753339292919,
          0.06982457792310015,
          0.0838863204583807,
          0.10193900560002063,
          0.09646472580305437,
          0.10086689234288185,
          0.1051645858333485,
          0.10635184594437103,
          0.1126733033833716,
          0.12185998307580975,
          0.14919825562492406,
          0.11878493038460501,
          0.12804610610717082,
          0.13273268450666365,
          0.1482101496750147,
          0.19159968311432748,
          0.14802018369443382,
          0.15265084930264924,
          0.15277254097512696,
          0.15598780911907506,
          0.15517431880678859,
          0.18074459977169285,
          0.18093190095832293,
          0.17059935942855495,
          0.18471757888459578,
          0.18587822966668524,
          0.1798704794881533,
          0.18689683273559,
          0.20536622150003597,
          0.19865979366664194,
          0.21980410422919286,
          0.2243921863939727,
          0.22050326901957948,
          0.2140040896571788,
          0.2545180845370312,
          0.22161892262158436,
          0.22814517580704968,
          0.24540159539318643,
          0.25598602478328303,
          0.21099661464284844,
          0.2618159637208813,
          0.24123156549444758,
          0.2505020799246991,
          0.3067259223042959,
          0.25957591906523664,
          0.2697872684467605,
          0.26001312603127263,
          0.2700944668673791,
          0.2797883386300327,
          0.3858796549215649,
          0.2913157372980689,
          0.3012074966037495,
          0.40310163108330455,
          0.4706759400727027,
          0.2999617325892895,
          0.2976983465438073,
          0.3114546721982653,
          0.31719916992374314,
          0.30958710332492956,
          0.31064649877049066,
          0.3380605859919665,
          0.3186679736349366,
          0.446676342046878,
          0.31711636424624184,
          0.3798192396287835,
          0.4544527462313766,
          0.3702591288309002,
          0.4760703501956545,
          0.3686936720999971,
          0.3835167126760328,
          0.4546172254305879,
          0.43561444306850383,
          0.5669019691824446,
          0.531294949306648,
          0.5695715193223326,
          0.7635881872922208,
          0.6312568748974405,
          0.7375402287594924,
          0.705871948093727,
          0.8194668204876642,
          0.8052669690487386,
          0.8844556235602609,
          0.8288524332857227,
          0.8482122715234897,
          0.9070355701860376,
          0.9219668872758915,
          0.9568068841477281,
          0.9283857481460841,
          1.0009682624000107,
          1.0083008243681193,
          1.034175129516311,
          1.0697264405107783,
          1.181880144898964,
          1.137311023789478,
          1.1115188464530898,
          1.1991610036340352,
          1.1737354773112323,
          1.2497804882615478,
          1.141559642249972,
          1.2062558439802116,
          1.3066040453186138,
          1.4648528823505171,
          1.432660791989894,
          1.2432317569808777,
          1.3871779338640946,
          1.3125690560094103,
          1.3818813879814797,
          1.4031211577617013,
          1.4436779294540776,
          1.5517083671633516,
          1.4471918497416334,
          1.510594530505032,
          1.5457598121911806,
          1.6743124402785712,
          1.5166455123414357,
          1.5303918012259776,
          1.5588890721924893,
          1.7223953456250092,
          1.7952037906896634,
          1.7353376118557235,
          1.7423281113588671,
          1.8742245357733616,
          1.6870486412559347,
          1.6602260465740655,
          1.9058621540426555,
          1.976712762328481,
          2.0464063935687244
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=llama-7b@tgi-1.3.3<br>batch=%{x}<br>avg_first_token_latency=%{y}<extra></extra>",
         "legendgroup": "llama-7b@tgi-1.3.3",
         "line": {
          "color": "#ab63fa",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b@tgi-1.3.3",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.04407292479954776,
          0.05470827120043396,
          0.06257032880009016,
          0.06811862604981797,
          0.11653407976009475,
          0.07359979403336184,
          0.07349380674277199,
          0.08409491802503907,
          0.08153858559983039,
          0.08520270868011721,
          0.08632459093889933,
          0.0884770547916105,
          0.09696337659614493,
          0.09868043123229395,
          0.09926669340005295,
          0.12983052576555565,
          0.11593354201477268,
          0.12755196479174125,
          0.136834466328908,
          0.8164373614156345,
          1.4928739701026643,
          2.1512007133925075,
          2.8345868182125287,
          3.406048651604848,
          4.035935578183055,
          4.6351933673613965,
          5.18865464594046,
          5.744911006870483,
          6.478709147802346,
          6.98066082629874,
          7.482366778681746,
          7.978228435988691,
          8.471962633133343,
          8.961880774615365,
          9.492249726184859,
          9.912728271634576,
          10.377665958000065,
          10.851012567621058,
          10.883936533864585,
          10.907138072629001,
          10.96399848359189,
          10.965737889999971,
          11.002743193159967,
          11.073567599435691,
          11.093728042804,
          11.12856233435932,
          11.162968942567378,
          11.19032814374285,
          11.277254963801925,
          11.221624761990638,
          11.257176950194493,
          11.294776085724786,
          11.36142141134541,
          11.370966202261231,
          11.381987374749965,
          11.38345603034506,
          11.442025829727946,
          11.432685653408724,
          11.435705473594775,
          11.51162066064964,
          11.537125310559238,
          11.62844769331938,
          11.603579177141743,
          11.659110285975139,
          11.671992045442524,
          11.688110751561005,
          11.81582378489507,
          11.869477431160034,
          11.883689162523787,
          11.918741617440872,
          12.001438062109315,
          11.976989582372111,
          12.058571849299998,
          12.010571513061056,
          12.04217391009093,
          12.097277748804377,
          12.110829323828362,
          12.13194912360733,
          12.175274333999877,
          12.162064767620373,
          12.202953590666686,
          12.198817338582748,
          12.219784705028633,
          12.264005348148977,
          12.281325994845107,
          12.300012366307605,
          12.385123480979233,
          12.333416063855262,
          12.355819745828864,
          12.355298432877598,
          12.469634566141924,
          12.413864086966475,
          12.439987469166638,
          12.442634439682136,
          12.468847023999974,
          12.474485954176535,
          12.513278706077779,
          12.505603821438713,
          12.517937937884676,
          12.54444215700003,
          12.565005109373441,
          12.574759089559704,
          12.600919286506302,
          12.652155808329214,
          12.7054472221975,
          12.624042045447794,
          12.703551231817173,
          12.673320153890833,
          12.773281533885596,
          12.693928463365273,
          12.775351845369025,
          12.719493878485157,
          12.741323957305864,
          12.74005491657307,
          12.75719246847666,
          12.78544165734103,
          12.799343318143666,
          12.807805386234229,
          12.820864410818258,
          12.821592045627078,
          12.849510265612428,
          12.93032434340237,
          12.861306621350003,
          12.942402063679564,
          12.898963769631916,
          12.91597005101085,
          12.926452899733773,
          13.062753518308199
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=llama-7b-eetq@tgi-1.3.3<br>batch=%{x}<br>avg_first_token_latency=%{y}<extra></extra>",
         "legendgroup": "llama-7b-eetq@tgi-1.3.3",
         "line": {
          "color": "#FFA15A",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b-eetq@tgi-1.3.3",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.04195532600008139,
          0.04254945243746988,
          0.05650931116671624,
          0.04416613231262545,
          0.04446792274984546,
          0.045761248958304655,
          0.047535240571245856,
          0.048650933749838386,
          0.05967702061894266,
          0.0660119337857915,
          0.06826878377921877,
          0.0713218192858048,
          0.07387221402214081,
          0.0779358137754812,
          0.07748955990474704,
          0.09074265947521493,
          0.09223604942149798,
          0.10102845392588346,
          0.1079215742369534,
          0.10665599425013474,
          0.11110473346822786,
          0.1109817197120306,
          0.13078182467405355,
          0.1315747856667334,
          0.12871347822664878,
          0.1498985421409928,
          0.14519692479359125,
          0.1272877796234209,
          0.13547604280948586,
          0.15205593488009375,
          0.15661862961286743,
          0.15754463772505006,
          0.15705300381212675,
          0.5015873754818821,
          0.8476375831018188,
          1.1842825211488703,
          1.5222974410709178,
          1.8677363366058939,
          2.173924055157915,
          2.5067931978489235,
          2.8842886830116607,
          3.156955518879401,
          3.4576053394743003,
          3.7626934473068085,
          4.079543562355923,
          4.363221869466379,
          4.6631956965642605,
          4.96954243319446,
          5.30089016375144,
          5.556026343054967,
          5.851011167098328,
          6.12337925190206,
          6.475195553978463,
          6.740263683758067,
          7.072209473010586,
          7.3314037929308435,
          7.584193476756544,
          7.856519102815755,
          8.12732469565968,
          8.335573938729235,
          8.579746788010372,
          8.861201173994878,
          9.06301219114873,
          9.330708878086783,
          9.520459296446715,
          9.761912501257545,
          9.799311132929674,
          9.838512127405028,
          9.879591796477529,
          9.920020694613935,
          9.965521858768481,
          10.003572125093047,
          10.048340429517134,
          10.090361270232995,
          10.114712929913024,
          10.207170414028916,
          10.203665471588483,
          10.253155743690426,
          10.334472217255959,
          10.46026015839152,
          10.54928120150232,
          10.543071700686902,
          10.599877026697733,
          10.760032099106494,
          10.720105740165963,
          10.745065442476992,
          10.77890575071237,
          10.81032954828178,
          10.897133932185508,
          10.95733154236231,
          10.882286793865523,
          10.962722641017917,
          10.98549977205781,
          11.044993395916007,
          11.103318806057796,
          11.292801597755544,
          11.140537412043892,
          12.040073826545909,
          11.615995015774246,
          11.328011293842042,
          11.714465741698634,
          11.436051271753255,
          11.598740289618823,
          11.676861534423395,
          11.594741504951312,
          12.216426168556591,
          11.996908165917052,
          11.554266906931035,
          11.885702124315616,
          11.789990765687214,
          12.000335070008898,
          11.919042254672556,
          11.912013277293857,
          11.900957441486792,
          11.908260097013061,
          11.96668637411644,
          11.86847904789316,
          11.84353700536028,
          11.775668676903479,
          11.825582682716496,
          11.735781077628094,
          11.708772235483542,
          11.661068357016246,
          11.802864029423414,
          11.686575693871994,
          11.679420248150745,
          11.53287206485828,
          11.603571357644505
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=llama-7b-awq@tgi-1.3.3<br>batch=%{x}<br>avg_first_token_latency=%{y}<extra></extra>",
         "legendgroup": "llama-7b-awq@tgi-1.3.3",
         "line": {
          "color": "#19d3f3",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b-awq@tgi-1.3.3",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.04167407283330249,
          0.042591082500207864,
          0.04477750903016854,
          0.04932447440000942,
          0.05772703906004608,
          0.06766981648330936,
          0.0770387442538026,
          0.08609929231935591,
          0.0972506076913204,
          0.11816277039999931,
          0.12519203342057236,
          0.13497684956246303,
          0.14634732991334415,
          0.1549234398303432,
          0.1638005227963125,
          0.17538668489290948,
          0.18278952494129758,
          0.2070190628888095,
          0.2118691581052732,
          0.22227182286672661,
          0.2322813534285218,
          0.23496497546210032,
          0.24337103313498826,
          0.257829682566747,
          0.2678293697760237,
          0.27822394055386285,
          0.299314463392619,
          0.3087140547285344,
          0.32477370649660237,
          0.32309788942654755,
          0.33896654114196867,
          0.3416943186124513,
          0.3504995631439918,
          0.38589157088238285,
          0.38631336530003996,
          0.3918650322500146,
          0.4076163266352376,
          0.40687481901312295,
          0.7289948362483651,
          1.0314769887986763,
          1.472807826793618,
          1.6310434585448559,
          1.9386050649489706,
          2.2760052141582237,
          2.5601090482452045,
          2.8400936312874707,
          3.343633699316749,
          3.378957173302398,
          3.7326543041350653,
          3.8926165191402418,
          4.414611761460589,
          4.5807841502590305,
          4.817354221574887,
          5.056266762702318,
          5.276813226603547,
          5.69058274025878,
          6.022462056906419,
          6.064718694970864,
          6.358105394479777,
          6.79637615263215,
          7.0221695533771396,
          7.160414340369326,
          7.37230610224868,
          7.499651335247231,
          7.85834902584363,
          7.814700718244381,
          8.044396767093868,
          8.541854997291132,
          8.707838730776007,
          8.888988993092298,
          9.04733661790809,
          9.19007026505375,
          9.631151856780729,
          9.472103518414896,
          9.895905707550343,
          10.432610823573764,
          10.435027694753872,
          10.370433804828025,
          10.389854844367802,
          10.353668010422599,
          10.517032410138377,
          10.143589935040781,
          10.285252998700441,
          10.59200893408076,
          10.547018286070344,
          10.528985041619917,
          10.517980650189184,
          10.481091602702978,
          10.640172906571454,
          10.282045543426426,
          10.417520848321972,
          10.69836583072334,
          10.663731494280203,
          10.679535952418345,
          10.687227260574192,
          10.539764325561945,
          10.731102861393385,
          10.373099762103823,
          10.524307497755789,
          10.767045549415828,
          10.810908645832498,
          10.750100794361044,
          10.730718721207381,
          10.655284177623857,
          10.829327316191753,
          10.712911861887267,
          10.621187929013507,
          10.884934230518132,
          10.837345795470874,
          10.827202202763406,
          10.818893811377809,
          10.810845706495545,
          10.960099607017522,
          10.592427655627073,
          10.72832300790833,
          10.9909923784043,
          11.044779082342005,
          10.98629177504305,
          10.930923615781056,
          10.908684680132453,
          11.103935925455362,
          10.735969538499962,
          10.880790819109713,
          11.111418724949583,
          11.078130764410025,
          11.16271798024577,
          11.075021787165946,
          11.059591335975178
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=llama-7b-gptq@tgi-1.3.3<br>batch=%{x}<br>avg_first_token_latency=%{y}<extra></extra>",
         "legendgroup": "llama-7b-gptq@tgi-1.3.3",
         "line": {
          "color": "#FF6692",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "llama-7b-gptq@tgi-1.3.3",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.04235638053323783,
          0.05196514376900897,
          0.05959054219445332,
          0.08475871820830132,
          0.09071134810900813,
          0.09391383727278012,
          0.0980654750569946,
          0.0991997789860256,
          0.10502925892601693,
          0.11064605858330955,
          0.11212593371210185,
          0.13757560305566585,
          0.11793161662822407,
          0.15207485713883923,
          0.14253976933323428,
          0.13944946157498633,
          0.13992329779410498,
          0.149155963666519,
          0.1478958764869276,
          0.14696731018761966,
          0.1502504607975598,
          0.15216450167054063,
          0.1709434182065034,
          0.18223832086469352,
          0.16846848335439915,
          0.18094895265370817,
          0.23211053339501733,
          0.17910095065480208,
          0.2898938625056317,
          0.20516655851119545,
          0.20665619615060532,
          0.21103578161451728,
          0.21730810060593414,
          0.21803247086261443,
          0.22113296119040612,
          0.21438940023147182,
          0.2702196344054411,
          0.23806973451742683,
          0.23650177041884327,
          0.2375516142666432,
          0.22271895835887257,
          0.5864835732470323,
          0.9205753148736671,
          1.1768119778404031,
          1.523432659774151,
          1.9526000848695335,
          2.134032753574502,
          2.4216787854105575,
          2.647799191683849,
          3.1440102760000412,
          3.291140254402092,
          3.358395106692342,
          3.5815885297736214,
          3.7837436485925973,
          4.023810513229372,
          4.543630511155216,
          4.452949544517829,
          5.224686397356575,
          5.311119544647625,
          5.159295695964925,
          5.195988781598242,
          5.428547340909748,
          6.0080356190870665,
          6.319482453859692,
          6.047582318531669,
          7.032426112594581,
          7.096748061382586,
          6.332804685255662,
          6.607741289931779,
          7.332223546177433,
          7.6551439338759675,
          7.83819413443437,
          7.634738453441969,
          8.736742745439688,
          8.829271024177924,
          7.844687020926889,
          8.350562544409003,
          9.027645733808065,
          9.260804059543982,
          9.566300072612927,
          9.345730788776928,
          10.11095063170724,
          10.064577329376043,
          9.796274658192376,
          10.134904420806246,
          10.10551889406972,
          10.219976139100734,
          10.336106522317808,
          10.181441802714364,
          10.409727341580073,
          10.444037897947027,
          10.328691309881536,
          10.515938628701447,
          10.546010386607398,
          10.654122265897032,
          10.632333121270085,
          10.645308788840735,
          10.680012654122326,
          10.71105915865003,
          10.595744793000023,
          10.769905939119704,
          10.801911061937126,
          10.831197600722172,
          10.862304466944787,
          10.88870651356151,
          10.91959696400682,
          10.946595939033823,
          11.137420606704628,
          11.003163454013253,
          11.02892259699341,
          11.114864500322435,
          11.089134258372617,
          11.10759598922722,
          11.134550681451572,
          11.225628659185892,
          11.18536804795536,
          11.208409700645564,
          11.325786599635174,
          11.27975534629378,
          11.349296206111763,
          11.405967680598728,
          11.332319571128973,
          11.353550492689054,
          11.422628582957678,
          11.397339948277038,
          11.41991995346702,
          11.557859763154804,
          11.46422856693491
         ],
         "yaxis": "y"
        }
       ],
       "layout": {
        "legend": {
         "title": {
          "text": "type"
         },
         "tracegroupgap": 0
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "Llama Average First Token Latency"
        },
        "xaxis": {
         "anchor": "y",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "batch"
         }
        },
        "yaxis": {
         "anchor": "x",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "avg_first_token_latency"
         }
        }
       }
      },
      "text/html": [
       "<div>                            <div id=\"cdbe155d-c0dd-4d2f-8554-71d77df248b8\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"cdbe155d-c0dd-4d2f-8554-71d77df248b8\")) {                    Plotly.newPlot(                        \"cdbe155d-c0dd-4d2f-8554-71d77df248b8\",                        [{\"hovertemplate\":\"type=llama-7b@vllm-0.2.5\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_first_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b@vllm-0.2.5\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b@vllm-0.2.5\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.058835917399983374,0.03813487970001006,0.04515944600000997,0.06457146570000986,0.06881393540000318,0.07118065423332307,0.08099228248571698,0.07043429429999719,0.0890274482888799,0.0824064050599918,0.08683974941817393,0.09100914358331996,0.09623095466071975,0.152705304482138,0.10651397461667406,0.11617754334375263,0.12281072564705736,0.12561062927776825,0.1522523864342253,0.1239577232125015,0.15813189348810802,0.13228031764772005,0.1441537042826044,0.14865968271874172,0.14663487336000344,0.15628160583652026,0.16105728835185573,0.19232931160713815,0.16459665726724856,0.22584658345834327,0.27790370029032374,0.341290981007802,0.3673320560538601,0.45375632925000636,0.488757730985078,0.415489102399999,0.4827771898087042,0.5134150172881237,0.5374975214871849,0.572639630733344,0.5941705762032502,0.6185319853174666,0.6244504822170517,0.6159056330227227,0.6480805961111181,0.653802473304345,0.681398825421419,0.6927900375774612,0.7181051535857201,0.7412455269580253,0.7713191131408461,0.7281089725101992,0.7516023347397444,0.7733061755850285,0.8127115837919564,0.822599544543618,0.7855772629483938,0.8952047110128208,0.838701966044598,0.8265143136025688,0.9165684656815255,0.8251735096687483,0.8624744839625009,0.8868440780188608,0.8954305323687549,1.0859695029000036,1.1480340123375057,1.2719484162784949,1.7461908953170717,1.7227909286335454,1.7219322154250059,1.6400776847405054,1.5058526064187405,1.5231446459750004,1.9264487348553447,2.1793019840445838,2.4209151446898765,2.8207020632173947,3.2124879346455675,3.2671099679374933,3.2732624823024734,3.3913553901829303,3.4870632870060203,3.709676402880946,3.64121875749411,3.653081999319778,3.6814276891149498,3.7334131204602374,3.7840215137921365,3.8350843456388892,3.8378120671538407,3.866392940157599,3.9421263278387055,3.868324142170219,3.9703712516578853,3.934384187234048,3.968551261788665,3.885136304719378,4.183554407803099,4.1764065050855566,4.061296817380004,4.198322738373694,4.358730496722504,4.192881267790579,4.337823682510419,4.3782982340989705,4.2556043054690615,4.011422309985512,4.193590529750001,4.393772548595955,4.277771191616502,4.47920499473501,4.752323044494999,4.521386473328576,4.797423604489889,4.733539008777219,4.623601991682927,4.736413511882918,4.717419656610832,4.661943101281553,4.806857630879812,5.165636263864075,5.176727761630329,5.378880631051402,5.416820673071096,5.720703710712921,5.869616206355135,6.2781982552844],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=llama-7b-awq@vllm-0.2.5\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_first_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b-awq@vllm-0.2.5\",\"line\":{\"color\":\"#EF553B\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b-awq@vllm-0.2.5\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.019464123916729175,0.028330771818197965,0.03693724678784995,0.04627445359092648,0.059786729919942445,0.07112013513333296,0.08328931511429931,0.09456613612499293,0.11634796440740154,0.13000515760001893,0.1321420646464788,0.14450953397916066,0.1725918869422965,0.17033594299999372,0.18118380924167543,0.18339910298374623,0.1943392884608124,0.21407508840739967,0.2202303563596312,0.22888459619166496,0.24085585133334894,0.25036819127269677,0.26568878051448075,0.253648733316564,0.2732106541142587,0.3159157242384446,0.2975587275777966,0.30700427774286904,0.32411109178622705,0.3387297986933269,0.34693886986451616,0.34459731340624844,0.3499319261742495,0.38714591216176375,0.4205047729214162,0.3936521469305553,0.4055380039999732,0.40647260206579977,0.422014811955128,0.4251858914500076,0.48043195120124815,0.46664048510117917,0.5336639283198005,0.4714959863466003,0.4901327111722139,0.489886999184794,0.5182116830552251,0.46393970722619404,0.4952953167687069,0.5678290082066633,0.5666209950784185,0.550370814314093,0.5816298372578785,0.5548178023641945,0.589810875103015,0.5753721321905151,0.6351488054386166,0.6788872733390805,0.6172264209434986,0.7087209888111071,0.6636188552896003,0.6508907918010806,0.6448743967671964,0.8153880382448051,0.9482114074205198,0.891104534095245,1.0206405475172184,1.040138912496982,1.1160943871597722,1.2871025774730658,1.4194971100687552,1.4819661633889076,1.5718770252909247,1.8916952020926965,1.8054935044203613,1.7883764543703644,2.064612102318187,1.9937426055062588,2.1566053665785963,2.2553405110749707,2.3625584340185033,2.4491004099268445,2.3960533000662707,2.47636313065475,2.566872421517653,2.6890977974709225,2.7497805594655085,2.643020788232948,2.690156817691017,2.9047642076388884,2.779194836021986,2.8290884049891187,2.9729399760430026,2.896174527563826,3.1618803087105287,2.9303740617604555,3.175039309649494,3.36584153876166,3.0679363714040497,3.1031345803950057,3.344709210731339,3.2751552187990134,3.21513302886408,3.2041069732692313,3.4740757647999976,4.217111779180344,3.8839343641170334,3.6799189250151434,3.688991141402829,3.5027921130744266,4.039075035059705,3.9080685277253906,3.983902567034003,4.714611416437826,4.4505953837150605,4.450314861409558,4.159244008787882,4.02175758427672,4.74125285589005,4.180763937268301,4.218288550103947,4.8887274778912015,4.418879322849554,4.581166031570699,4.427966952589612,4.765414865793826,4.771548837228849,4.956309427125618],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=llama-7b-gptq@vllm-gptq branch\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_first_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b-gptq@vllm-gptq branch\",\"line\":{\"color\":\"#00cc96\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b-gptq@vllm-gptq branch\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.03033113406660656,0.06207753339292919,0.06982457792310015,0.0838863204583807,0.10193900560002063,0.09646472580305437,0.10086689234288185,0.1051645858333485,0.10635184594437103,0.1126733033833716,0.12185998307580975,0.14919825562492406,0.11878493038460501,0.12804610610717082,0.13273268450666365,0.1482101496750147,0.19159968311432748,0.14802018369443382,0.15265084930264924,0.15277254097512696,0.15598780911907506,0.15517431880678859,0.18074459977169285,0.18093190095832293,0.17059935942855495,0.18471757888459578,0.18587822966668524,0.1798704794881533,0.18689683273559,0.20536622150003597,0.19865979366664194,0.21980410422919286,0.2243921863939727,0.22050326901957948,0.2140040896571788,0.2545180845370312,0.22161892262158436,0.22814517580704968,0.24540159539318643,0.25598602478328303,0.21099661464284844,0.2618159637208813,0.24123156549444758,0.2505020799246991,0.3067259223042959,0.25957591906523664,0.2697872684467605,0.26001312603127263,0.2700944668673791,0.2797883386300327,0.3858796549215649,0.2913157372980689,0.3012074966037495,0.40310163108330455,0.4706759400727027,0.2999617325892895,0.2976983465438073,0.3114546721982653,0.31719916992374314,0.30958710332492956,0.31064649877049066,0.3380605859919665,0.3186679736349366,0.446676342046878,0.31711636424624184,0.3798192396287835,0.4544527462313766,0.3702591288309002,0.4760703501956545,0.3686936720999971,0.3835167126760328,0.4546172254305879,0.43561444306850383,0.5669019691824446,0.531294949306648,0.5695715193223326,0.7635881872922208,0.6312568748974405,0.7375402287594924,0.705871948093727,0.8194668204876642,0.8052669690487386,0.8844556235602609,0.8288524332857227,0.8482122715234897,0.9070355701860376,0.9219668872758915,0.9568068841477281,0.9283857481460841,1.0009682624000107,1.0083008243681193,1.034175129516311,1.0697264405107783,1.181880144898964,1.137311023789478,1.1115188464530898,1.1991610036340352,1.1737354773112323,1.2497804882615478,1.141559642249972,1.2062558439802116,1.3066040453186138,1.4648528823505171,1.432660791989894,1.2432317569808777,1.3871779338640946,1.3125690560094103,1.3818813879814797,1.4031211577617013,1.4436779294540776,1.5517083671633516,1.4471918497416334,1.510594530505032,1.5457598121911806,1.6743124402785712,1.5166455123414357,1.5303918012259776,1.5588890721924893,1.7223953456250092,1.7952037906896634,1.7353376118557235,1.7423281113588671,1.8742245357733616,1.6870486412559347,1.6602260465740655,1.9058621540426555,1.976712762328481,2.0464063935687244],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=llama-7b@tgi-1.3.3\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_first_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b@tgi-1.3.3\",\"line\":{\"color\":\"#ab63fa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b@tgi-1.3.3\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.04407292479954776,0.05470827120043396,0.06257032880009016,0.06811862604981797,0.11653407976009475,0.07359979403336184,0.07349380674277199,0.08409491802503907,0.08153858559983039,0.08520270868011721,0.08632459093889933,0.0884770547916105,0.09696337659614493,0.09868043123229395,0.09926669340005295,0.12983052576555565,0.11593354201477268,0.12755196479174125,0.136834466328908,0.8164373614156345,1.4928739701026643,2.1512007133925075,2.8345868182125287,3.406048651604848,4.035935578183055,4.6351933673613965,5.18865464594046,5.744911006870483,6.478709147802346,6.98066082629874,7.482366778681746,7.978228435988691,8.471962633133343,8.961880774615365,9.492249726184859,9.912728271634576,10.377665958000065,10.851012567621058,10.883936533864585,10.907138072629001,10.96399848359189,10.965737889999971,11.002743193159967,11.073567599435691,11.093728042804,11.12856233435932,11.162968942567378,11.19032814374285,11.277254963801925,11.221624761990638,11.257176950194493,11.294776085724786,11.36142141134541,11.370966202261231,11.381987374749965,11.38345603034506,11.442025829727946,11.432685653408724,11.435705473594775,11.51162066064964,11.537125310559238,11.62844769331938,11.603579177141743,11.659110285975139,11.671992045442524,11.688110751561005,11.81582378489507,11.869477431160034,11.883689162523787,11.918741617440872,12.001438062109315,11.976989582372111,12.058571849299998,12.010571513061056,12.04217391009093,12.097277748804377,12.110829323828362,12.13194912360733,12.175274333999877,12.162064767620373,12.202953590666686,12.198817338582748,12.219784705028633,12.264005348148977,12.281325994845107,12.300012366307605,12.385123480979233,12.333416063855262,12.355819745828864,12.355298432877598,12.469634566141924,12.413864086966475,12.439987469166638,12.442634439682136,12.468847023999974,12.474485954176535,12.513278706077779,12.505603821438713,12.517937937884676,12.54444215700003,12.565005109373441,12.574759089559704,12.600919286506302,12.652155808329214,12.7054472221975,12.624042045447794,12.703551231817173,12.673320153890833,12.773281533885596,12.693928463365273,12.775351845369025,12.719493878485157,12.741323957305864,12.74005491657307,12.75719246847666,12.78544165734103,12.799343318143666,12.807805386234229,12.820864410818258,12.821592045627078,12.849510265612428,12.93032434340237,12.861306621350003,12.942402063679564,12.898963769631916,12.91597005101085,12.926452899733773,13.062753518308199],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=llama-7b-eetq@tgi-1.3.3\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_first_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b-eetq@tgi-1.3.3\",\"line\":{\"color\":\"#FFA15A\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b-eetq@tgi-1.3.3\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.04195532600008139,0.04254945243746988,0.05650931116671624,0.04416613231262545,0.04446792274984546,0.045761248958304655,0.047535240571245856,0.048650933749838386,0.05967702061894266,0.0660119337857915,0.06826878377921877,0.0713218192858048,0.07387221402214081,0.0779358137754812,0.07748955990474704,0.09074265947521493,0.09223604942149798,0.10102845392588346,0.1079215742369534,0.10665599425013474,0.11110473346822786,0.1109817197120306,0.13078182467405355,0.1315747856667334,0.12871347822664878,0.1498985421409928,0.14519692479359125,0.1272877796234209,0.13547604280948586,0.15205593488009375,0.15661862961286743,0.15754463772505006,0.15705300381212675,0.5015873754818821,0.8476375831018188,1.1842825211488703,1.5222974410709178,1.8677363366058939,2.173924055157915,2.5067931978489235,2.8842886830116607,3.156955518879401,3.4576053394743003,3.7626934473068085,4.079543562355923,4.363221869466379,4.6631956965642605,4.96954243319446,5.30089016375144,5.556026343054967,5.851011167098328,6.12337925190206,6.475195553978463,6.740263683758067,7.072209473010586,7.3314037929308435,7.584193476756544,7.856519102815755,8.12732469565968,8.335573938729235,8.579746788010372,8.861201173994878,9.06301219114873,9.330708878086783,9.520459296446715,9.761912501257545,9.799311132929674,9.838512127405028,9.879591796477529,9.920020694613935,9.965521858768481,10.003572125093047,10.048340429517134,10.090361270232995,10.114712929913024,10.207170414028916,10.203665471588483,10.253155743690426,10.334472217255959,10.46026015839152,10.54928120150232,10.543071700686902,10.599877026697733,10.760032099106494,10.720105740165963,10.745065442476992,10.77890575071237,10.81032954828178,10.897133932185508,10.95733154236231,10.882286793865523,10.962722641017917,10.98549977205781,11.044993395916007,11.103318806057796,11.292801597755544,11.140537412043892,12.040073826545909,11.615995015774246,11.328011293842042,11.714465741698634,11.436051271753255,11.598740289618823,11.676861534423395,11.594741504951312,12.216426168556591,11.996908165917052,11.554266906931035,11.885702124315616,11.789990765687214,12.000335070008898,11.919042254672556,11.912013277293857,11.900957441486792,11.908260097013061,11.96668637411644,11.86847904789316,11.84353700536028,11.775668676903479,11.825582682716496,11.735781077628094,11.708772235483542,11.661068357016246,11.802864029423414,11.686575693871994,11.679420248150745,11.53287206485828,11.603571357644505],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=llama-7b-awq@tgi-1.3.3\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_first_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b-awq@tgi-1.3.3\",\"line\":{\"color\":\"#19d3f3\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b-awq@tgi-1.3.3\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.04167407283330249,0.042591082500207864,0.04477750903016854,0.04932447440000942,0.05772703906004608,0.06766981648330936,0.0770387442538026,0.08609929231935591,0.0972506076913204,0.11816277039999931,0.12519203342057236,0.13497684956246303,0.14634732991334415,0.1549234398303432,0.1638005227963125,0.17538668489290948,0.18278952494129758,0.2070190628888095,0.2118691581052732,0.22227182286672661,0.2322813534285218,0.23496497546210032,0.24337103313498826,0.257829682566747,0.2678293697760237,0.27822394055386285,0.299314463392619,0.3087140547285344,0.32477370649660237,0.32309788942654755,0.33896654114196867,0.3416943186124513,0.3504995631439918,0.38589157088238285,0.38631336530003996,0.3918650322500146,0.4076163266352376,0.40687481901312295,0.7289948362483651,1.0314769887986763,1.472807826793618,1.6310434585448559,1.9386050649489706,2.2760052141582237,2.5601090482452045,2.8400936312874707,3.343633699316749,3.378957173302398,3.7326543041350653,3.8926165191402418,4.414611761460589,4.5807841502590305,4.817354221574887,5.056266762702318,5.276813226603547,5.69058274025878,6.022462056906419,6.064718694970864,6.358105394479777,6.79637615263215,7.0221695533771396,7.160414340369326,7.37230610224868,7.499651335247231,7.85834902584363,7.814700718244381,8.044396767093868,8.541854997291132,8.707838730776007,8.888988993092298,9.04733661790809,9.19007026505375,9.631151856780729,9.472103518414896,9.895905707550343,10.432610823573764,10.435027694753872,10.370433804828025,10.389854844367802,10.353668010422599,10.517032410138377,10.143589935040781,10.285252998700441,10.59200893408076,10.547018286070344,10.528985041619917,10.517980650189184,10.481091602702978,10.640172906571454,10.282045543426426,10.417520848321972,10.69836583072334,10.663731494280203,10.679535952418345,10.687227260574192,10.539764325561945,10.731102861393385,10.373099762103823,10.524307497755789,10.767045549415828,10.810908645832498,10.750100794361044,10.730718721207381,10.655284177623857,10.829327316191753,10.712911861887267,10.621187929013507,10.884934230518132,10.837345795470874,10.827202202763406,10.818893811377809,10.810845706495545,10.960099607017522,10.592427655627073,10.72832300790833,10.9909923784043,11.044779082342005,10.98629177504305,10.930923615781056,10.908684680132453,11.103935925455362,10.735969538499962,10.880790819109713,11.111418724949583,11.078130764410025,11.16271798024577,11.075021787165946,11.059591335975178],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=llama-7b-gptq@tgi-1.3.3\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_first_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"llama-7b-gptq@tgi-1.3.3\",\"line\":{\"color\":\"#FF6692\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"llama-7b-gptq@tgi-1.3.3\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.04235638053323783,0.05196514376900897,0.05959054219445332,0.08475871820830132,0.09071134810900813,0.09391383727278012,0.0980654750569946,0.0991997789860256,0.10502925892601693,0.11064605858330955,0.11212593371210185,0.13757560305566585,0.11793161662822407,0.15207485713883923,0.14253976933323428,0.13944946157498633,0.13992329779410498,0.149155963666519,0.1478958764869276,0.14696731018761966,0.1502504607975598,0.15216450167054063,0.1709434182065034,0.18223832086469352,0.16846848335439915,0.18094895265370817,0.23211053339501733,0.17910095065480208,0.2898938625056317,0.20516655851119545,0.20665619615060532,0.21103578161451728,0.21730810060593414,0.21803247086261443,0.22113296119040612,0.21438940023147182,0.2702196344054411,0.23806973451742683,0.23650177041884327,0.2375516142666432,0.22271895835887257,0.5864835732470323,0.9205753148736671,1.1768119778404031,1.523432659774151,1.9526000848695335,2.134032753574502,2.4216787854105575,2.647799191683849,3.1440102760000412,3.291140254402092,3.358395106692342,3.5815885297736214,3.7837436485925973,4.023810513229372,4.543630511155216,4.452949544517829,5.224686397356575,5.311119544647625,5.159295695964925,5.195988781598242,5.428547340909748,6.0080356190870665,6.319482453859692,6.047582318531669,7.032426112594581,7.096748061382586,6.332804685255662,6.607741289931779,7.332223546177433,7.6551439338759675,7.83819413443437,7.634738453441969,8.736742745439688,8.829271024177924,7.844687020926889,8.350562544409003,9.027645733808065,9.260804059543982,9.566300072612927,9.345730788776928,10.11095063170724,10.064577329376043,9.796274658192376,10.134904420806246,10.10551889406972,10.219976139100734,10.336106522317808,10.181441802714364,10.409727341580073,10.444037897947027,10.328691309881536,10.515938628701447,10.546010386607398,10.654122265897032,10.632333121270085,10.645308788840735,10.680012654122326,10.71105915865003,10.595744793000023,10.769905939119704,10.801911061937126,10.831197600722172,10.862304466944787,10.88870651356151,10.91959696400682,10.946595939033823,11.137420606704628,11.003163454013253,11.02892259699341,11.114864500322435,11.089134258372617,11.10759598922722,11.134550681451572,11.225628659185892,11.18536804795536,11.208409700645564,11.325786599635174,11.27975534629378,11.349296206111763,11.405967680598728,11.332319571128973,11.353550492689054,11.422628582957678,11.397339948277038,11.41991995346702,11.557859763154804,11.46422856693491],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"batch\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"avg_first_token_latency\"}},\"legend\":{\"title\":{\"text\":\"type\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Llama Average First Token Latency\"}},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('cdbe155d-c0dd-4d2f-8554-71d77df248b8');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                });            </script>        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig_llama_avg_latency = px.line(llamas, x='batch', y='avg_first_token_latency', color='type', title='Llama Average First Token Latency')\n",
    "fig_llama_avg_latency.write_image('images/llama2-7b-first-token-lat.png', scale=2, width=1440)\n",
    "fig_llama_avg_latency.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "hovertemplate": "type=qwen-7b-awq@vllm-0.2.5<br>batch=%{x}<br>token_per_s=%{y}<extra></extra>",
         "legendgroup": "qwen-7b-awq@vllm-0.2.5",
         "line": {
          "color": "#636efa",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "qwen-7b-awq@vllm-0.2.5",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          65.5,
          126.95,
          186.16666666666666,
          239.4,
          291.3666666666667,
          338.7,
          381.43333333333334,
          421.45,
          460.31666666666666,
          498.3333333333333,
          540.4,
          569.05,
          600.2166666666667,
          622.9,
          654.5833333333334,
          680.3333333333334,
          613.6166666666667,
          634.3666666666667,
          666.9333333333333,
          682.5666666666667,
          712.8666666666667,
          727.2166666666667,
          737.4333333333333,
          752.2333333333333,
          775.9333333333333,
          795.55,
          818.9,
          831.5333333333333,
          854.6166666666667,
          863.85,
          864.4833333333333,
          865.7333333333333,
          776.5333333333333,
          788.0833333333334,
          808.8833333333333,
          820.3,
          842.65,
          851.55,
          848.85,
          846.1833333333333,
          869.5166666666667,
          872.7,
          892.7166666666667,
          897.2666666666667,
          899.2333333333333,
          906.1833333333333,
          901.7333333333333,
          894.8833333333333,
          837.1666666666666,
          831.1833333333333,
          836.8833333333333,
          840.3166666666667,
          845.6833333333333,
          847.8166666666667,
          832.7333333333333,
          816.2833333333333,
          831.1833333333333,
          826.2,
          825.25,
          829.9833333333333,
          835.35,
          834.6833333333333,
          818.2166666666667,
          804.4833333333333,
          797.5166666666667,
          800.8333333333334,
          805.1666666666666,
          804.6666666666666,
          818.8833333333333,
          816.5166666666667,
          813.3833333333333,
          805.0833333333334,
          819.4166666666666,
          818.4,
          828.6333333333333,
          835.1833333333333,
          838.65,
          847.3833333333333,
          844.8833333333333,
          843.6333333333333,
          825.7333333333333,
          822.4,
          823.0666666666667,
          828.3333333333334,
          835.35,
          839.4,
          822.7833333333333,
          805.0333333333333,
          825.8,
          803.0333333333333,
          788.0666666666667,
          777.0666666666667,
          764.2166666666667,
          762.3166666666667,
          754.45,
          740.1,
          742.5333333333333,
          733.35,
          740.7166666666667,
          732.55,
          734.0666666666667,
          734.0666666666667,
          719.6666666666666,
          724.8,
          731,
          716.6333333333333,
          717.4333333333333,
          724.6333333333333,
          728.8666666666667,
          719.9333333333333,
          707.3333333333334,
          699.5333333333333,
          706.4333333333333,
          696.3,
          701.2166666666667,
          701.5666666666667,
          714.4166666666666,
          715.6,
          713.3833333333333,
          711.4833333333333,
          723.3833333333333,
          703.0333333333333,
          707.7833333333333,
          714.7666666666667,
          711.5,
          708.0333333333333,
          721,
          692.6166666666667
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=qwen-7b@vllm-0.2.5<br>batch=%{x}<br>token_per_s=%{y}<extra></extra>",
         "legendgroup": "qwen-7b@vllm-0.2.5",
         "line": {
          "color": "#EF553B",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "qwen-7b@vllm-0.2.5",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          34.93333333333333,
          67.86666666666666,
          100.36666666666666,
          132.06666666666666,
          162.95,
          193.53333333333333,
          223.2,
          252.21666666666667,
          280.2,
          308.1,
          334.1333333333333,
          359.05,
          386.23333333333335,
          411.4166666666667,
          436.3833333333333,
          460.43333333333334,
          466.48333333333335,
          489.5833333333333,
          510.51666666666665,
          530.9833333333333,
          551.6666666666666,
          585.5333333333333,
          598.7166666666667,
          617.2166666666667,
          632.3166666666667,
          646.6833333333333,
          664.7666666666667,
          677.95,
          689.4833333333333,
          700.1333333333333,
          712.05,
          724.3166666666667,
          734.1333333333333,
          743.55,
          735.5166666666667,
          740.4,
          739.4666666666667,
          747,
          743.55,
          737.5666666666667,
          744.2666666666667,
          749.45,
          751.4833333333333,
          751.0166666666667,
          750.5833333333334,
          757.1166666666667,
          762.4833333333333,
          757.7333333333333,
          763.0833333333334,
          766.3833333333333,
          766.6166666666667,
          777.5,
          779.4833333333333,
          776.1333333333333,
          771.05,
          771.4333333333333,
          770.05,
          776.5333333333333,
          768.3333333333334,
          776.45,
          777.4833333333333,
          780.5166666666667,
          784.3333333333334,
          789.9666666666667,
          787.4666666666667,
          791.1833333333333,
          784.5,
          789.9166666666666,
          791.5666666666667,
          795.4333333333333,
          791,
          795.7166666666667,
          789.6833333333333,
          781.6333333333333,
          779.8333333333334,
          777.3333333333334,
          776.1666666666666,
          777.3666666666667,
          773.75,
          776.5166666666667,
          777.65,
          777.9,
          779.95,
          780.1833333333333,
          786.8333333333334,
          781.9333333333333,
          781.6666666666666,
          791.6666666666666,
          791.4333333333333,
          793.85,
          793.0833333333334,
          783.9166666666666,
          784.55,
          794.0666666666667,
          795.1166666666667,
          798.9333333333333,
          785.95,
          785.9666666666667,
          784.8833333333333,
          797.45,
          800.0666666666667,
          801.8666666666667,
          797.4,
          786.9,
          784.4166666666666,
          803.8833333333333,
          801.0833333333334,
          800.8,
          788.1833333333333,
          794.85,
          781.45,
          789.6666666666666,
          801.15,
          800.65,
          792.6833333333333,
          791.1833333333333,
          790.35,
          805.95,
          794.5666666666667,
          779.4166666666666,
          777.0833333333334,
          793.5166666666667,
          815.2,
          794.7,
          791.4166666666666,
          773.8333333333334,
          797.85,
          810.8
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=qwen-7b-int4@vllm-2acd76f<br>batch=%{x}<br>token_per_s=%{y}<extra></extra>",
         "legendgroup": "qwen-7b-int4@vllm-2acd76f",
         "line": {
          "color": "#00cc96",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "qwen-7b-int4@vllm-2acd76f",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          86.85,
          159.2,
          226.28333333333333,
          287.76666666666665,
          347.9166666666667,
          373.35,
          400.3333333333333,
          422.5833333333333,
          345.21666666666664,
          367.93333333333334,
          393.51666666666665,
          417.65,
          441.31666666666666,
          450.0833333333333,
          459.0833333333333,
          510.68333333333334,
          440.98333333333335,
          454.5,
          471.4166666666667,
          487.2,
          502.51666666666665,
          508.1,
          512.9666666666667,
          554.4333333333333,
          495.96666666666664,
          505.26666666666665,
          517.5333333333333,
          529.2333333333333,
          538.9833333333333,
          541.0833333333334,
          545.5,
          578.3833333333333,
          525.1166666666667,
          531.2333333333333,
          540.2166666666667,
          550.2166666666667,
          560.05,
          561.9833333333333,
          562.3,
          591.8,
          549.0833333333334,
          555.7166666666667,
          562.2666666666667,
          566.8333333333334,
          573.4166666666666,
          578.75,
          579.9166666666666,
          602.7333333333333,
          563,
          564.8666666666667,
          662.1333333333333,
          672.4,
          678.8833333333333,
          691.0166666666667,
          699.9,
          709.3666666666667,
          721.0333333333333,
          732.7833333333333,
          742.6166666666667,
          751.3333333333334,
          760.3333333333334,
          765.85,
          772.7833333333333,
          781.4666666666667,
          770.2833333333333,
          774.4166666666666,
          786.5166666666667,
          795.3833333333333,
          803.6333333333333,
          808.85,
          816.8166666666667,
          817.6333333333333,
          815.9,
          827.0333333333333,
          831.7833333333333,
          839.8166666666667,
          844.5,
          851.1166666666667,
          850.7166666666667,
          866.5166666666667,
          868.7333333333333,
          874.1,
          877.5333333333333,
          882.6833333333333,
          886.65,
          894.6166666666667,
          891.8166666666667,
          901.3666666666667,
          907.2,
          913.45,
          913.6166666666667,
          920.1,
          925,
          932.4,
          936.9166666666666,
          941.4833333333333,
          951.8166666666667,
          954.6833333333333,
          957.2166666666667,
          962.0666666666667,
          965.1166666666667,
          973.2833333333333,
          972.1833333333333,
          982.5166666666667,
          984.15,
          973.3833333333333,
          978.2,
          985.6833333333333,
          988.4833333333333,
          993.2833333333333,
          999.65,
          1002.5,
          998.0333333333333,
          1002.9166666666666,
          1004,
          1011.2666666666667,
          1007.45,
          1014.2,
          1018.2166666666667,
          1021.0333333333333,
          1022.4833333333333,
          1022.85,
          1048.7,
          1048.9333333333334,
          1047.4333333333334,
          1065.2166666666667,
          1053.7,
          1062.75
         ],
         "yaxis": "y"
        }
       ],
       "layout": {
        "legend": {
         "title": {
          "text": "type"
         },
         "tracegroupgap": 0
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "Qwen 7B Throughput"
        },
        "xaxis": {
         "anchor": "y",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "batch"
         }
        },
        "yaxis": {
         "anchor": "x",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "token_per_s"
         }
        }
       }
      },
      "text/html": [
       "<div>                            <div id=\"951ce648-63e2-407a-8660-3e4021abd641\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"951ce648-63e2-407a-8660-3e4021abd641\")) {                    Plotly.newPlot(                        \"951ce648-63e2-407a-8660-3e4021abd641\",                        [{\"hovertemplate\":\"type=qwen-7b-awq@vllm-0.2.5\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003etoken_per_s=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"qwen-7b-awq@vllm-0.2.5\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"qwen-7b-awq@vllm-0.2.5\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[65.5,126.95,186.16666666666666,239.4,291.3666666666667,338.7,381.43333333333334,421.45,460.31666666666666,498.3333333333333,540.4,569.05,600.2166666666667,622.9,654.5833333333334,680.3333333333334,613.6166666666667,634.3666666666667,666.9333333333333,682.5666666666667,712.8666666666667,727.2166666666667,737.4333333333333,752.2333333333333,775.9333333333333,795.55,818.9,831.5333333333333,854.6166666666667,863.85,864.4833333333333,865.7333333333333,776.5333333333333,788.0833333333334,808.8833333333333,820.3,842.65,851.55,848.85,846.1833333333333,869.5166666666667,872.7,892.7166666666667,897.2666666666667,899.2333333333333,906.1833333333333,901.7333333333333,894.8833333333333,837.1666666666666,831.1833333333333,836.8833333333333,840.3166666666667,845.6833333333333,847.8166666666667,832.7333333333333,816.2833333333333,831.1833333333333,826.2,825.25,829.9833333333333,835.35,834.6833333333333,818.2166666666667,804.4833333333333,797.5166666666667,800.8333333333334,805.1666666666666,804.6666666666666,818.8833333333333,816.5166666666667,813.3833333333333,805.0833333333334,819.4166666666666,818.4,828.6333333333333,835.1833333333333,838.65,847.3833333333333,844.8833333333333,843.6333333333333,825.7333333333333,822.4,823.0666666666667,828.3333333333334,835.35,839.4,822.7833333333333,805.0333333333333,825.8,803.0333333333333,788.0666666666667,777.0666666666667,764.2166666666667,762.3166666666667,754.45,740.1,742.5333333333333,733.35,740.7166666666667,732.55,734.0666666666667,734.0666666666667,719.6666666666666,724.8,731.0,716.6333333333333,717.4333333333333,724.6333333333333,728.8666666666667,719.9333333333333,707.3333333333334,699.5333333333333,706.4333333333333,696.3,701.2166666666667,701.5666666666667,714.4166666666666,715.6,713.3833333333333,711.4833333333333,723.3833333333333,703.0333333333333,707.7833333333333,714.7666666666667,711.5,708.0333333333333,721.0,692.6166666666667],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=qwen-7b@vllm-0.2.5\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003etoken_per_s=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"qwen-7b@vllm-0.2.5\",\"line\":{\"color\":\"#EF553B\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"qwen-7b@vllm-0.2.5\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[34.93333333333333,67.86666666666666,100.36666666666666,132.06666666666666,162.95,193.53333333333333,223.2,252.21666666666667,280.2,308.1,334.1333333333333,359.05,386.23333333333335,411.4166666666667,436.3833333333333,460.43333333333334,466.48333333333335,489.5833333333333,510.51666666666665,530.9833333333333,551.6666666666666,585.5333333333333,598.7166666666667,617.2166666666667,632.3166666666667,646.6833333333333,664.7666666666667,677.95,689.4833333333333,700.1333333333333,712.05,724.3166666666667,734.1333333333333,743.55,735.5166666666667,740.4,739.4666666666667,747.0,743.55,737.5666666666667,744.2666666666667,749.45,751.4833333333333,751.0166666666667,750.5833333333334,757.1166666666667,762.4833333333333,757.7333333333333,763.0833333333334,766.3833333333333,766.6166666666667,777.5,779.4833333333333,776.1333333333333,771.05,771.4333333333333,770.05,776.5333333333333,768.3333333333334,776.45,777.4833333333333,780.5166666666667,784.3333333333334,789.9666666666667,787.4666666666667,791.1833333333333,784.5,789.9166666666666,791.5666666666667,795.4333333333333,791.0,795.7166666666667,789.6833333333333,781.6333333333333,779.8333333333334,777.3333333333334,776.1666666666666,777.3666666666667,773.75,776.5166666666667,777.65,777.9,779.95,780.1833333333333,786.8333333333334,781.9333333333333,781.6666666666666,791.6666666666666,791.4333333333333,793.85,793.0833333333334,783.9166666666666,784.55,794.0666666666667,795.1166666666667,798.9333333333333,785.95,785.9666666666667,784.8833333333333,797.45,800.0666666666667,801.8666666666667,797.4,786.9,784.4166666666666,803.8833333333333,801.0833333333334,800.8,788.1833333333333,794.85,781.45,789.6666666666666,801.15,800.65,792.6833333333333,791.1833333333333,790.35,805.95,794.5666666666667,779.4166666666666,777.0833333333334,793.5166666666667,815.2,794.7,791.4166666666666,773.8333333333334,797.85,810.8],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=qwen-7b-int4@vllm-2acd76f\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003etoken_per_s=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"qwen-7b-int4@vllm-2acd76f\",\"line\":{\"color\":\"#00cc96\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"qwen-7b-int4@vllm-2acd76f\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[86.85,159.2,226.28333333333333,287.76666666666665,347.9166666666667,373.35,400.3333333333333,422.5833333333333,345.21666666666664,367.93333333333334,393.51666666666665,417.65,441.31666666666666,450.0833333333333,459.0833333333333,510.68333333333334,440.98333333333335,454.5,471.4166666666667,487.2,502.51666666666665,508.1,512.9666666666667,554.4333333333333,495.96666666666664,505.26666666666665,517.5333333333333,529.2333333333333,538.9833333333333,541.0833333333334,545.5,578.3833333333333,525.1166666666667,531.2333333333333,540.2166666666667,550.2166666666667,560.05,561.9833333333333,562.3,591.8,549.0833333333334,555.7166666666667,562.2666666666667,566.8333333333334,573.4166666666666,578.75,579.9166666666666,602.7333333333333,563.0,564.8666666666667,662.1333333333333,672.4,678.8833333333333,691.0166666666667,699.9,709.3666666666667,721.0333333333333,732.7833333333333,742.6166666666667,751.3333333333334,760.3333333333334,765.85,772.7833333333333,781.4666666666667,770.2833333333333,774.4166666666666,786.5166666666667,795.3833333333333,803.6333333333333,808.85,816.8166666666667,817.6333333333333,815.9,827.0333333333333,831.7833333333333,839.8166666666667,844.5,851.1166666666667,850.7166666666667,866.5166666666667,868.7333333333333,874.1,877.5333333333333,882.6833333333333,886.65,894.6166666666667,891.8166666666667,901.3666666666667,907.2,913.45,913.6166666666667,920.1,925.0,932.4,936.9166666666666,941.4833333333333,951.8166666666667,954.6833333333333,957.2166666666667,962.0666666666667,965.1166666666667,973.2833333333333,972.1833333333333,982.5166666666667,984.15,973.3833333333333,978.2,985.6833333333333,988.4833333333333,993.2833333333333,999.65,1002.5,998.0333333333333,1002.9166666666666,1004.0,1011.2666666666667,1007.45,1014.2,1018.2166666666667,1021.0333333333333,1022.4833333333333,1022.85,1048.7,1048.9333333333334,1047.4333333333334,1065.2166666666667,1053.7,1062.75],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"batch\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"token_per_s\"}},\"legend\":{\"title\":{\"text\":\"type\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Qwen 7B Throughput\"}},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('951ce648-63e2-407a-8660-3e4021abd641');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                });            </script>        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "hovertemplate": "type=qwen-7b-awq@vllm-0.2.5<br>batch=%{x}<br>avg_token_latency=%{y}<extra></extra>",
         "legendgroup": "qwen-7b-awq@vllm-0.2.5",
         "line": {
          "color": "#636efa",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "qwen-7b-awq@vllm-0.2.5",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.015150871137562985,
          0.015600082068633073,
          0.015914874238991964,
          0.01644466466826341,
          0.016860417164594205,
          0.01733858223863535,
          0.01791027830163014,
          0.018467885247248426,
          0.019001233422656777,
          0.019419565774742378,
          0.019695663323528845,
          0.020339049438367222,
          0.0208250901238371,
          0.021642321951224893,
          0.02195433222001726,
          0.022539981179709602,
          0.026567698566066524,
          0.02712459738886533,
          0.02725482659410125,
          0.02799718197581259,
          0.02808177427961441,
          0.028695486699655985,
          0.02981748910958991,
          0.030335411881826365,
          0.03070900268775741,
          0.03108158837523273,
          0.03129776813344272,
          0.03191957998076115,
          0.03204125035006702,
          0.032768622237492794,
          0.03375239941245888,
          0.03465987948855458,
          0.0401028506390529,
          0.040855898920610784,
          0.04088145477980233,
          0.04145171656440815,
          0.04131407535642397,
          0.041841048188646465,
          0.04307559935803539,
          0.04422760981228514,
          0.04415616363676548,
          0.0449584524781481,
          0.04496012819366018,
          0.04571005891231817,
          0.04605763027669208,
          0.046927288763841446,
          0.04791989011180796,
          0.04888027648195763,
          0.05312406280340967,
          0.05424815230479693,
          0.0545354487632314,
          0.05509646432763298,
          0.05534807552124951,
          0.05591709191201937,
          0.05791979870255452,
          0.059582832656269326,
          0.059290035094802435,
          0.06062583097183963,
          0.061271025534147656,
          0.06200578979571618,
          0.062028242615381424,
          0.06327969106467397,
          0.06534797487727877,
          0.06775657718339308,
          0.07194945536490845,
          0.07537650123210578,
          0.07601436878392609,
          0.07701587842223076,
          0.07646556004676654,
          0.0771936641196471,
          0.07773596656481974,
          0.0794089164298116,
          0.0793435099162432,
          0.08000882498983307,
          0.07988641088644977,
          0.08058332846669432,
          0.08098645607964954,
          0.08108532030834034,
          0.08205236126309792,
          0.08269076150986202,
          0.08579721734909919,
          0.08696288183846602,
          0.08713530514575386,
          0.08776229662596365,
          0.08787620677719742,
          0.08885602257060451,
          0.09106359638534783,
          0.09418517363781742,
          0.09288906205977382,
          0.09605068248165075,
          0.09958029332992092,
          0.1017720737994818,
          0.10426379776551742,
          0.10575351754002113,
          0.10729595121972323,
          0.10983932968160214,
          0.1115424946146999,
          0.11319337012564919,
          0.11209011170809005,
          0.11569119076984047,
          0.11539378717310808,
          0.11617969608943186,
          0.11992398879873321,
          0.12058958892402054,
          0.12058148516128285,
          0.1243533949682646,
          0.12397355630882875,
          0.1249291530914942,
          0.12417071332522645,
          0.127729019935295,
          0.13038231576737544,
          0.13138831006662502,
          0.1299873901637967,
          0.13214710202038318,
          0.12992959120542538,
          0.13079826392836127,
          0.12602614431096823,
          0.12730214211915367,
          0.12832343307699243,
          0.1306229875768168,
          0.12842656680669143,
          0.13538260273309746,
          0.13406212115089988,
          0.13410839495649465,
          0.13549966695887655,
          0.13846552817315563,
          0.13671105296527872,
          0.14688766411781562
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=qwen-7b@vllm-0.2.5<br>batch=%{x}<br>avg_token_latency=%{y}<extra></extra>",
         "legendgroup": "qwen-7b@vllm-0.2.5",
         "line": {
          "color": "#EF553B",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "qwen-7b@vllm-0.2.5",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.028442294297625497,
          0.029251332000985793,
          0.0296144977343698,
          0.02998198482494244,
          0.030392584144347413,
          0.030641278966825267,
          0.030994415872635933,
          0.031316432361286825,
          0.03168420626233855,
          0.031985274728567166,
          0.032282865457177436,
          0.03285161723525935,
          0.033066337407720345,
          0.03338753762171061,
          0.03366404034124792,
          0.034018641029817207,
          0.03564711067292859,
          0.03594564229874779,
          0.0363537086547155,
          0.03661003702246473,
          0.03712705525645684,
          0.036398301877873095,
          0.037165337461060585,
          0.037506000414368315,
          0.037972914749402585,
          0.03850560737237437,
          0.03883871254215588,
          0.03941514644819476,
          0.04003846746479522,
          0.04074105166724794,
          0.04129751943337703,
          0.04189716042214032,
          0.04254990799026965,
          0.04327542919758816,
          0.04495658450500199,
          0.04584433322225246,
          0.04706591438592847,
          0.04773741712261287,
          0.04839975873517459,
          0.05016672665407648,
          0.0508641275828388,
          0.05176487969109283,
          0.05245137853458329,
          0.05373140662073632,
          0.05524508917934316,
          0.05626891707143307,
          0.056848094477726395,
          0.05824307145496063,
          0.058914048132600656,
          0.0594761288555523,
          0.06047502358286252,
          0.06053221981391659,
          0.061672584150830045,
          0.06323171237389494,
          0.06457120542152398,
          0.06469886519219542,
          0.065054751444771,
          0.06446584616842088,
          0.06570014669418664,
          0.06581771657892638,
          0.06734528617589579,
          0.06801440238547708,
          0.0685236961067285,
          0.06896610677069746,
          0.0697690944730954,
          0.07034659298472698,
          0.07262447818662139,
          0.07291755583832299,
          0.07371514234272862,
          0.07370661752391487,
          0.07433022907895426,
          0.07531361151444974,
          0.07661130676936026,
          0.0782530333188607,
          0.07982810159577479,
          0.08013616335216524,
          0.08071795300185215,
          0.08091390090341405,
          0.08236139298307682,
          0.08446799787791662,
          0.08553203801562739,
          0.08620813325439595,
          0.08831508824295446,
          0.0887748490170395,
          0.08945021813818578,
          0.09065257817264846,
          0.09325000311152755,
          0.09166027588074727,
          0.09263147876932198,
          0.09291809305170799,
          0.09379635403123693,
          0.09437508561947051,
          0.0959004555083452,
          0.0937824779859813,
          0.09404378391575326,
          0.09357691125621852,
          0.09632371558971388,
          0.09686127809168865,
          0.09722713541111241,
          0.09409318509509648,
          0.09403281367466873,
          0.09357322215770239,
          0.09435972112345407,
          0.09613270985858705,
          0.09846086267582999,
          0.09479615594211946,
          0.09547598439025759,
          0.09540756234221492,
          0.09758928167988287,
          0.09745679758930215,
          0.10091543492884683,
          0.09921827237516588,
          0.09956801323351826,
          0.10045996285981675,
          0.10183501060564366,
          0.1032627949169306,
          0.10332584209351528,
          0.10238639029567309,
          0.10480984578352272,
          0.10787931696333078,
          0.10829029432616737,
          0.10715379255092279,
          0.10549178669897667,
          0.1087587000079894,
          0.11013809456842835,
          0.1141960479883138,
          0.11085035270069485,
          0.11061829963454177
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=qwen-7b-int4@vllm-2acd76f<br>batch=%{x}<br>avg_token_latency=%{y}<extra></extra>",
         "legendgroup": "qwen-7b-int4@vllm-2acd76f",
         "line": {
          "color": "#00cc96",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "qwen-7b-int4@vllm-2acd76f",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.011328733096767652,
          0.012295940237952172,
          0.012913652161704267,
          0.013500276015664543,
          0.013952679513266921,
          0.015573866504962652,
          0.01696090940530604,
          0.018374710009787222,
          0.025445858571387203,
          0.02649851524642658,
          0.027235670381244286,
          0.027983253155618736,
          0.028669619287627235,
          0.030230866390741427,
          0.03176652947371632,
          0.030356354088291217,
          0.03742148438213851,
          0.03854242022793125,
          0.039181082655653475,
          0.03979952719812276,
          0.04053300655603579,
          0.041891688496417105,
          0.04341314212978128,
          0.04179732663378608,
          0.04890836812241426,
          0.049887239767686845,
          0.05054635656971237,
          0.051213676518690944,
          0.051803872306364,
          0.05342816937953117,
          0.054861140451738906,
          0.05337437923649241,
          0.06045001661249973,
          0.06167969983871537,
          0.06255362020026735,
          0.06319460327892043,
          0.0637891809401309,
          0.06527637949773717,
          0.06660932953431367,
          0.06517023874393578,
          0.07196058535186012,
          0.07263711760534446,
          0.07349289553388912,
          0.07433726760727087,
          0.07492983035072587,
          0.07621608065939266,
          0.07756056746612042,
          0.07619839272557122,
          0.08333093308080756,
          0.08452762338446754,
          0.0734195635115067,
          0.07383074635593782,
          0.07423119225473065,
          0.0745461050590555,
          0.07481206968931826,
          0.07516905189745274,
          0.07502650838750102,
          0.07512908362587922,
          0.07541389190713303,
          0.07566779579562383,
          0.07593346816566772,
          0.07664394925065982,
          0.07720952455132746,
          0.07733311307806219,
          0.07950962749502444,
          0.07985318493592333,
          0.08041934678477629,
          0.080617810081605,
          0.08094722158048087,
          0.08133942389618729,
          0.0816029216865573,
          0.08268023731488632,
          0.08345653454249614,
          0.08384520796072228,
          0.08440879419001364,
          0.08482018883816271,
          0.08515776040488748,
          0.08565911904004557,
          0.08605763982613723,
          0.08576142759588827,
          0.08621028853782146,
          0.08671965231774974,
          0.08740136489734111,
          0.08771661387816948,
          0.08932687123890191,
          0.08927682602822412,
          0.0902069612230371,
          0.09044565081785133,
          0.09091285812404179,
          0.09116055393788858,
          0.09205606415554879,
          0.09238510408592324,
          0.09275150162392054,
          0.09269617315377446,
          0.09334888310420758,
          0.09386383658149966,
          0.09389720263889978,
          0.09406207750633597,
          0.09509933515210056,
          0.09483468926711044,
          0.09551817921878242,
          0.09579841374858789,
          0.09620776767952025,
          0.09684676692866268,
          0.09709491189547197,
          0.098792676784246,
          0.09938375345774632,
          0.09966145993984335,
          0.09990721182717795,
          0.10052288620247106,
          0.10038426685236726,
          0.10130382522780743,
          0.10236814052389047,
          0.10285690481272956,
          0.10357836500999466,
          0.10319934726671427,
          0.10423645927717247,
          0.10463483704243949,
          0.10489303327549807,
          0.10582853614688321,
          0.10615609096824834,
          0.10668105650137956,
          0.10505734280428942,
          0.10557305535902362,
          0.10603591256771161,
          0.10603842521300264,
          0.10724409192976028,
          0.10721985988629308
         ],
         "yaxis": "y"
        }
       ],
       "layout": {
        "legend": {
         "title": {
          "text": "type"
         },
         "tracegroupgap": 0
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "Qwen 7B Average non-first Token Latency"
        },
        "xaxis": {
         "anchor": "y",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "batch"
         }
        },
        "yaxis": {
         "anchor": "x",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "avg_token_latency"
         }
        }
       }
      },
      "text/html": [
       "<div>                            <div id=\"193aeaf0-bd9d-4cbf-bb76-bc7be013b95e\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"193aeaf0-bd9d-4cbf-bb76-bc7be013b95e\")) {                    Plotly.newPlot(                        \"193aeaf0-bd9d-4cbf-bb76-bc7be013b95e\",                        [{\"hovertemplate\":\"type=qwen-7b-awq@vllm-0.2.5\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"qwen-7b-awq@vllm-0.2.5\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"qwen-7b-awq@vllm-0.2.5\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.015150871137562985,0.015600082068633073,0.015914874238991964,0.01644466466826341,0.016860417164594205,0.01733858223863535,0.01791027830163014,0.018467885247248426,0.019001233422656777,0.019419565774742378,0.019695663323528845,0.020339049438367222,0.0208250901238371,0.021642321951224893,0.02195433222001726,0.022539981179709602,0.026567698566066524,0.02712459738886533,0.02725482659410125,0.02799718197581259,0.02808177427961441,0.028695486699655985,0.02981748910958991,0.030335411881826365,0.03070900268775741,0.03108158837523273,0.03129776813344272,0.03191957998076115,0.03204125035006702,0.032768622237492794,0.03375239941245888,0.03465987948855458,0.0401028506390529,0.040855898920610784,0.04088145477980233,0.04145171656440815,0.04131407535642397,0.041841048188646465,0.04307559935803539,0.04422760981228514,0.04415616363676548,0.0449584524781481,0.04496012819366018,0.04571005891231817,0.04605763027669208,0.046927288763841446,0.04791989011180796,0.04888027648195763,0.05312406280340967,0.05424815230479693,0.0545354487632314,0.05509646432763298,0.05534807552124951,0.05591709191201937,0.05791979870255452,0.059582832656269326,0.059290035094802435,0.06062583097183963,0.061271025534147656,0.06200578979571618,0.062028242615381424,0.06327969106467397,0.06534797487727877,0.06775657718339308,0.07194945536490845,0.07537650123210578,0.07601436878392609,0.07701587842223076,0.07646556004676654,0.0771936641196471,0.07773596656481974,0.0794089164298116,0.0793435099162432,0.08000882498983307,0.07988641088644977,0.08058332846669432,0.08098645607964954,0.08108532030834034,0.08205236126309792,0.08269076150986202,0.08579721734909919,0.08696288183846602,0.08713530514575386,0.08776229662596365,0.08787620677719742,0.08885602257060451,0.09106359638534783,0.09418517363781742,0.09288906205977382,0.09605068248165075,0.09958029332992092,0.1017720737994818,0.10426379776551742,0.10575351754002113,0.10729595121972323,0.10983932968160214,0.1115424946146999,0.11319337012564919,0.11209011170809005,0.11569119076984047,0.11539378717310808,0.11617969608943186,0.11992398879873321,0.12058958892402054,0.12058148516128285,0.1243533949682646,0.12397355630882875,0.1249291530914942,0.12417071332522645,0.127729019935295,0.13038231576737544,0.13138831006662502,0.1299873901637967,0.13214710202038318,0.12992959120542538,0.13079826392836127,0.12602614431096823,0.12730214211915367,0.12832343307699243,0.1306229875768168,0.12842656680669143,0.13538260273309746,0.13406212115089988,0.13410839495649465,0.13549966695887655,0.13846552817315563,0.13671105296527872,0.14688766411781562],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=qwen-7b@vllm-0.2.5\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"qwen-7b@vllm-0.2.5\",\"line\":{\"color\":\"#EF553B\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"qwen-7b@vllm-0.2.5\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.028442294297625497,0.029251332000985793,0.0296144977343698,0.02998198482494244,0.030392584144347413,0.030641278966825267,0.030994415872635933,0.031316432361286825,0.03168420626233855,0.031985274728567166,0.032282865457177436,0.03285161723525935,0.033066337407720345,0.03338753762171061,0.03366404034124792,0.034018641029817207,0.03564711067292859,0.03594564229874779,0.0363537086547155,0.03661003702246473,0.03712705525645684,0.036398301877873095,0.037165337461060585,0.037506000414368315,0.037972914749402585,0.03850560737237437,0.03883871254215588,0.03941514644819476,0.04003846746479522,0.04074105166724794,0.04129751943337703,0.04189716042214032,0.04254990799026965,0.04327542919758816,0.04495658450500199,0.04584433322225246,0.04706591438592847,0.04773741712261287,0.04839975873517459,0.05016672665407648,0.0508641275828388,0.05176487969109283,0.05245137853458329,0.05373140662073632,0.05524508917934316,0.05626891707143307,0.056848094477726395,0.05824307145496063,0.058914048132600656,0.0594761288555523,0.06047502358286252,0.06053221981391659,0.061672584150830045,0.06323171237389494,0.06457120542152398,0.06469886519219542,0.065054751444771,0.06446584616842088,0.06570014669418664,0.06581771657892638,0.06734528617589579,0.06801440238547708,0.0685236961067285,0.06896610677069746,0.0697690944730954,0.07034659298472698,0.07262447818662139,0.07291755583832299,0.07371514234272862,0.07370661752391487,0.07433022907895426,0.07531361151444974,0.07661130676936026,0.0782530333188607,0.07982810159577479,0.08013616335216524,0.08071795300185215,0.08091390090341405,0.08236139298307682,0.08446799787791662,0.08553203801562739,0.08620813325439595,0.08831508824295446,0.0887748490170395,0.08945021813818578,0.09065257817264846,0.09325000311152755,0.09166027588074727,0.09263147876932198,0.09291809305170799,0.09379635403123693,0.09437508561947051,0.0959004555083452,0.0937824779859813,0.09404378391575326,0.09357691125621852,0.09632371558971388,0.09686127809168865,0.09722713541111241,0.09409318509509648,0.09403281367466873,0.09357322215770239,0.09435972112345407,0.09613270985858705,0.09846086267582999,0.09479615594211946,0.09547598439025759,0.09540756234221492,0.09758928167988287,0.09745679758930215,0.10091543492884683,0.09921827237516588,0.09956801323351826,0.10045996285981675,0.10183501060564366,0.1032627949169306,0.10332584209351528,0.10238639029567309,0.10480984578352272,0.10787931696333078,0.10829029432616737,0.10715379255092279,0.10549178669897667,0.1087587000079894,0.11013809456842835,0.1141960479883138,0.11085035270069485,0.11061829963454177],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=qwen-7b-int4@vllm-2acd76f\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"qwen-7b-int4@vllm-2acd76f\",\"line\":{\"color\":\"#00cc96\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"qwen-7b-int4@vllm-2acd76f\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.011328733096767652,0.012295940237952172,0.012913652161704267,0.013500276015664543,0.013952679513266921,0.015573866504962652,0.01696090940530604,0.018374710009787222,0.025445858571387203,0.02649851524642658,0.027235670381244286,0.027983253155618736,0.028669619287627235,0.030230866390741427,0.03176652947371632,0.030356354088291217,0.03742148438213851,0.03854242022793125,0.039181082655653475,0.03979952719812276,0.04053300655603579,0.041891688496417105,0.04341314212978128,0.04179732663378608,0.04890836812241426,0.049887239767686845,0.05054635656971237,0.051213676518690944,0.051803872306364,0.05342816937953117,0.054861140451738906,0.05337437923649241,0.06045001661249973,0.06167969983871537,0.06255362020026735,0.06319460327892043,0.0637891809401309,0.06527637949773717,0.06660932953431367,0.06517023874393578,0.07196058535186012,0.07263711760534446,0.07349289553388912,0.07433726760727087,0.07492983035072587,0.07621608065939266,0.07756056746612042,0.07619839272557122,0.08333093308080756,0.08452762338446754,0.0734195635115067,0.07383074635593782,0.07423119225473065,0.0745461050590555,0.07481206968931826,0.07516905189745274,0.07502650838750102,0.07512908362587922,0.07541389190713303,0.07566779579562383,0.07593346816566772,0.07664394925065982,0.07720952455132746,0.07733311307806219,0.07950962749502444,0.07985318493592333,0.08041934678477629,0.080617810081605,0.08094722158048087,0.08133942389618729,0.0816029216865573,0.08268023731488632,0.08345653454249614,0.08384520796072228,0.08440879419001364,0.08482018883816271,0.08515776040488748,0.08565911904004557,0.08605763982613723,0.08576142759588827,0.08621028853782146,0.08671965231774974,0.08740136489734111,0.08771661387816948,0.08932687123890191,0.08927682602822412,0.0902069612230371,0.09044565081785133,0.09091285812404179,0.09116055393788858,0.09205606415554879,0.09238510408592324,0.09275150162392054,0.09269617315377446,0.09334888310420758,0.09386383658149966,0.09389720263889978,0.09406207750633597,0.09509933515210056,0.09483468926711044,0.09551817921878242,0.09579841374858789,0.09620776767952025,0.09684676692866268,0.09709491189547197,0.098792676784246,0.09938375345774632,0.09966145993984335,0.09990721182717795,0.10052288620247106,0.10038426685236726,0.10130382522780743,0.10236814052389047,0.10285690481272956,0.10357836500999466,0.10319934726671427,0.10423645927717247,0.10463483704243949,0.10489303327549807,0.10582853614688321,0.10615609096824834,0.10668105650137956,0.10505734280428942,0.10557305535902362,0.10603591256771161,0.10603842521300264,0.10724409192976028,0.10721985988629308],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"batch\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"avg_token_latency\"}},\"legend\":{\"title\":{\"text\":\"type\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Qwen 7B Average non-first Token Latency\"}},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('193aeaf0-bd9d-4cbf-bb76-bc7be013b95e');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                });            </script>        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "hovertemplate": "type=qwen-7b-awq@vllm-0.2.5<br>batch=%{x}<br>avg_first_token_latency=%{y}<extra></extra>",
         "legendgroup": "qwen-7b-awq@vllm-0.2.5",
         "line": {
          "color": "#636efa",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "qwen-7b-awq@vllm-0.2.5",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.02378948636363599,
          0.03758339269970747,
          0.050399235566631735,
          0.06763743820019955,
          0.08515849157758769,
          0.10682501425900329,
          0.12231751690457698,
          0.13940968506959608,
          0.16319674686110375,
          0.19026836712473597,
          0.18833509193171433,
          0.20758024647924836,
          0.22648213051927366,
          0.24564708412238973,
          0.25994985610485033,
          0.2760877979642698,
          0.30349078915694955,
          0.3314279936944499,
          0.32655650528948554,
          0.3334237917416734,
          0.35091849556331345,
          0.3865387301891105,
          0.3825245943043505,
          0.4273718530001133,
          0.40634656559166615,
          0.42624814694611884,
          0.44296822796311097,
          0.4518827997929942,
          0.4926327225103611,
          0.49204222615347437,
          0.5129624485998413,
          0.552022788381214,
          0.5990731500150464,
          0.5435459704707521,
          0.5618304676356795,
          0.5663461701666519,
          0.607582765648486,
          0.6555503765920501,
          0.6281014823654494,
          0.672635313925116,
          0.6532643732072491,
          0.674588069577177,
          0.6802411775058931,
          0.6981535433409053,
          0.8853042333887869,
          0.8964909509940125,
          1.0817017891446372,
          1.3303979391596386,
          1.37825091920396,
          1.4980952971800434,
          1.6273130373921327,
          1.7094686443141864,
          1.8428551197233123,
          1.9393012531542841,
          1.9633762643111257,
          2.161536519841542,
          2.273397866603595,
          2.2763359447485283,
          2.4525550972155945,
          2.468840503365248,
          2.6528560193134996,
          2.6042385693977446,
          2.7007696544503674,
          2.822619479592927,
          2.853573473045935,
          2.2984972406287247,
          2.4570166631717676,
          2.318113227058829,
          2.4986561862101486,
          2.538989793064372,
          2.59003992778874,
          2.5751577203681233,
          2.597893237561674,
          2.6300425704728303,
          2.6907576161863833,
          2.5860156841974073,
          2.7021400218635048,
          2.681337209359182,
          2.6789642003163805,
          2.7890837855812607,
          2.6783513051728214,
          2.7377268810914757,
          2.9668006083674223,
          2.779166347541637,
          2.944037243941142,
          2.8199566887383827,
          2.9920333743334124,
          2.978717705835069,
          2.9511427384437168,
          3.334795503712523,
          2.9199048665933924,
          3.103342304824764,
          3.101065534592378,
          3.449133191803347,
          3.275224268394782,
          3.8263634528106754,
          3.6007443658763805,
          3.605070569716534,
          4.328681564288242,
          3.6254420942034584,
          3.971558502339231,
          4.243334124864658,
          4.025446273072779,
          4.294617090976023,
          4.406774614684288,
          4.085382904714397,
          4.740579865804599,
          4.404527827114859,
          4.811532015458397,
          4.71160625405814,
          4.993741539924806,
          5.115546688624406,
          5.489651456323724,
          5.562980506524213,
          6.566040575643822,
          6.647926454561743,
          7.341724233329759,
          7.213195004350264,
          7.468918741915658,
          7.267900180880753,
          7.4138875719547315,
          7.48197917969033,
          7.626333884838762,
          6.733426623010091,
          7.597991371760817,
          6.986378261393223,
          7.302733505300612,
          7.487792416358189
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=qwen-7b@vllm-0.2.5<br>batch=%{x}<br>avg_first_token_latency=%{y}<extra></extra>",
         "legendgroup": "qwen-7b@vllm-0.2.5",
         "line": {
          "color": "#EF553B",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "qwen-7b@vllm-0.2.5",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.031810265577708684,
          0.04440722044091672,
          0.06122555901917318,
          0.06943323034405087,
          0.08287470294162631,
          0.10794483465142549,
          0.10838048307757292,
          0.12018317873589694,
          0.1303465930124124,
          0.14134804462082684,
          0.1990855479443615,
          0.17071464498682568,
          0.17710952537420852,
          0.19201604697986374,
          0.2133367525351544,
          0.21789777628146112,
          0.2261059064527645,
          0.2317865851180007,
          0.24191603943411458,
          0.2995646928763017,
          0.272271081422557,
          0.3386305342682383,
          0.40698820533797,
          0.489618966797578,
          0.5559534961357713,
          0.5960747028763692,
          0.6181044476839749,
          0.646865295050832,
          0.682447721683902,
          0.7053586227482795,
          0.757117941432322,
          0.7664465815533044,
          0.8055783713422716,
          0.8068813868625047,
          0.8434652067678234,
          0.8783370760380665,
          0.932223428485708,
          0.9734955481557257,
          1.2441825454256363,
          1.2218790140534213,
          1.254375414778318,
          1.2703619494504768,
          1.4156908757705875,
          1.4300565721020355,
          1.380244151917255,
          1.3681290081779782,
          1.4218620631200005,
          1.5615004577072493,
          1.6002004019928728,
          1.7171115253617366,
          1.842487146637599,
          1.956803862886453,
          1.9977899844822442,
          1.9628834914183244,
          2.211833068552953,
          2.4328640714080096,
          2.7265305453476807,
          3.0353684039215647,
          3.413679969058936,
          3.3908524472350017,
          3.3988946979324663,
          3.547448365807894,
          3.652025666577407,
          3.7033343622283,
          3.728609863424208,
          3.811337077125375,
          3.8592557645931747,
          3.9603852538455184,
          4.000056747903638,
          4.086988583901389,
          4.3628995832902975,
          4.259504738374657,
          4.382698440499714,
          4.62756237237426,
          4.586842199434807,
          4.8774212554437355,
          5.043866185683581,
          5.245160399976987,
          5.291479175891403,
          5.040938314946149,
          5.047190111250296,
          5.216058209881578,
          4.837702956197826,
          5.072683532135508,
          5.036510686807313,
          5.206998335519345,
          5.229371073939343,
          5.226839628282257,
          5.255396921098693,
          5.3665893810180325,
          5.428607477404812,
          5.82368431377994,
          6.049615147618717,
          6.290660352302665,
          6.406176892138938,
          6.767492546022615,
          7.033107649738361,
          7.38715892282152,
          7.729292209051843,
          7.845216443735223,
          8.003077552782392,
          8.453537985536968,
          8.446848405962239,
          8.62772799232661,
          9.002202052770334,
          8.937140856757198,
          9.24766069317123,
          9.47901287397379,
          9.922102969160875,
          9.6601266169536,
          9.930584688134509,
          9.895647759528675,
          9.727093003256538,
          9.82496157006872,
          10.191341544844482,
          9.896528246081411,
          10.18784079282288,
          10.322332080748328,
          10.205061898413048,
          10.720681912347374,
          10.972046471945502,
          10.414499842943975,
          10.137098516514866,
          10.385736234867686,
          10.782757034659559,
          11.258303562551312,
          10.667834541869826,
          10.353239321109852
         ],
         "yaxis": "y"
        },
        {
         "hovertemplate": "type=qwen-7b-int4@vllm-2acd76f<br>batch=%{x}<br>avg_first_token_latency=%{y}<extra></extra>",
         "legendgroup": "qwen-7b-int4@vllm-2acd76f",
         "line": {
          "color": "#00cc96",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "qwen-7b-int4@vllm-2acd76f",
         "orientation": "v",
         "showlegend": true,
         "type": "scatter",
         "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128
         ],
         "xaxis": "x",
         "y": [
          0.04328242675675189,
          0.06537372471939307,
          0.08867632606998085,
          0.09933232509841522,
          0.10877972976969821,
          0.12466265267143264,
          0.13054218699246467,
          0.13935647654579952,
          0.15181969401116172,
          0.1591344366548583,
          0.17874107613773427,
          0.19310569566386265,
          0.19872732941511567,
          0.20889799382860716,
          0.23044143057379282,
          0.23456803958314204,
          0.2545419292405759,
          0.2762107770211752,
          0.29502396920303764,
          0.326564709902741,
          0.3209037498260538,
          0.34934645890766247,
          0.3387522675744865,
          0.37257894173574946,
          0.4005434444267303,
          0.4110935859095592,
          0.421355220635801,
          0.4324585402916585,
          0.5105103840770456,
          0.4973722826029795,
          0.47071138608284413,
          0.479784803421353,
          0.5167885699140077,
          0.506951004795422,
          0.5278310521990061,
          0.5516106674188107,
          0.586575272163221,
          0.6274702601407751,
          0.7302363977088766,
          0.6439687321606146,
          0.668139951394462,
          0.7208893177106918,
          0.7213258431978928,
          0.7857524714305658,
          0.8444196599777098,
          0.7616578595699284,
          0.7999248538291782,
          0.8026183286500681,
          0.7861844321022279,
          0.8348553822779408,
          0.8767285587911318,
          0.8489686884374047,
          0.9257511878737582,
          0.8636802613712203,
          0.9014968124726279,
          0.8981943377958876,
          0.9591390051176412,
          0.9585895421031606,
          0.955902483056452,
          0.9911463704374102,
          1.0116779711428276,
          1.0068579594209108,
          0.9988998456306204,
          1.0539161130169912,
          1.0953788964149471,
          1.1983131206937774,
          1.0569779580547034,
          1.078606763603531,
          1.0818772295912826,
          1.140068473043807,
          1.161682403362203,
          1.1578370751640588,
          1.2808445770374275,
          1.1940404779575597,
          1.2140942395602663,
          1.1910714371264668,
          1.2564268702947539,
          1.2407349111973984,
          1.400398707832701,
          1.3573431662186826,
          1.4751685474391356,
          1.4876913450739473,
          1.5247375872139455,
          1.5610176187117524,
          1.4760383677782742,
          1.5705299561336767,
          1.7471190682591193,
          1.7166294879915023,
          1.8188072931021453,
          1.8073002990600098,
          1.8904474102935502,
          1.92960654696266,
          1.9952349900136845,
          2.0649642894412716,
          2.079378737709507,
          2.1145299728893905,
          2.200828222920844,
          2.203186353262446,
          2.183969506095055,
          2.3843513120922983,
          2.376533005363094,
          2.396059893320726,
          2.570609396701241,
          2.5085753501222756,
          2.51903467493688,
          2.7339308160414286,
          2.6667006347970865,
          2.595514269762702,
          2.698548835658389,
          2.7106096472802825,
          2.8218636578641125,
          2.740567623953601,
          2.816074144612355,
          2.769686520792058,
          2.8125634641346076,
          2.938706497174124,
          3.0149125014491474,
          2.960229872392443,
          3.0139487689740547,
          2.927358562295558,
          3.0287718244112476,
          3.107587920043801,
          3.068297037334234,
          3.1483085570770557,
          3.28404685186781,
          3.04640346314455,
          3.244430391374594,
          3.231409094789342
         ],
         "yaxis": "y"
        }
       ],
       "layout": {
        "legend": {
         "title": {
          "text": "type"
         },
         "tracegroupgap": 0
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "Qwen 7B Average First Token Latency"
        },
        "xaxis": {
         "anchor": "y",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "batch"
         }
        },
        "yaxis": {
         "anchor": "x",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "avg_first_token_latency"
         }
        }
       }
      },
      "text/html": [
       "<div>                            <div id=\"b00fe38e-67a9-442d-b3b8-63788ae87163\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"b00fe38e-67a9-442d-b3b8-63788ae87163\")) {                    Plotly.newPlot(                        \"b00fe38e-67a9-442d-b3b8-63788ae87163\",                        [{\"hovertemplate\":\"type=qwen-7b-awq@vllm-0.2.5\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_first_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"qwen-7b-awq@vllm-0.2.5\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"qwen-7b-awq@vllm-0.2.5\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.02378948636363599,0.03758339269970747,0.050399235566631735,0.06763743820019955,0.08515849157758769,0.10682501425900329,0.12231751690457698,0.13940968506959608,0.16319674686110375,0.19026836712473597,0.18833509193171433,0.20758024647924836,0.22648213051927366,0.24564708412238973,0.25994985610485033,0.2760877979642698,0.30349078915694955,0.3314279936944499,0.32655650528948554,0.3334237917416734,0.35091849556331345,0.3865387301891105,0.3825245943043505,0.4273718530001133,0.40634656559166615,0.42624814694611884,0.44296822796311097,0.4518827997929942,0.4926327225103611,0.49204222615347437,0.5129624485998413,0.552022788381214,0.5990731500150464,0.5435459704707521,0.5618304676356795,0.5663461701666519,0.607582765648486,0.6555503765920501,0.6281014823654494,0.672635313925116,0.6532643732072491,0.674588069577177,0.6802411775058931,0.6981535433409053,0.8853042333887869,0.8964909509940125,1.0817017891446372,1.3303979391596386,1.37825091920396,1.4980952971800434,1.6273130373921327,1.7094686443141864,1.8428551197233123,1.9393012531542841,1.9633762643111257,2.161536519841542,2.273397866603595,2.2763359447485283,2.4525550972155945,2.468840503365248,2.6528560193134996,2.6042385693977446,2.7007696544503674,2.822619479592927,2.853573473045935,2.2984972406287247,2.4570166631717676,2.318113227058829,2.4986561862101486,2.538989793064372,2.59003992778874,2.5751577203681233,2.597893237561674,2.6300425704728303,2.6907576161863833,2.5860156841974073,2.7021400218635048,2.681337209359182,2.6789642003163805,2.7890837855812607,2.6783513051728214,2.7377268810914757,2.9668006083674223,2.779166347541637,2.944037243941142,2.8199566887383827,2.9920333743334124,2.978717705835069,2.9511427384437168,3.334795503712523,2.9199048665933924,3.103342304824764,3.101065534592378,3.449133191803347,3.275224268394782,3.8263634528106754,3.6007443658763805,3.605070569716534,4.328681564288242,3.6254420942034584,3.971558502339231,4.243334124864658,4.025446273072779,4.294617090976023,4.406774614684288,4.085382904714397,4.740579865804599,4.404527827114859,4.811532015458397,4.71160625405814,4.993741539924806,5.115546688624406,5.489651456323724,5.562980506524213,6.566040575643822,6.647926454561743,7.341724233329759,7.213195004350264,7.468918741915658,7.267900180880753,7.4138875719547315,7.48197917969033,7.626333884838762,6.733426623010091,7.597991371760817,6.986378261393223,7.302733505300612,7.487792416358189],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=qwen-7b@vllm-0.2.5\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_first_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"qwen-7b@vllm-0.2.5\",\"line\":{\"color\":\"#EF553B\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"qwen-7b@vllm-0.2.5\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.031810265577708684,0.04440722044091672,0.06122555901917318,0.06943323034405087,0.08287470294162631,0.10794483465142549,0.10838048307757292,0.12018317873589694,0.1303465930124124,0.14134804462082684,0.1990855479443615,0.17071464498682568,0.17710952537420852,0.19201604697986374,0.2133367525351544,0.21789777628146112,0.2261059064527645,0.2317865851180007,0.24191603943411458,0.2995646928763017,0.272271081422557,0.3386305342682383,0.40698820533797,0.489618966797578,0.5559534961357713,0.5960747028763692,0.6181044476839749,0.646865295050832,0.682447721683902,0.7053586227482795,0.757117941432322,0.7664465815533044,0.8055783713422716,0.8068813868625047,0.8434652067678234,0.8783370760380665,0.932223428485708,0.9734955481557257,1.2441825454256363,1.2218790140534213,1.254375414778318,1.2703619494504768,1.4156908757705875,1.4300565721020355,1.380244151917255,1.3681290081779782,1.4218620631200005,1.5615004577072493,1.6002004019928728,1.7171115253617366,1.842487146637599,1.956803862886453,1.9977899844822442,1.9628834914183244,2.211833068552953,2.4328640714080096,2.7265305453476807,3.0353684039215647,3.413679969058936,3.3908524472350017,3.3988946979324663,3.547448365807894,3.652025666577407,3.7033343622283,3.728609863424208,3.811337077125375,3.8592557645931747,3.9603852538455184,4.000056747903638,4.086988583901389,4.3628995832902975,4.259504738374657,4.382698440499714,4.62756237237426,4.586842199434807,4.8774212554437355,5.043866185683581,5.245160399976987,5.291479175891403,5.040938314946149,5.047190111250296,5.216058209881578,4.837702956197826,5.072683532135508,5.036510686807313,5.206998335519345,5.229371073939343,5.226839628282257,5.255396921098693,5.3665893810180325,5.428607477404812,5.82368431377994,6.049615147618717,6.290660352302665,6.406176892138938,6.767492546022615,7.033107649738361,7.38715892282152,7.729292209051843,7.845216443735223,8.003077552782392,8.453537985536968,8.446848405962239,8.62772799232661,9.002202052770334,8.937140856757198,9.24766069317123,9.47901287397379,9.922102969160875,9.6601266169536,9.930584688134509,9.895647759528675,9.727093003256538,9.82496157006872,10.191341544844482,9.896528246081411,10.18784079282288,10.322332080748328,10.205061898413048,10.720681912347374,10.972046471945502,10.414499842943975,10.137098516514866,10.385736234867686,10.782757034659559,11.258303562551312,10.667834541869826,10.353239321109852],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"type=qwen-7b-int4@vllm-2acd76f\\u003cbr\\u003ebatch=%{x}\\u003cbr\\u003eavg_first_token_latency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"qwen-7b-int4@vllm-2acd76f\",\"line\":{\"color\":\"#00cc96\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"qwen-7b-int4@vllm-2acd76f\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128],\"xaxis\":\"x\",\"y\":[0.04328242675675189,0.06537372471939307,0.08867632606998085,0.09933232509841522,0.10877972976969821,0.12466265267143264,0.13054218699246467,0.13935647654579952,0.15181969401116172,0.1591344366548583,0.17874107613773427,0.19310569566386265,0.19872732941511567,0.20889799382860716,0.23044143057379282,0.23456803958314204,0.2545419292405759,0.2762107770211752,0.29502396920303764,0.326564709902741,0.3209037498260538,0.34934645890766247,0.3387522675744865,0.37257894173574946,0.4005434444267303,0.4110935859095592,0.421355220635801,0.4324585402916585,0.5105103840770456,0.4973722826029795,0.47071138608284413,0.479784803421353,0.5167885699140077,0.506951004795422,0.5278310521990061,0.5516106674188107,0.586575272163221,0.6274702601407751,0.7302363977088766,0.6439687321606146,0.668139951394462,0.7208893177106918,0.7213258431978928,0.7857524714305658,0.8444196599777098,0.7616578595699284,0.7999248538291782,0.8026183286500681,0.7861844321022279,0.8348553822779408,0.8767285587911318,0.8489686884374047,0.9257511878737582,0.8636802613712203,0.9014968124726279,0.8981943377958876,0.9591390051176412,0.9585895421031606,0.955902483056452,0.9911463704374102,1.0116779711428276,1.0068579594209108,0.9988998456306204,1.0539161130169912,1.0953788964149471,1.1983131206937774,1.0569779580547034,1.078606763603531,1.0818772295912826,1.140068473043807,1.161682403362203,1.1578370751640588,1.2808445770374275,1.1940404779575597,1.2140942395602663,1.1910714371264668,1.2564268702947539,1.2407349111973984,1.400398707832701,1.3573431662186826,1.4751685474391356,1.4876913450739473,1.5247375872139455,1.5610176187117524,1.4760383677782742,1.5705299561336767,1.7471190682591193,1.7166294879915023,1.8188072931021453,1.8073002990600098,1.8904474102935502,1.92960654696266,1.9952349900136845,2.0649642894412716,2.079378737709507,2.1145299728893905,2.200828222920844,2.203186353262446,2.183969506095055,2.3843513120922983,2.376533005363094,2.396059893320726,2.570609396701241,2.5085753501222756,2.51903467493688,2.7339308160414286,2.6667006347970865,2.595514269762702,2.698548835658389,2.7106096472802825,2.8218636578641125,2.740567623953601,2.816074144612355,2.769686520792058,2.8125634641346076,2.938706497174124,3.0149125014491474,2.960229872392443,3.0139487689740547,2.927358562295558,3.0287718244112476,3.107587920043801,3.068297037334234,3.1483085570770557,3.28404685186781,3.04640346314455,3.244430391374594,3.231409094789342],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"batch\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"avg_first_token_latency\"}},\"legend\":{\"title\":{\"text\":\"type\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Qwen 7B Average First Token Latency\"}},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('b00fe38e-67a9-442d-b3b8-63788ae87163');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                });            </script>        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "qwen =  pl.concat([\n",
    "    pl.from_records(json.load(open('result/2023_12_14_23_59_00_qwen_vllm.json'))['results']).drop('raw_ticks').sort('batch').with_columns(type=pl.lit('qwen-7b-awq@vllm-0.2.5')),\n",
    "    pl.from_records(json.load(open('result/2023_12_16_17_55_20_qwen_vllm.json'))['results']).drop('raw_ticks').sort('batch').with_columns(type=pl.lit('qwen-7b@vllm-0.2.5')),\n",
    "    pl.from_records(json.load(open('result/2023_12_17_01_14_42_qwen_vllm.json'))['results']).drop('raw_ticks').sort('batch').with_columns(type=pl.lit('qwen-7b-int4@vllm-2acd76f')),\n",
    "])\n",
    "titles = {\n",
    "    'token_per_s': 'Throughput',\n",
    "    'avg_token_latency': 'Average non-first Token Latency',\n",
    "    'avg_first_token_latency': 'Average First Token Latency',\n",
    "}\n",
    "\n",
    "for i, title in titles.items():\n",
    "    fig = px.line(qwen, x='batch', y=i, color='type', title=f'Qwen 7B {title}')\n",
    "    fig.write_image(f'images/qwen-7b-{i}.png', scale=2, width=1440)\n",
    "    fig.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}