paper-figures.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5be93e3b",
   "metadata": {},
   "source": [
    "# Graphs Used in the Shared Task Challenge Paper\n",
    "\n",
    "Authors: Hadi Asghari & Freya Hewett\n",
    "Date: July 2022\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "82aea0c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "c0f1e9da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['idx', 'sentence', 'label']\n"
     ]
    }
   ],
   "source": [
    "# Load Training Data\n",
    "score = []\n",
    "sentence = []\n",
    "with open(\"data_training.csv\", 'r', encoding='ISO-8859-1') as file:\n",
    "    reader = csv.reader(file)\n",
    "    for i, row in enumerate(reader):\n",
    "        if i == 0:\n",
    "            print(row)\n",
    "            continue # skip header\n",
    "        sentence.append(row[1])\n",
    "        score.append(float(row[2]))  # 5 on detailed dataset\n",
    "        \n",
    "score = np.array(score)\n",
    "assert len(score) == len(sentence)  # sanity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "f171a6ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score range: 1.0 6.333333333\n",
      "basic baseline: 3.015705657663 1.1802792931882462\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEGCAYAAACevtWaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAQsUlEQVR4nO3df6xfdX3H8edLytChE5Rr07XNLnGdC5pR8AYxGMMkKD+MYLKxsg2JISl/4AKZmxb+Qc3YaqKymWwklTLLhnSMH6ET4mTIwvxD9BYrP1qJFUpoU+hVREA3HfW9P+5hfi23vfd+v/f22/vp85GcfM/5nM85533S5HVPP99zzjdVhSSpLa8adgGSpLlnuEtSgwx3SWqQ4S5JDTLcJalBi4ZdAMBxxx1Xo6Ojwy5DkhaUzZs3/6CqRqZad0iE++joKOPj48MuQ5IWlCRP7m+dwzKS1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktSgaZ9QTfJq4H7gqK7/rVV1dZLjgY3AG4HNwEVV9fMkRwE3Am8Hfgj8UVXtmKf6dYgaXXPXsEuYMzvWnjvsEqRZm8mV+8+A91TVicBK4KwkpwKfBq6tqt8GfgRc0vW/BPhR135t10+SdBBNG+416cVu8chuKuA9wK1d+wbg/G7+vG6Zbv0ZSTJXBUuSpjejMfckRyTZAuwB7gG+DzxXVS91XXYCS7v5pcBTAN36HzM5dLPvPlcnGU8yPjExMdBJSJJ+1YzCvar2VtVKYBlwCvC7gx64qtZV1VhVjY2MTPnGSklSn2Z1t0xVPQfcB7wTOCbJy1/ILgN2dfO7gOUA3frXM/nFqiTpIJk23JOMJDmmm38NcCawjcmQ/4Ou28XAnd38pm6Zbv3XqqrmsGZJ0jRm8mMdS4ANSY5g8o/BLVX15SRbgY1J/gr4NrC+678e+Kck24FngVXzULck6QCmDfeqegg4aYr2x5kcf9+3/X+AP5yT6iRJffEJVUlqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIatGjYBeiXRtfcNewSJDXCK3dJatC04Z5keZL7kmxN8miSy7v2TyTZlWRLN53Ts82VSbYneSzJ++bzBCRJrzSTYZmXgI9W1YNJXgdsTnJPt+7aqvpMb+ckJwCrgLcCvwn8R5Lfqaq9c1m4JGn/pr1yr6rdVfVgN/8CsA1YeoBNzgM2VtXPquoJYDtwylwUK0mamVmNuScZBU4CHuiaPpLkoSQ3JDm2a1sKPNWz2U6m+GOQZHWS8STjExMTs69ckrRfMw73JK8FbgOuqKrngeuANwMrgd3AZ2dz4KpaV1VjVTU2MjIym00lSdOYUbgnOZLJYL+pqm4HqKpnqmpvVf0C+AK/HHrZBSzv2XxZ1yZJOkhmcrdMgPXAtqr6XE/7kp5uHwQe6eY3AauSHJXkeGAF8M25K1mSNJ2Z3C1zGnAR8HCSLV3bVcCFSVYCBewALgWoqkeT3AJsZfJOm8u8U0aSDq5pw72qvg5kilV3H2Cba4BrBqhLkjQAn1CVpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAZNG+5Jlie5L8nWJI8mubxrf0OSe5J8r/s8tmtPks8n2Z7koSQnz/dJSJJ+1Uyu3F8CPlpVJwCnApclOQFYA9xbVSuAe7tlgLOBFd20GrhuzquWJB3Qouk6VNVuYHc3/0KSbcBS4Dzg9K7bBuA/gY937TdWVQHfSHJMkiXdfubc6Jq75mO3krSgzWrMPckocBLwALC4J7CfBhZ380uBp3o229m17buv1UnGk4xPTEzMtm5J0gHMONyTvBa4Dbiiqp7vXdddpddsDlxV66pqrKrGRkZGZrOpJGkaMwr3JEcyGew3VdXtXfMzSZZ065cAe7r2XcDyns2XdW2SpINkJnfLBFgPbKuqz/Ws2gRc3M1fDNzZ0/6h7q6ZU4Efz9d4uyRpatN+oQqcBlwEPJxkS9d2FbAWuCXJJcCTwAXduruBc4DtwE+BD89lwZKk6c3kbpmvA9nP6jOm6F/AZQPWJUkagE+oSlKDDHdJatBMxtylw1orD8rtWHvusEvQQeSVuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJatC04Z7khiR7kjzS0/aJJLuSbOmmc3rWXZlke5LHkrxvvgqXJO3fTK7cvwicNUX7tVW1spvuBkhyArAKeGu3zT8kOWKuipUkzcy04V5V9wPPznB/5wEbq+pnVfUEsB04ZYD6JEl9GGTM/SNJHuqGbY7t2pYCT/X02dm1vUKS1UnGk4xPTEwMUIYkaV/9hvt1wJuBlcBu4LOz3UFVrauqsaoaGxkZ6bMMSdJU+gr3qnqmqvZW1S+AL/DLoZddwPKersu6NknSQdRXuCdZ0rP4QeDlO2k2AauSHJXkeGAF8M3BSpQkzdai6TokuRk4HTguyU7gauD0JCuBAnYAlwJU1aNJbgG2Ai8Bl1XV3nmpXJK0X9OGe1VdOEXz+gP0vwa4ZpCiJEmD8QlVSWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWrQtOGe5IYke5I80tP2hiT3JPle93ls154kn0+yPclDSU6ez+IlSVObyZX7F4Gz9mlbA9xbVSuAe7tlgLOBFd20GrhubsqUJM3GtOFeVfcDz+7TfB6woZvfAJzf035jTfoGcEySJXNUqyRphvodc19cVbu7+aeBxd38UuCpnn47u7ZXSLI6yXiS8YmJiT7LkCRNZeAvVKuqgOpju3VVNVZVYyMjI4OWIUnq0W+4P/PycEv3uadr3wUs7+m3rGuTJB1E/Yb7JuDibv5i4M6e9g91d82cCvy4Z/hGknSQLJquQ5KbgdOB45LsBK4G1gK3JLkEeBK4oOt+N3AOsB34KfDheahZkjSNacO9qi7cz6ozpuhbwGWDFiVJGoxPqEpSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgxYNsnGSHcALwF7gpaoaS/IG4F+AUWAHcEFV/WiwMiVJszEXV+6/X1Urq2qsW14D3FtVK4B7u2VJ0kE0H8My5wEbuvkNwPnzcAxJ0gEMGu4FfDXJ5iSru7bFVbW7m38aWDzVhklWJxlPMj4xMTFgGZKkXgONuQPvqqpdSd4E3JPku70rq6qS1FQbVtU6YB3A2NjYlH0kSf0Z6Mq9qnZ1n3uAO4BTgGeSLAHoPvcMWqQkaXb6vnJPcjTwqqp6oZt/L/ApYBNwMbC2+7xzLgqVNJjRNXcNu4Q5s2PtucMu4ZA3yLDMYuCOJC/v50tV9ZUk3wJuSXIJ8CRwweBlSpJmo+9wr6rHgROnaP8hcMYgRUmSBuMTqpLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUoEXzteMkZwF/BxwBXF9Va+frWJIOL6Nr7hp2CXNmx9pz52W/83LlnuQI4O+Bs4ETgAuTnDAfx5IkvdJ8DcucAmyvqser6ufARuC8eTqWJGkf8zUssxR4qmd5J/CO3g5JVgOru8UXkzzW57GOA37Q57aHGs/l0NTKubRyHtDQueTTA53Lb+1vxbyNuU+nqtYB6wbdT5Lxqhqbg5KGznM5NLVyLq2cB3guMzFfwzK7gOU9y8u6NknSQTBf4f4tYEWS45P8GrAK2DRPx5Ik7WNehmWq6qUkHwH+nclbIW+oqkfn41jMwdDOIcRzOTS1ci6tnAd4LtNKVc3HfiVJQ+QTqpLUIMNdkhq0YMM9yQ1J9iR5ZNi1DCrJ8iT3Jdma5NEklw+7pn4keXWSbyb5Tncenxx2TYNKckSSbyf58rBrGUSSHUkeTrIlyfiw6xlEkmOS3Jrku0m2JXnnsGuarSRv6f4tXp6eT3LFnB5joY65J3k38CJwY1W9bdj1DCLJEmBJVT2Y5HXAZuD8qto65NJmJUmAo6vqxSRHAl8HLq+qbwy5tL4l+XNgDPiNqnr/sOvpV5IdwFhVLfgHf5JsAP6rqq7v7sb79ap6bshl9a17Xcsu4B1V9eRc7XfBXrlX1f3As8OuYy5U1e6qerCbfwHYxuRTvgtKTXqxWzyymxbm1QOQZBlwLnD9sGvRpCSvB94NrAeoqp8v5GDvnAF8fy6DHRZwuLcqyShwEvDAkEvpSzeMsQXYA9xTVQvyPDp/C3wM+MWQ65gLBXw1yebu1R8L1fHABPCP3XDZ9UmOHnZRA1oF3DzXOzXcDyFJXgvcBlxRVc8Pu55+VNXeqlrJ5FPJpyRZkENmSd4P7KmqzcOuZY68q6pOZvJNrZd1w5oL0SLgZOC6qjoJ+AmwZrgl9a8bVvoA8K9zvW/D/RDRjVHfBtxUVbcPu55Bdf9Vvg84a8il9Os04APdWPVG4D1J/nm4JfWvqnZ1n3uAO5h8c+tCtBPY2fM/wluZDPuF6mzgwap6Zq53bLgfArovItcD26rqc8Oup19JRpIc082/BjgT+O5Qi+pTVV1ZVcuqapTJ/zZ/rar+dMhl9SXJ0d0X9XRDGO8FFuRdZlX1NPBUkrd0TWcAC+rGg31cyDwMycAQ3wo5qCQ3A6cDxyXZCVxdVeuHW1XfTgMuAh7uxqsBrqqqu4dXUl+WABu6b/9fBdxSVQv6FsJGLAbumLyGYBHwpar6ynBLGsifATd1QxqPAx8ecj196f7QnglcOi/7X6i3QkqS9s9hGUlqkOEuSQ0y3CWpQYa7JDXIcJekBhnuOiwlqd6HkpIsSjLR+/bHJOcneah78+DDSc7vWXdqkge6N/ptS/KJg3sG0oEt2PvcpQH9BHhbktdU1X8zeb/x//+Ie5ITgc8AZ1bVE0mOB+5J8nhVPQRsAC6oqu909/W/ZYpjSEPjlbsOZ3cz+dZHeOWTgn8B/HVVPQHQff4N8Jfd+jcBu7t1exfa65nVPsNdh7ONwKokrwZ+j199E+dbmXyvfq/xrh3gWuCxJHckubTbh3TIMNx12OqGV0aZvGqf1asequpTTP6Ix1eBPwYW8uP8apDhrsPdJibH1vd9edNW4O37tL0dePTlhar6flVdx+TLq05M8sb5LFSaDcNdh7sbgE9W1cP7tH8GuLL78ZSXf0TlKuCz3fK53ds8AVYAe4HnDkK90ox4t4wOa1W1E/j8FO1bknwc+LfuXfv/C3ysqrZ0XS4Crk3yU+Al4E+qau9BKlualm+FlKQGOSwjSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KD/g/H+mBw6xTjmgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Range & Histogram\n",
    "print(\"score range:\", score.min(), score.max())\n",
    "print(\"basic baseline:\", score.mean(), score.std())\n",
    "\n",
    "fig, ax = plt.subplots(1, 1)\n",
    "ax.hist(score, bins=[1,2,3,4,5,6,7])\n",
    "ax.set_xlabel(\"MOS\")\n",
    "fig.savefig(\"fig_hist.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "8cfc05dd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.1802792931882462\n"
     ]
    }
   ],
   "source": [
    "# Baseline RMSE\n",
    "\n",
    "def root_mean_squared_error(y_true, y_pred):\n",
    "        return np.sqrt(np.mean(np.square(y_pred - y_true)))\n",
    "    \n",
    "baseline = np.ones(len(score)) * score.mean()\n",
    "\n",
    "print(root_mean_squared_error(score, baseline))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "c66ab486",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R: (0.8493523220744107, 1.232087997985623e-59) \n",
      "\n",
      "DIFF X-F: ID\n",
      "2124   -1.628149\n",
      "2187   -1.565224\n",
      "2070   -1.411102\n",
      "dtype: float64 \n",
      " ID\n",
      "2053    0.729828\n",
      "2112    0.777076\n",
      "2075    1.062819\n",
      "dtype: float64\n",
      "avg diff: -0.2548087053406571 19\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQgAAAEGCAYAAACZ/AuPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAncUlEQVR4nO2de5gdZZngf293TuB0gHSQ6EJLCOIYFmRIoAfQoCuoxBkuRlBZB2YHdeRRGeWicYOiBGXGjBnFWZ1RM9x0YdgAgRZhxnghoEbASdPBEEl0Fbk0ukShuYQmdLrf/aOqOtWn66v6zqX6VJ/z/p7nPDlVp+qr93ROvfV971VUFcMwjCQ6mi2AYRjFxRSEYRhOTEEYhuHEFIRhGE5MQRiG4WRGswWIs99+++n8+fObLYZhtB39/f1/UNW5lfsLpSDmz5/Pxo0bmy2GYbQdIvJI0n5bYhiG4cQUhGEYTkxBGIbhxBSEYRhOTEEYhuHEFIRhGE5MQRiG4cQUhGEYTkxBGIbhxBSEYRhOTEEYhuHEFIRhGE5MQRiG4SRXBSEi3SJys4hsFZGHROR1eV7PMIzGkne69z8B31XVd4rITKAr5+sZhtFAclMQIjIbeCNwDoCqvgS8lNf1DMNoPHnOIA4GtgPXiMiRQD9wvqruiB8kIucC5wLMmzcvR3EMI5m+gUFWrdvGE0PDHNBdZtmSBSxd1NNssQpBnjaIGcBRwNdUdRGwA1heeZCqrlbVXlXtnTt3UsUrw8iVvoFBLr5lM4NDwygwODTMxbdspm9gsNmiFYI8FcTjwOOqel+4fTOBwjCMwrBq3TaGR0Yn7BseGWXVum1NkqhY5KYgVPX3wGMisiDc9WbgF3ldzzBq4Ymh4ar2txt5ezE+AlwfejB+A7w35+sZRlUc0F1mMEEZHNBdboI0xSPXOAhV3RTaF/5UVZeq6tN5Xs8wqmXZkgWUS50T9pVLnSxbssBxRntRqLL3hjHVRN4K82IkYwrCaHuWLuoxheDAcjEMw3BiCsIwDCemIAzDcGIKwjAMJ2akNAqB5UMUE1MQRtOJ8iGikOcoHwIwJdFkbIlhNB3LhygupiCMpmP5EMXFFITRdFx5D5YP0XxMQRhNx/IhiosZKY2mY/kQxcUUhFEILB+imNgSwzAMJ6YgDMNwYksMI3csSnL6YgrCyJWiRkma0vLDlhhGrhQxStJK3ftjCsLIlSJGSRZRaRUVUxBGrhQxSrKISquomIIwcqWIUZJFVFpFxRSEkStLF/Xw+dOPoKe7jAA93WU+f/oRdRsE+wYGWbzyTg5efgeLV95Zlf2giEqrqJgXw8idRkdJ1usZsdBuf0xBGNOONCOj701uod1+ZCoIEVkMrAAOCo8XQFX1VfmKZhjJmJFx6vCZQVwFXAj0A6MZxxpG7lg/zanDx0j5jKr+h6o+qap/jF65S2YYDszIOHX4zCDWi8gq4BZgZ7RTVe/POlFEfgs8RzDz2KWqvTXKaRjjmJFx6vBREMeG/8ZvbgVO9LzGCar6h6qkMowMzMg4NWQqCFU9YSoEMYy8sMSs2sm0QYjIbBH5kohsDF9fFJHZnuMr8D0R6ReRcx3jnxuNvX379mpkN4xMLDGrPnyMlFcT2BHeHb6eBa7xHP94VT0K+HPgPBF5Y+UBqrpaVXtVtXfu3LmewxqGH5aYVR8+NohDVPWM2PZlIrLJZ3BVHQz/fVJEbgWOAX5UtZSGUSMWM1EfPjOIYRE5PtoIA6cy/7oiMktE9o7eAycBD9YqqGHUgiVm1YePgvgQ8M8i8lsReQT4KvBBj/NeAfxERB4AfgbcoarfrV1Uw6gei5moDx8vxibgSBHZJ9x+1mdgVf0NcGRd0hlGnVjMRH04FYSInK2q14nIRRX7AVDVL+Usm2E0BIuZqJ20GcSs8N+9Ez7THGQxDKNgOBWEqn4jfPsDVd0Q/yw0VBqGN/UGK+UZ7NQ3MMhl39nC0y+MANBdLrHitMNt1oGfkfIrnvsMI5F6g5XyDHbqGxhk2c0PjCsHgKHhEZbd9IAFU5Fug3gd8HpgboUdYh+gM/kso1Vo5BO73gIvjSgQk0TfwCAfu/EBRnXyinlkTOsevxVIs0HMBPYKj4nbIZ4F3pmnUEZzaXSzm3qDlfIIdoq+Y5JyaMT4rUKaDeJu4G4RuVZVH5lCmYwm0+gndr0FXvIoEJP0HRs5fqvgY4O4UkS6ow0RmSMi6/ITyWg2jX5i+wQrpVWpziPYKeu7lDrEgqnwUxD7qepQtKGqTwMvz00io+k0Ojw5q/S9jxFSYp51ETjj6PpiG9K+S3e5xKp3Hdn29gfwS9YaE5F5qvoogIgchMVBtDTLliyYYIOA+p/YacFKWRmXy256gJGx3T85VVjzn4/Re9C+Nd/Eru/YiJ4drYSPgvgUQU7F3QQVrd8AJNZ2MFqDqQ5PTlvSrFq3bYJyiBgZ3e1lcHlc0jwxFoLth2iKFXf8IJH9gOPCzXvzKiHX29urGzduzGNoIyca4Q5dvPLORCNkT3eZJ8JlRxICXHHmwsSZwBlH97C2f9BrhmAVp0BE+pNqxjptECJyaPjvUcA84InwNS/cZ7Q5jQpgSjNCptkKDuguO5cnN9z3mFehGKs4lU6akfJj4b9fTHj9Y85yGdOARlVrSjNiLluygFKHTDqn1Bl4GVzLE1d8Q+XxVnEqnbQ4iA+E/1rRWiORRrpDXUbMaN+K27YwNByEQ8/pKnHpqUGuxKp12xKXJ50iiUqickbiknVwaJjFK+9s62UHpIdan552oqre0nhxjOnEVHW4SvOAuLwRLhtEpSfG9R0ExvfXG0k6nUlbYpwavt5P0H7vrPB1JfC+/EUzik4RqjW5lieXLz0iNfYi7TsIk/347brsSFtivBdARL4HHKaqvwu39weunRLpjEJTFFdh2vIkS5ak75A0o4D2zM3wiYM4MFIOIf+PwKthGC1RranyO7jcru2Ym+GjIH4Y5l7cEG6fCfwgP5GM6UQrxhDkEUk6XfEpWvu3IvIOIGp6s1pVb81XLGM60Oi08KJQlKVTEfCZQQDcDzynqj8QkS4R2VtVn8tTMKP45FXIpV4aMatphaVTI8hUECLyAYLci32BQ4Ae4OvAm/MVzSg6U9m1yvemb9VZTbPwSfc+D1hMUEkKVf0Vlu5tUFtaeFrdh7RzfMOhLTKysfgoiJ2q+lK0ISIzsHRvg+rjIGrNe6jmprdenI3FxwZxt4h8EiiLyFuBDwPfyVcsYzpQrTGvVptF2k1fufTo7ipNqFAd0Y4uykaQme4tQSutvyFovivAOuBK9ckTrxJL925NopvYFYAEcPZx87h86RGJn7niErrLJXbuGpugdEodAhLUi4iwQjDZuNK9U2cQItIJbFHVQ4F/rfHCncBGYFBVT6llDKM5NMIbUGk0dHHdvY8CJCoJV1yCCJPGHRlTusslRBifSewxw2clbSSR+pdT1VFgm4jUEzl5PvBQHecbTaBRdRJ8qkdH3HDfY4n7XfkWQwlLCQga37w4MjZh22o81IaPDWIOsEVEfgbsiHaq6mlZJ4rIK4GTgb8DLso43CgQjYpxqMY4mNajIikuIS3Vu4jxGdMRHwXx6TrG/zLwCZIbAAMgIucS1ricN89SPIpCvd6AaHlSjaGqQ6iqBoNr6eGasZgno3p8Qq3vFpH/AhxD4N78T1X9fdZ5InIK8KSq9ovIm1LGXw2shsBI6Sm3kQNxm0OHZ8GVpHO7u0o8/+KuxGKzqejEGgwXrNnEitu2OBvpurworpmFeTKqxyeS8m+AzwB3EngxviIin1XVqzNOXQycJiJ/AewJ7CMi16nq2fUKbTSeSmNiknJIinGo7IwNJLoZfRhL2BfZDyA5EtIVEm3JVo3Bx825DXi9qv4x3H4Z8FNV9f5rhzOIj2d5MczN2Viq8UK4XImdIoypJp7v66HIIm1ZENHTXWbD8hO9x2zFLNM8qcnNGfJHIJ6Y9Vy4zygw1eYkuNbnY6o8vPLkxM+q8VCk8fnTj8iMk6jWfmDJVo3BR0H8X+A+Efk2gQ3i7cDPReQiAFX9UtYAqnoXcFftYhrVUq0XIq2+pOtpXM1Nm1TGDYKZQSRP2mzE7AfNwSeC5NdAH7v/f78NPEzgmXB6J4zmUq0XwpVXccKhc53xEL43bXe5xFnHzUvN24hiHeZ0lRLH2LFzl8UxNAEfL8ZlUyGI0ViqrTgd9wgMDg2PxxLccN9jkwyW0Uwkyc2YxKw9ZnD50iPoPWjfSTMRmOjavPTUwwEmGT6zjJW1YraKdCwGdZqSlTZdS8XpqFFNudQ5rhRcwUuDQ8PjT/0sIkW1dFEPG5afyMMrTx43OCbNTgC6Zk5+djU6bdu6amVjCmIa4vPDTutWlYav4VGAS/o2e92wEsrsc61ICUxF2rbVjsjGt+ScUSB8DZC1WPJ9b0AFrr/3Ua9ISSXojFU5lU9TAlPRlMdqR2ST1lnrK6QUhlHVj+YikTGBpDVyo3/Y8Ws43Q0JVBMnOTQ8Mt46L5rxzC6XxvfFib5n3sFOU9UZbDqTtsTYCPQTREEeBfwqfC0EZuYuWRuQZUdwLSVml5Mt/bX8sCuv0fgqH8kMj4wigtNOUusSqRqK0Bms6KR11vomgIh8CDheVXeF218Hfjw14rUuPoFMrqXEnqWOSdGHtf6waw12qmKi4eTpF0boLpfYs9TB0Asjk7wIeQc7WXn7bHzTvfcBngq39wr3GXXgY0dwLRmGXhjhijMXNuSHXUuwU3e5xMjoGDteqj+KMt6xu0ht+4wAHwWxEhgQkfUEv5E3AivyFKodSGs7f/DyOzLrK1b+sKPliq/CqCUde09H1+xG8PQL+cQ5GPWR6eZU1WuAY4FbgVuA10XLD6N20uwFkb3h+Rd3UeqUCZ+5Miqr8efHj6+GKHCq0cohPr65GItFpoIIi9a+BThSVb8NzBSRY3KXrMVJMpBVMjKmzJo5I9NQV60/v54kq7SqT43AXIzFwmeJ8S8EqfonAp8lyOZcC/xZjnK1PJUGMtdtNzQ84iyYAsFsoNp29fXchB0C1daBqQZzMRYLn0jKY1X1POBFAFV9GnNzNoR46HFPyo3hWi5ESwUXConu025HQlREpwiLD9l30gxHqE85zOkqjX9Pkcmfm4uxePjMIEbC0vUKICJzSS7+Y9TBsiULWHbTA4ll2lxp2itu25K5VEhyn7pWCd3lEpsuPWl8O97Pol63ZrnUyaWnHj7JsFqZHBYti8xQWQx8ZhD/i8BA+XIR+TvgJ8Df5ypVu5LwVI2oXBb0DQwmRiEmUWmPeMZxnms/1Kcc5nSVxpO64oFhwKTkMEuYKhY+6d7Xi0g/QTdvAZaqqvW5aDCr1m2b0A2qkmhZ4NOlKom4gkkLcY5oVDk52J2ZmRQYtmepw0rUFxiforWHAA+r6j+HtSXfKiK/U9WhnGVrK7IMh8+/uIuz/vUefvrrp2p6mkc3f9/AIDte2jXp81KHTFj/N6qcHATfzeVpsRL1xcZnibEWGBWRVwPfAA4E/i1XqdqQLOv9yJiyoUblEDf+uWYqo6pcuGbTuFGzkTfoAd3lqsczb0Yx8FEQY2EexunAV1V1GbB/vmK1Hz5xEdXQKZIYO+EuTsuEIKssT0cSc7pKzuQn1w1fDvNKks4xmo+PghgRkfcA/wO4PdxX/a/HSCWevdgIRh2l6n2ezMMjo6hOzrRMI/JSuDIwly1ZEHTermDXmHLG0T25Zm0atePTF+Mw4IPAPap6g4gcDLxbVf+h0cJYX4yAvoFBp8uzWsqlzgk3nK/xUYCzjpvnVRRmTldpkgsziUWf/V5ibkm1PS+MxlNzXwxV/QXw0dj2w0DDlYNRQYrLsxriLs4oanN2LMU6rcXe+q3bE5VDWjOdNFwdt6r1yBhTh48X40+AzwOHERSPAUBVX5WjXC1DvFrT7HIJERJrH8TJcnlWS2RXiGYNQ8MjlEudXHHmQjY+8hTX3fvopHNOOHQu1yfsh4nNdKrJIu10KKPOpLBKoxD4RFJeA1wKXAGcALwXK3brReV0Ph57kNbpqtEuvihKMU5W5uT6rdudJdk6RDh4+R3MLpfY8dKucWWW1XDXleiVdwKYUTs+N3pZVX9IYK94RFVXAMm92IwJZMUSuG7SWl183eVkL4LrBnxiaDg1ocvlWRlVRQkUXtJMJ+phURkN6TLANsowazQeHwWxU0Q6gF+JyN+KyDsIqkoZGfjMBJKOqcXlWS51suK0ZC+C6wY8oLvsVEYdIly4ZhN7ljroLpcQqlsKJCk/qwE5/fBZYpwPdBEYKj9HkPb911kniciewI+APcLr3Kyql9Yu6vTDNUWvPKaSaGq+4rYtXvkWnSITPBVJNoC0CtFJXo1o1vH0C7vtFReu2ZQpS5xK5Wc1IKcfmW7O8QNF9gFUVZ/LPJjxQjOzVPV5ESkRJHmdr6r3us6Zzm7OpPL0kN6QttIFWcnilXdmKpisMdLki7s+s/I7ollINR4Hc19OH1xuTp84iF4CQ2XUqPcZ4H2q2l/FxbsIFMSHVPU+13HTVUEkxRZENy5QtRejmoSsL5+5sCFP4Kz4iCguIsnjsceMDnbumlgBwFdxGcXApSB8bBBXAx9W1fmqOh84j0Bh+Fy0U0Q2AU8C309SDiJyrohsFJGN27dv9xm2cGRVqI5CjaN06tnl0ngCU1ovjCx6wuK1vqT14cgyqEZxEUnst9cefPnMhRYN2YL42CBGVXW8D4aq/kREJqcDJqCqo8BCEekGbhWR16rqgxXHrAZWQzCD8Ja8QLhu5sGh4UlP5niwkG8vjCSqNe65+nBsfOQp1m/dnqqQomu5bBBPhI18TSG0Hmmt944K394tIt8AbiDI5zkTuKuai6jqUFg2/23Ag1nHF53K9byrTmOHwMdufCDVz+/bCyNOT8zG4Ruk5JrlZIVSxw2grmWPZV62LmlLjC+GryOB1xAES60A/itB+71URGRuOHNARMrAW4GtdUlbAJJKzLtSJsbULwgorhR86kVGhr9qSt27FE+adOVSJ19895HjSsfclO1HWuu9E+oce3/gm2E9yw7gRlW9PeOcwtPIQioRUa/NvoFBnn8xffU2qsrilXeyY+euxBlBUhftpYt6vFyucXoSZiTmpmw/vN2cACJyu6qekpcw08GLcfDyO+ruSVnJnK4SA585ycutWS2lTmHWzBkMDY94F54192T7UY8XI07bPyrS1ttzukrjVvyukv+fdig0XOZRZm1kVMeDrZTdSaKuqEgBWzIY41SrIAZykWIasWzJAmcmdtfMGTy88mQ2LD+RmTP8Q6UjpTMVxj4lUGBjjpmjYiXnjd14KQgRKYvIAlV9X94CFZ2li3qc0/T4DCCthHycuJGvlhyMqEhTT3eZOZ5l4iL7QRKWOGXE8enNeSqwCfhuuL1QRG7LWa5Ck5b8BIGxscMxhe8ul5wBRfGyc77JUWO6W8lceurhXgomMi6aR8LIwidQagVwDGHsg6puCsvOtS3LlixwJj9d0rfZGVsQZVymTeHjAUcHL7/DS54oliIyLEZehu6uEs+/uGtC6bq4Ethjxu6eFL5l44z2wqv1nqo+IxOfZtMy4rFRuNx9gFM5VGZcxnElUlXjmoyOixTMJX2bueG+xybEYcQDrCoV3Isj2d0U0xK+jNbER0FsEZG/BDrD8nMfBX6ar1jFJym0ePHKO52ac0zVqRySQqAheabiIr4cuaRvs7OMHCRHd2Z1s0qT05RE6+JjpPwIcDiwk6BhzjPABTnK1HTSkprSSHNTuoyCWYleUUZoFvEb/ob7Hks85vp7H+XiWzanVphykSan0bqkKogwCvIOVf2Uqv5Z+LpEVV+cIvmmnKRQat9msi4lkBZb4Lopo0SvpYt6vD0LkYwuBaCQmbHpIq00ndG6pCqIMBtzTERmT5E8TaeeJ6XLTakE0/pL+jaP74tmKWnGnAvWbGL+8ju87RCXfWcLUFuV6CwPhkt5WKJWa+Njg3ge2Cwi3wd2RDtV9aPuU6Yv9Twp48bLypt6VJXr7n2U6+59lDkJ3oVGEKWSv+fYAxNtELNmdrLjpckzCJcBNW6U7O4qUeoQp0fEaE18bBC3AJ8mqC/ZH3u1JPU+KZcu6mHD8hNT+948/cJIw5VDnMuXHsHZx82bIMOsmZ2846iexNiHeMZmROVS6+kXRkAYL2BrRWHaA5/OWt+cCkGKQlqMgy99A4NN8QN3l3dHUvYetC9r+wfHv8eOl0ZZ2z/IGUf3sH7r9kxXZdJSa2RUmbXHDDZdelK+X8QoDD6dtR4mIe6hVTtrNSKluVbLvk+25ZyuEs/v3DWpH0UHsOK0wyfIkGRLWb91e2KmZmWMg8vuYUbJ9sLHBhFPAd0TeBewbz7iFIPKGId4ezmfwrO13EQ+yqHUIVx6aqAELvvOlnGbQ3e5NClCsxpbSlKMg0seM0q2Fz5LjD9W7PqyiPQDn8lHpOZS+SQ94dC5E6bqle3zLlyziY2PPMXlS3fHK1RbnMW3TsOqdx05IW8jDZcMSTd40mwjSg2Py2VGyfbDJ1nrqNirV0Q+iN/MY9qRFANx/b2PpsYOKEEAUjxOIsnd2dkx2WwZ7cmcOXRK1eXtq0nGSitHZ5Wq2xufG/2Lsfe7gIeBd+cjTnNxPUmz0PDcyqd7Uq5G5AL1nTV0h0uaC9dsYtW6bd72kEqXa9TAN7KPxMdwzTasspThoyDer6q/ie9o1WzOegxwSW3mkm7kpYt6qiott3PXWM35D9ExWTkUjfDcGK2JTxzEzZ77pj31GOB8z+0bGPRWDtFTP87wyCgXrNnknSPiExlaWYfClhNGRFpfjEMJkrRmi8jpsY/2IfBmtBxJT1KfpYDv0zaycfhQ6pRJrsw4vrMJX2+GNb4xkkibQSwATgG6gVNjr6OAD+QuWRNIepJmKYdqnrZZJfNFAoXUVepIVQ4RPjkilkNh1ENaX4xvA98Wkdep6j1TKFNTqXySptkLqjXiZdo4FK44c6GzxV0tY5p9wagHHyPlgIicR7DcGF9atEsB22VLFnBBSk/KiL6Bwczgpaz4iAO6y6xat62qMO2smYA1uzHqwUdB/G+ClnlLgM8CZwEP5SlUkVi6qGfCjR8nXqR22c0PTFgWDA2PcNGaTVz2nS3jUZeVQVdxshrkuoiqRGV9B1MIRi34eDFeraqfBnaEiVsnA8fmK1axSKoWHZ+mr1q3LdFmMEaQBRkFXa3tH+SoebMn1WuY01VijxkdXLhmk7Matov1W7dXdbxhVINX0drw3yEReS3we+Dl+YlUPLKm6b7xE8Mjo2z49VMT9pU6ZELyVVI1qDRPiiVPGXnioyBWi8gcgpoQtwF74ZGHISIHAt8CXkHw+16tqv9Uh6xV08gqzGnT9GpzL+K46kJ0ijCmOi53UhGa6NqGkRc+yVpXhm/vBqpJ8d4FfExV7xeRvYF+Efm+qv6iBjmrZiqrMC9bsmCSDaJexlR5eOXJE/aZN8KYanyStV4hIleJyH+E24eJyPuzzlPV36nq/eH75wgMm1NmKZvKKsxLF/Ww6p1HMmtmdW3z0qicGVi0o9EMfJYY1wLXAJ8Kt38JrAGu8r2IiMwHFgH3JXx2LnAuwLx583yHzKSRVZh9lipLF/Wwat02drxU3filjiA6Kj77cM0MzBthTDU+CmI/Vb1RRC4GUNVdIpLdySVERPYC1gIXqOqzlZ+r6mpgNUBvb2/D5ujV1EMAtxKoZqniq3wio2NPRZanxSkYRcNHQewQkZcRGtJF5DiC5jmZiEiJQDlcr6q31CxlDVQTQZimBLIa28RJM1ZWKoWkGYhhFA0fBXERgffiEBHZAMwF3pl1kgTNPK8CHlLVL9UlZQ1UE0GYpgSqWaqktcqLlIPVVzCmE2nZnPNU9dHQC/HfCJK3BNimqpPDCiezGPgrgp4am8J9n1TVf69XaF981+xpSqCapUp0LZ/QbMOYDqR5Mfpi79eo6hZVfdBTOaCqP1FVUdU/VdWF4WvKlEM1pGU8VlO6DUhtldch4t3n0zCKQJqCiMf8tmSJ+4g0JeDrXow3/N2xcxelzskh06Oq3n0+DaMIpCkIdbxvObKUwNJFPSxbsoADuss8MTTMqnXbJtzklcVuh4ZHQCGhTq11xDamFWlGyiNF5FmCmUQ5fE+4raq6T+7STQGV7s0rEqpHZ7k6E7tQpbTWM1uEMV1wziBUtVNV91HVvVV1Rvg+2m4Z5VBZ5j5pCZAVlVntDW/5E8Z0wSfdu2XJuvEju4IrtmFwaJjFK+90rr+6y6WqDJyGUTRasgGOL2nuzcplRRICTuVRLnWO98q0KEljutK2CqJvYJAOkcT6C1Hptyzl4Jo5VEZLmkIwpittucSIZgdJyiFaAqTZFdKqXQuwYfmJphSMlqAtFYRrdtApMu7edBkSo3BpVzCUGSCNVqItFYRrdjCmOqEdXZqBsdoIS8OYjrSlDcInvyIr2cvKyRvtgGjCOrxZ9Pb26saNG3O/TpKHolzqtApNRtsiIv2q2lu5vy2XGEsX9XDG0T3j5ec7RTjjaKvWZBiVtOUSo29gkLX9g+NejFFV1vYP0nvQvk4lkVRxCmyJYbQ2bbnEyOq3WXmjJy1JSp0COjHnwpYpxnTFlhgx0mIckvIxEpOxRnVSQpZlahqtRlsuMbIa3QyPjPKpWzePLx+qmWNZpqbRSrTlDCIphqGSHS+Njmd5VoMFShmtRFvOIOIxDLW2zHPZICxQymgl2lJBwO6Ctn0Dg84is0kImBfDaBvaVkFELF3Uw4rbtgRl4jJIKltvCsFoZdpCQWS1zltx2uGZtR+Slg+N7B5uGEWk5RWET+u8pLyKEw6dy/qt2503/1R2DzeMZtHyCsK3dV61jXGraclnGNOVlndzNrLL91SMaxhFouUVRFrXrCKOaxhFouUVRF6FXaxgjNEOtLwNIq/CLlYwxmgHcsvmFJGrgVOAJ1X1tT7nTFU2p2EYE2lGNue1wNtyHN8wjJzJTUGo6o+Ap/Ia3zCM/Gm6kVJEzhWRjSKycfv27c0WxzCMGE1XEKq6WlV7VbV37ty5zRbHMIwYLe/FqAbLrTCMiZiCCLHcCsOYTG5LDBG5AbgHWCAij4vI+/O6ViNIy60wjHYltxmEqr4nr7HzwHIrDGMyTTdSFgXLrTCMyZiCCLHcCsOYjBkpQyy3wjAmYwoiRrVFYwyj1bElhmEYTkxBGIbhxBSEYRhOTEEYhuHEFIRhGE5MQRiG4cQUhGEYTkxBGIbhJLeitbUgItuBRxow1H7AHxowTiMpokxQTLlMJj8aKdNBqjqpYlOhFESjEJGNSRV6m0kRZYJiymUy+TEVMtkSwzAMJ6YgDMNw0qoKYnWzBUigiDJBMeUymfzIXaaWtEEYhtEYWnUGYRhGAzAFYRiGk5ZSECJytYg8KSIPNluWCBE5UETWi8gvRGSLiJxfAJn2FJGficgDoUyXNVumCBHpFJEBEbm92bJEiMhvRWSziGwSkUJ0lxaRbhG5WUS2ishDIvK6XK7TSjYIEXkj8DzwLd+O4nkjIvsD+6vq/SKyN9APLFXVXzRRJgFmqerzIlICfgKcr6r3NkumCBG5COgF9lHVU5otDwQKAuhV1cIESonIN4Efq+qVIjIT6FLVoUZfp6VmEEVsGKyqv1PV+8P3zwEPAU2ta6cBz4ebpfDV9CeFiLwSOBm4stmyFBkRmQ28EbgKQFVfykM5QIspiKIjIvOBRcB9TRYlmspvAp4Evq+qTZcJ+DLwCWCsyXJUosD3RKRfRM5ttjDAwcB24JpwOXaliMzK40KmIKYIEdkLWAtcoKrPNlseVR1V1YXAK4FjRKSpSzIROQV4UlX7mymHg+NV9Sjgz4HzwqVsM5kBHAV8TVUXATuA5XlcyBTEFBCu89cC16vqLc2WJ044NV0PvK3JoiwGTgvX+/8HOFFErmuuSAGqOhj++yRwK3BMcyXiceDx2KzvZgKF0XBMQeRMaBC8CnhIVb/UbHkARGSuiHSH78vAW4GtzZRJVS9W1Veq6nzgvwN3qurZzZQJQERmhcZlwmn8SUBTvWSq+nvgMRGJujq9GcjF6N1SfTHChsFvAvYTkceBS1X1quZKxWLgr4DN4Zof4JOq+u/NE4n9gW+KSCfBQ+JGVS2MW7FgvAK4NdDzzAD+TVW/21yRAPgIcH3owfgN8N48LtJSbk7DMBqLLTEMw3BiCsIwDCemIAzDcGIKwjAMJ6YgDMNwYgpiihCRl4XZgJtE5PciMhjbntngax0ajjsgIoc0cuwiICJvirI9ReQ0EXFGEYZZjx+ObR8gIjdPhZytgLk5m4CIrACeV9V/jO2boaq7GjT+cmCGql7uebwQ/BYakgNR63cRkU5VHfU47k3Ax32yPcP8l9uLkt073bAZRBMRkWtF5Osich/wBRE5RkTuCZ/8P40i5UTkHBG5RUS+KyK/EpEvhPs7wzEeDOsVXCgifwFcAHxIRNaHx10UHvOgiFwQ7psvIttE5FsEkYFvCGsLXCsivxSR60XkLSKyIbzmMeF5sySou/GzUM63x2S8TUTuBH5Y8T3nh2NfH9YuuFlEusLPfisi/yAi9wPvEpGTwr/B/SJyU5jDgoi8LRzjfuD02NjniMhXw/evEJFbJahz8YCIvB5YCRwSzqhWhbI8GB6/p4hcE/7tBkTkhLS/d1uiqvaa4hewAvg4cC1wO9AZ7t+H4MkP8BZgbfj+HIJoudnAngTNhQ4EjibIxIzG7Y6PH74/GtgMzAL2ArYQZJTOJ8iaPC48bj6wCziC4MHRD1wNCPB2oC887u+Bs6PrAb8Mxz6HIEdg34TvO58gI3JxuH11TL7fAp8I3+8H/IigVgXA/wQ+E37nx4A/CeW5kWBWEP1tvhq+X0OQDAfQGf695gMPVsjyYPj+Y8DV4ftDgUfDayX+vZv9u2nGy2YQzecm3T2tng3cFD7hrgAOjx33Q1V9RlVfJIi7P4jgR/wqEfmKiLwNSMoSPR64VVV3aFAD4hbgDeFnj+jEIjEPq+pmDZYaW8JrKoGCmR8ecxKwPAwbv4vgBpoXfvZ9VXXV43hMVTeE768L5YpYE/57HHAYsCEc/6/D73loKNuvQnlcSVwnAl+D8WzVZxzHRRwfjaWqWwkUwWvCz5L+3m1HS+ViTFN2xN5/Dlivqu8I1853xT7bGXs/SjDTeFpEjgSWAB8E3g28r8ZrV15jLLY9xu7figBnqOq2+IkicmzCeHEqjV3x7eg8IVAy76kYe2HKuHkx6e/dBBmajs0gisVsYDB8f07WwSKyH9ChqmuBS0hO+f0xsFREuiTIRnxHuK9W1gEfCQ2biMgiz/Pmye66iX9JUOauknuBxSLy6nDsWSLyGoJM0/kxj8x7Es6FwPbxofDcTgkqLz0H7O04/sfAWeHxryGYCW1zHNuWmIIoFl8APi8iA/g9sXqAu8Lp+HXAxZUHaFDu7lrgZwSVrK5U1YE6ZPwcQYm6n4vIlnDbh20ExVYeAuYQLgUqZN1OoBhvEJGfA/cAh4bT/HOBO0Ij5ZOOa5wPnCAimwlsKIep6h8JliwPisiqiuP/BegIj18DnKOqOzHGMTenkTvmapy+2AzCMAwnNoMwDMOJzSAMw3BiCsIwDCemIAzDcGIKwjAMJ6YgDMNw8v8B5Of2kIlJak8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Compare predictions of transformer-vs-feature models!\n",
    "\n",
    "# load & merge\n",
    "d_x = pd.read_csv(\"answer-xlmr.csv\", index_col=\"ID\")\n",
    "d_f = pd.read_csv(\"answer-feat.csv\", index_col=\"ID\")\n",
    "ds = d_x.join(d_f, lsuffix=\"_x\", rsuffix=\"_f\")\n",
    "#print(d_x.head(1)['MOS'], d_f.head(1)['MOS'], ds.head(1), \"\\n\")\n",
    "\n",
    "# plot\n",
    "fig, ax = plt.subplots(1, 1, figsize=(4,4))\n",
    "ax.scatter(ds.MOS_x, ds.MOS_f)\n",
    "ax.set_xlim(0.5, 6.5)\n",
    "ax.set_xlabel(\"Transformer prediction\")\n",
    "ax.set_ylim(0.5, 6.5)\n",
    "ax.set_ylabel(\"Feature-based prediction\")\n",
    "fig.savefig(\"fig_results_scatter.pdf\")\n",
    "print(\"R:\", stats.pearsonr(ds.MOS_x, ds.MOS_f), \"\\n\")  # 0.85 concordance\n",
    "\n",
    "# differences\n",
    "t = ds['MOS_x'] - ds['MOS_f']\n",
    "t = t.sort_values()\n",
    "print(\"DIFF X-F:\", t.head(3), \"\\n\", t.tail(3))  # so both negative and positive\n",
    "print(\"avg diff:\", t.mean(), sum(abs(t)>1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "764e2d7a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ID\n",
       "2124     True\n",
       "2187     True\n",
       "2070     True\n",
       "2115     True\n",
       "2031     True\n",
       "        ...  \n",
       "2017     True\n",
       "2087     True\n",
       "2053     True\n",
       "2112     True\n",
       "2075    False\n",
       "Length: 210, dtype: bool"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t<1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}