{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "\n",
    "from tdm.raw.breast_mibi import read_single_cell_df\n",
    "from tdm.model.selection import cross_validation2, plot_cross_validation_result\n",
    "from tdm.cell_types import FIBROBLAST, MACROPHAGE, TUMOR, ENDOTHELIAL\n",
    "\n",
    "single_cell_df = read_single_cell_df()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the settings to compare by constructing a dictionary that maps a setting name, to kwargs for the model and feature transforms:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "setting_dicts = {\n",
    "    'degree 2': {\n",
    "        'model_kwargs': {}, # see: LogisticRegressionModel for more options such as regularization\n",
    "        'polynomial_dataset_kwargs': {'degree': 2} # see: PolynomialDataset for more  feature transformations\n",
    "    },\n",
    "    'degree 1': {\n",
    "        'model_kwargs': {},\n",
    "        'polynomial_dataset_kwargs': {'degree': 1},\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run the cross validation (note: partitions the dataset at the level of tissues, not cells)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3/3 [00:03<00:00,  1.29s/it]\n"
     ]
    }
   ],
   "source": [
    "res = cross_validation2(\n",
    "    single_cell_df, \n",
    "    model_dicts=setting_dicts,\n",
    "    cell_types_to_model=[FIBROBLAST, MACROPHAGE],\n",
    "    allowed_neighbor_types=[FIBROBLAST, MACROPHAGE, TUMOR, ENDOTHELIAL],\n",
    "    neighborhood_mode='extrapolate',\n",
    "    n_splits=3\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot the result:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmwAAAEiCAYAAABEL46kAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvxklEQVR4nO3deVhV5d7/8Q/IDIIToKaApJiVCs5aTmmKp5ytY1loORzLOmlqR3OkstTMSp/TpEYOdXysnC5T82RqppZmYpo+KqZQOZUpqAgo3L8/+rlrByjD3uwFvF/XxSV7rbXv9f2uvffth7UnN2OMEQAAACzL3dUFAAAA4PoIbAAAABZHYAMAALA4AhsAAIDFEdgAAAAsjsAGAABgcQQ2AAAAiyOwAQAAWByBDQAAwOIIbCj3Bg0apF69etkud+jQQSNHjnRZPQCsibkCrkRgA4ohIiJCr732mqvLcIjNmzfLzc0t18/EiRNdXRpQ6pXFuaJy5crKyMiwW7dr1y7b3AHH8nB1AQCs5dChQwoMDLRdDggIcGE1AKyqYsWKWrFihR544AHbsgULFigsLEwpKSkurKxs4gwbSr2cnBzNnDlTdevWlbe3t8LCwjRt2jTb+h9//FH333+/KlWqpCpVqqhnz546fvx4sffboUMHJScna9SoUba/KC9duqTAwEB99NFHdtuuXLlS/v7+unDhgo4fPy43NzctXbpUbdq0kY+Pj26//XZt2bLF7jr79+9Xt27dFBAQoNDQUD388MP69ddfi133jYSEhKh69eq2HwIbygrmCscaOHCg3n33Xdvly5cva+nSpRo4cKDT910eEdhQ6o0fP17Tp0/XpEmTdODAAX3wwQcKDQ2VJF25ckVdu3ZVxYoVtXXrVm3btk0BAQGKjY1VVlZWsfa7fPly1apVS88995xOnjypkydPyt/fX/3791dCQoLdtgkJCerXr58qVqxoWzZ27FiNHj1ae/bsUevWrdW9e3edPXtWknT+/HndddddiomJ0TfffKP169fr9OnTuv/++/OtJyUlRQEBAdf9efHFF4vVM1CaMVf8zlFzxcMPP6ytW7fazqZ9/PHHioiIUJMmTYpymHAjBijF0tLSjLe3t5k3b16e6xcvXmzq169vcnJybMsyMzONr6+v+fTTT40xxgwcOND07NnTtr59+/bmqaeeKtD+w8PDzauvvmq37OuvvzYVKlQwJ06cMMYYc/r0aePh4WE2b95sjDHm2LFjRpKZPn267TpXrlwxtWrVMjNmzDDGGPP888+bLl262I37448/Gknm0KFDedZy5coVc+TIkev+nD17Nt9eNm3aZCQZf39/u59ff/21QMcCsDLmij84aq44d+6c6dWrl4mPjzfGGNOxY0fz+uuvmxUrVhjihePxGjaUagcPHlRmZqY6deqU5/q9e/cqKSnJ7q9VScrIyNDRo0edUlOLFi102223aeHChRo3bpyWLFmi8PBwtWvXzm671q1b23738PBQs2bNdPDgQVvdmzZtyvPpyKNHjyoqKirXcg8PD9WtW7fY9W/dutXueFWuXLnYYwKuxlzxB0fNFZL06KOP6qmnntJDDz2kHTt26MMPP9TWrVsdMjbsEdhQqvn6+l53/cWLF9W0aVO9//77udYFBwc7qywNGTJE//73vzVu3DglJCTokUceKdS7pi5evKju3btrxowZudbVqFEjz+ukpKTo1ltvve64zz77rJ599tnrblOnTh1VqlSpwLUCpQFzxR8cNVdIUrdu3TRs2DANHjxY3bt3V9WqVQtWOAqNwIZSrV69evL19dXGjRs1ZMiQXOubNGmi//3f/1VISIjdOx8dxcvLS9nZ2bmWP/TQQ3rmmWc0Z84cHThwIM8X4X711Ve2v6SvXr2q3bt364knnrDVfe31IB4eBXuY1qxZU4mJidfdpkqVKgUaCyhrmCv+4Mi5wsPDQ3FxcZo5c6bWrVtXoOugiFz9nCxQXFOnTjWVK1c2CxcuNElJSWbHjh1m/vz5xhhjLl26ZOrVq2c6dOhgvvjiC/PDDz+YTZs2mSeffNL8+OOPxpjivS7l7rvvNj169DA//fST+eWXX+zWPfjgg8bLy8vExsbaLb/2upSwsDCzfPlyc/DgQTNs2DATEBBgG+Pnn382wcHBpl+/fmbnzp0mKSnJrF+/3gwaNMhcvXq1iEfq+v78uhSgLGKucIy/zhWZmZnml19+sb3+j9ewOQfvEkWpN2nSJI0ePVqTJ09WgwYN9Pe//11nzpyRJPn5+emLL75QWFiY+vTpowYNGmjw4MHKyMhwyF/Rzz33nI4fP66bb74519MmgwcPVlZWlh599NE8rzt9+nRNnz5djRs31pdffqnVq1erWrVqkn7/C3jbtm3Kzs5Wly5d1LBhQ40cOVKVKlWSuzsPW6AomCucw8vLS9WqVePDcp3MzRhjXF0EUBYtXrxYo0aN0okTJ+Tl5WVbfvz4cdWpU0d79uxRdHS06woEYAnMFSgIXsMGOFh6erpOnjyp6dOn6x//+IfdBAwA1zBXoDB4bgXIx9atW6/7wZL5mTlzpm655RZVr15d48ePL8GKAbgCcwVKAk+JAvm4fPmyfv7553zXO+pzjACUbswVKAkENgAAAIvjKVEAAACLI7ABAABYHIHNYowxSktLE89UAygo5g2g7COwWcyFCxcUFBSkCxcuuLoUAKUE8wZQ9hHYAAAALI7ABgAAYHEENgAAAIsjsAEAAFgcgQ0AAMDiCGwAAAAWR2ADAACwOAIbAACAxRHYAAAALM7D1QWgdDh9+rRSU1NdXQbyERQUpNDQUFeXARQac0vpwBzjegQ23NDp06f10MNxupKV6epSkA9PL28tWbyICRWlCnNL6cEc43oENtxQamqqrmRl6nJke+X4BDltP+6Xz8v32Be6XKedcnwrOW0/ZY17Rqr0wxalpqYymaJUKam55XqYd26MOcYaCGwosByfIOX4V3P+fnwrlch+AFhDSc0t162BeQcWx5sOAAAALI7ABgAAYHEENgAAAIsjsAEAAFgcgQ0AAMDiCGwAAAAWR2ADAACwOAIbAACAxRHYAAAALI7ABgAAYHEENgAAAIsjsAEAAFgcgQ0AAMDiCGwAAAAWR2ADAACwOAIbAACAxRHYSrGMjAwdPnxYGRkZri4FKJXK+2OovPcPOJKzH08EtlIsJSVFw4YNU0pKiqtLAUql8v4YKu/9A47k7McTgQ0AAMDiCGwAAAAWR2ADAACwOAIbAACAxRHYAAAALI7ABgAAYHEENgAAAIsjsAEAAFgcgQ0AAMDiCGwAAAAWR2ADAACwOAIbAACAxRHYAAAALI7ABgAAYHEENgAAAIuzXGDr0KGDRo4c6eoyAAAALMNyga20mzdvntq2bavKlSurcuXK6ty5s3bu3OnqsgAAQClWLgNbVlaW08bevHmzHnjgAW3atEk7duxQ7dq11aVLF/38889O2ycAACjbXBrYLl26pLi4OAUEBKhGjRp65ZVXcm2TmZmpMWPG6KabbpK/v79atmypzZs3220zb9481a5dW35+furdu7dmz56tSpUq2dZPnTpV0dHRmj9/vurUqSMfHx9J0vnz5zVkyBAFBwcrMDBQd911l/bu3Ws39qpVq9SkSRP5+PgoMjJS8fHxunr1ar49vf/++3r88ccVHR2tW265RfPnz1dOTo42btxY9AMFAADKNQ9X7nzs2LHasmWLVq1apZCQED377LP69ttvFR0dbdvmiSee0IEDB7R06VLVrFlTK1asUGxsrPbt26d69epp27ZtGj58uGbMmKEePXros88+06RJk3LtKykpSR9//LGWL1+uChUqSJLuu+8++fr6at26dQoKCtLbb7+tTp066fDhw6pSpYq2bt2quLg4zZkzR23bttXRo0c1bNgwSdKUKVMK1GN6erquXLmiKlWq5Lk+MzNTmZmZtstpaWkFPXw2ycnJhb6OlcaHY3A7FV5pPWaOmDf+zFXHobQe//KK2+v6nH18XBbYLl68qAULFmjJkiXq1KmTJGnhwoWqVauWbZuUlBQlJCQoJSVFNWvWlCSNGTNG69evV0JCgl588UXNnTtX3bp105gxYyRJUVFR2r59u9asWWO3v6ysLC1atEjBwcGSpC+//FI7d+7UmTNn5O3tLUmaNWuWVq5cqY8++kjDhg1TfHy8xo0bp4EDB0qSIiMj9fzzz+uZZ54pcGD717/+pZo1a6pz5855rn/ppZcUHx9f0MOWp2nTphXr+igbuB+UH46YN/6M+w4KgvuJa7kssB09elRZWVlq2bKlbVmVKlVUv3592+V9+/YpOztbUVFRdtfNzMxU1apVJUmHDh1S79697da3aNEiV2ALDw+3hTVJ2rt3ry5evGgb55rLly/r6NGjtm22bdtmdyfNzs5WRkaG0tPT5efnd90ep0+frqVLl2rz5s22p2H/avz48Xr66adtl9PS0lS7du3rjvtXEyZMUHh4eKGuUxjJyck8UEsBZ98PyqLSet92xLzxZ66675TW419eMcdcn7Pvzy59SvRGLl68qAoVKmj37t22pzGvCQgIKNRY/v7+ucauUaNGrtfDSbK9/u3ixYuKj49Xnz59cm2TXwC7ZtasWZo+fbo+++wzNWrUKN/tvL29bWf4iio8PDxXqEX5w/2g/HDEvPFn3HdQENxPXMtlge3mm2+Wp6envv76a4WFhUmSzp07p8OHD6t9+/aSpJiYGGVnZ+vMmTNq27ZtnuPUr19fu3btslv218t5adKkiU6dOiUPDw9FRETku82hQ4dUt27dQnQmzZw5U9OmTdOnn36qZs2aFeq6AAAAf+WywBYQEKDBgwdr7Nixqlq1qkJCQjRhwgS5u//xxtWoqCgNGDBAcXFxeuWVVxQTE6NffvlFGzduVKNGjXTPPffoySefVLt27TR79mx1795dn3/+udatWyc3N7fr7r9z585q3bq1evXqpZkzZyoqKkonTpzQJ598ot69e6tZs2aaPHmy7r33XoWFhalfv35yd3fX3r17tX//fr3wwgt5jjtjxgxNnjxZH3zwgSIiInTq1Clbv4U9KwgAACC5+GM9Xn75ZbVt21bdu3dX586ddeedd6pp06Z22yQkJCguLk6jR49W/fr11atXL+3atct2Vu6OO+7QW2+9pdmzZ6tx48Zav369Ro0adcOnLN3c3LR27Vq1a9dOjzzyiKKiotS/f38lJycrNDRUktS1a1etWbNGGzZsUPPmzdWqVSu9+uqr130O/80331RWVpb69eunGjVq2H5mzZpVzKMFAADKK5e+hi0gIECLFy/W4sWLbcvGjh1rt42np6fi4+Ov+46ooUOHaujQoXaX//w05tSpUzV16tRc16tYsaLmzJmjOXPm5Dt2165d1bVr14K0I0k6fvx4gbcFAAAoCEu/6aCgZs2apbvvvlv+/v5at26dFi5cqDfeeMPVZQEAADhEmQhsO3fu1MyZM3XhwgVFRkZqzpw5GjJkiKvLAgAAcIgyEdiWLVvm6hIAAACcplx++TsAAEBpQmADAACwOAIbAACAxRHYAAAALI7ABgAAYHEENgAAAIsjsAEAAFgcgQ0AAMDiCGwAAAAWR2ADAACwOAIbAACAxRHYAAAALI7AVoqFhYXpnXfeUVhYmKtLAUql8v4YKu/9A47k7MeTh1NGRYnw8fFRVFSUq8sASq3y/hgq7/0DjuTsxxNn2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsLgifQ5bTEyM3Nzcci13c3OTj4+P6tatq0GDBqljx47FLhAAAKC8K9IZttjYWP3www/y9/dXx44d1bFjRwUEBOjo0aNq3ry5Tp48qc6dO2vVqlWOrhcAAKDcKdIZtl9//VWjR4/WpEmT7Ja/8MILSk5O1oYNGzRlyhQ9//zz6tmzp0MKBQAAKK+KdIZt2bJleuCBB3It79+/v5YtWyZJeuCBB3To0KHiVQcAAICiBTYfHx9t37491/Lt27fLx8dHkpSTk2P7HQAAAEVXpKdEn3zySQ0fPly7d+9W8+bNJUm7du3S/Pnz9eyzz0qSPv30U0VHRzusUAAAgPKqSIFt4sSJqlOnjv7nf/5HixcvliTVr19f8+bN04MPPihJGj58uB577DHHVQoAAFBOFSmwSdKAAQM0YMCAfNf7+voWdWgAAAD8SZEDmyRlZWXpzJkzysnJsVseFhZWrKIAAADwhyIFtiNHjujRRx/N9cYDY4zc3NyUnZ3tkOIAAABQxMA2aNAgeXh4aM2aNapRo0ae33qAssc9I9W5418+b/cvCsbZtwvgbK68DzPv3BhzjDUUKbAlJiZq9+7duuWWWxxdDywoKChInl7e0g9bSmR/vse+KJH9lCWeXt4KCgpydRlAoZT03HI9zDvXxxzjekUKbLfeeqt+/fVXR9cCiwoNDdWSxYuUmspfWVYVFBSk0NBQV5cBFApzS+nBHON6bsYYU9grff7555o4caJefPFFNWzYUJ6ennbrAwMDHVZgeZOWlqagoCClpqZyHAEUCPMGUPYVKbC5u//+BQl/fe0abzooPiZeAIXFvAGUfUV6SnTTpk2OrgMAAAD5KFJga9++vaPrAAAAQD4KHNi+++473X777XJ3d9d333133W0bNWpU7MIAAADwuwIHtujoaJ06dUohISGKjo6Wm5ub8nr5G69hAwAAcKwCB7Zjx44pODjY9jsAAABKRoEDW3h4uO335ORktWnTRh4e9le/evWqtm/fbrctAAAAise9KFfq2LGjfvvtt1zLU1NT1bFjx2IXBQAAgD8UKbBd+7y1vzp79qz8/f2LXRQAAAD+UKiP9ejTp4+k399YMGjQIHl7e9vWZWdn67vvvlObNm0cWyEAAEA5V6jAdu2LX40xqlixonx9fW3rvLy81KpVKw0dOtSxFQIAAJRzhQpsCQkJkqSIiAiNHTtWfn5+TikKAAAAfyjSa9i2bNmirKysXMvT0tJ01113FbsoAAAA/MGhgS0jI0Nbt24tdlEAAAD4Q6GeEr32lVTGGB04cECnTp2yrcvOztb69et10003ObZCAACAcq5Qge3aV1K5ubnl+dSnr6+v5s6d67DiAAAAUMjAduzYMRljFBkZqZ07d9q+qkr6/V2iISEhqlChgsOLROlz+vRppaamuroM5CEoKEihoaGuLgPlEPNC6cN8YR2FCmzXvnIqJyfHKcWgbDh9+rQeejhOV7IyXV0K8uDp5a0lixcxCaNEMS+UTswX1lGowPZnixcv1ltvvaVjx45px44dCg8P16uvvqrIyEj17NnTkTWilElNTdWVrExdjmyvHJ8gp+zD/fJ5+R77QpfrtFOObyWn7KMscs9IlX7YotTUVCZglKiSmBfywlxRdMwX1lKkwPbmm29q8uTJGjlypKZNm6bs7GxJUuXKlfXaa68R2CBJyvEJUo5/Nefuw7eS0/cBwHFKYl7Ic7/MFSjlivSxHnPnztW8efM0YcIEu9esNWvWTPv27XNYcQAAAChiYDt27JhiYmJyLff29talS5eKXRQAAAD+UKTAVqdOHSUmJuZavn79ejVo0KC4NQEAAOBPivQatqefflojRoxQRkaGjDHauXOn/vOf/+ill17S/PnzHV0jAABAuVakwDZkyBD5+vpq4sSJSk9P14MPPqibbrpJr7/+uvr37+/oGgEAAMq1IgW2y5cvq3fv3howYIDS09O1f/9+bdu2TbVq1XJ0fQAAAOVekV7D1rNnTy1atEiSlJWVpR49emj27Nnq1auX3nzzTYcWCAAAUN4VKbB9++23atu2rSTpo48+UmhoqJKTk7Vo0SLNmTPHoQUCAACUd0UKbOnp6apYsaIkacOGDerTp4/c3d3VqlUrJScnO7RAAACA8q5Iga1u3bpauXKlfvzxR3366afq0qWLJOnMmTMKDAx0aIEAAADlXZEC2+TJkzVmzBhFRESoZcuWat26taTfz7bl9YG6AAAAKLoivUu0X79+uvPOO3Xy5Ek1btzYtrxTp07q3bu3w4oDAABAEQObJFWvXl3Vq1e3W9aiRYtiFwQAAAB7RXpKFAAAACWHwAYAAGBxBDYAAACLI7ABAABYHIGtHMnIyNDhw4eVkZHh6lKAUqOsPW7KWj+AFTnjcUZgK0dSUlI0bNgwpaSkuLoUoNQoa4+bstYPYEXOeJwR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHIENAADA4ghsAAAAFme5wNahQweNHDnS1WUAAABYhuUCW2n3/fffq2/fvoqIiJCbm5tee+01V5cEAABKuXIZ2LKyspw2dnp6uiIjIzV9+nRVr17dafsBAADlh0sD26VLlxQXF6eAgADVqFFDr7zySq5tMjMzNWbMGN10003y9/dXy5YttXnzZrtt5s2bp9q1a8vPz0+9e/fW7NmzValSJdv6qVOnKjo6WvPnz1edOnXk4+MjSTp//ryGDBmi4OBgBQYG6q677tLevXvtxl61apWaNGkiHx8fRUZGKj4+XlevXs23p+bNm+vll19W//795e3tXfSDAwAA8P+5NLCNHTtWW7Zs0apVq7RhwwZt3rxZ3377rd02TzzxhHbs2KGlS5fqu+++03333afY2FgdOXJEkrRt2zYNHz5cTz31lBITE3X33Xdr2rRpufaVlJSkjz/+WMuXL1diYqIk6b777tOZM2e0bt067d69W02aNFGnTp3022+/SZK2bt2quLg4PfXUUzpw4IDefvttvffee3mODwAA4CwertrxxYsXtWDBAi1ZskSdOnWSJC1cuFC1atWybZOSkqKEhASlpKSoZs2akqQxY8Zo/fr1SkhI0Isvvqi5c+eqW7duGjNmjCQpKipK27dv15o1a+z2l5WVpUWLFik4OFiS9OWXX2rnzp06c+aM7UzYrFmztHLlSn300UcaNmyY4uPjNW7cOA0cOFCSFBkZqeeff17PPPOMpkyZ4pDjkJmZqczMTNvltLQ0h4x7PcnJyaV6fBQft1HBWfFYOWLeKOm+rHgcUTDcdoXnjGPmssB29OhRZWVlqWXLlrZlVapUUf369W2X9+3bp+zsbEVFRdldNzMzU1WrVpUkHTp0SL1797Zb36JFi1yBLTw83BbWJGnv3r26ePGibZxrLl++rKNHj9q22bZtm90ZtezsbGVkZCg9PV1+fn5Fad3OSy+9pPj4+GKPUxicIQT3gdLNEfMG9wEUFPcVa3BZYCuIixcvqkKFCtq9e7cqVKhgty4gIKBQY/n7++cau0aNGrleDyfJ9vq3ixcvKj4+Xn369Mm1zbXXwRXX+PHj9fTTT9sup6WlqXbt2g4ZOz8TJkxQeHi408ZPTk7mAW5xzr4PlCVWvD87Yt4o6fuAFY8jCob5ovCccX93WWC7+eab5enpqa+//lphYWGSpHPnzunw4cNq3769JCkmJkbZ2dk6c+aM2rZtm+c49evX165du+yW/fVyXpo0aaJTp07Jw8NDERER+W5z6NAh1a1btxCdFY63t3eJvzkhPDw811lLlC/cB0o3R8wb3AdQUNxXrMFlgS0gIECDBw/W2LFjVbVqVYWEhGjChAlyd//jfRBRUVEaMGCA4uLi9MorrygmJka//PKLNm7cqEaNGumee+7Rk08+qXbt2mn27Nnq3r27Pv/8c61bt05ubm7X3X/nzp3VunVr9erVSzNnzlRUVJROnDihTz75RL1791azZs00efJk3XvvvQoLC1O/fv3k7u6uvXv3av/+/XrhhRfyHDcrK0sHDhyw/f7zzz8rMTFRAQEBTg1+AACg7HLpu0RffvlltW3bVt27d1fnzp115513qmnTpnbbJCQkKC4uTqNHj1b9+vXVq1cv7dq1y3ZW7o477tBbb72l2bNnq3Hjxlq/fr1GjRp1w6cs3dzctHbtWrVr106PPPKIoqKi1L9/fyUnJys0NFSS1LVrV61Zs0YbNmxQ8+bN1apVK7366qvXPTV84sQJxcTEKCYmRidPntSsWbMUExOjIUOGFPNoAQCA8sqlr2ELCAjQ4sWLtXjxYtuysWPH2m3j6emp+Pj4677AdujQoRo6dKjd5T+fzZo6daqmTp2a63oVK1bUnDlzNGfOnHzH7tq1q7p27VqQdiRJERERMsYUeHsAAIAbsfSbDgpq1qxZuvvuu+Xv769169Zp4cKFeuONN1xdFgAAgEOUicC2c+dOzZw5UxcuXFBkZKTmzJnDU5AAAKDMKBOBbdmyZa4uAQAAwGnK5Ze/AwAAlCYENgAAAIsjsAEAAFgcgQ0AAMDiCGwAAAAWR2ADAACwOAIbAACAxRHYAAAALI7ABgAAYHEENgAAAIsjsAEAAFgcgQ0AAMDiCGwAAAAWR2ArR8LCwvTOO+8oLCzM1aUApUZZe9yUtX4AK3LG48zDYSPB8nx8fBQVFeXqMoBSpaw9bspaP4AVOeNxxhk2AAAAiyOwAQAAWByBDQAAwOIIbAAAABZHYAMAALA4AhsAAIDFEdgAAAAsjsAGAABgcQQ2AAAAiyOwAQAAWByBDQAAwOIIbAAAABZHYAMAALA4AhsAAIDFEdgAAAAsjsAGAABgcR6uLgBll3tGqvPGvnze7l8UjDNvE6AgSvo+yFxRdMwX1kJgg8MFBQXJ08tb+mGL0/fle+wLp++jrPH08lZQUJCry0A5U5LzQl6YK4qG+cI63IwxxtVF4A9paWkKCgpSamqqAgMDXV1OkZ0+fVqpqfx1ZkVBQUEKDQ11dRlwoNIybzAvlD7MF9bBGTY4RWhoKA9yAHaYF4Ci400HAAAAFkdgAwAAsDgCGwAAgMUR2AAAACyOwAYAAGBxBDYAAACLI7ABAABYHJ/DZjHXPsc4LS3NxZUAKKqKFSvKzc2txPbHvAGUbgWZMwhsFnPhwgVJUu3atV1cCYCiKulvHGDeAEq3gswZfDWVxeTk5OjEiRMF/gs9LS1NtWvX1o8//mjpr6QpjLLYk1Q2+yqLPUnF76ukz7AVdt4oCWX1vnEj9E3fRcEZtlLI3d1dtWrVKvT1AgMDy9yDpCz2JJXNvspiT1Lp6auo80ZJKC3H0NHou3wpib550wEAAIDFEdgAAAAsjsBWynl7e2vKlCny9vZ2dSkOUxZ7kspmX2WxJ6ns9lWSyusxpG/6dhbedAAAAGBxnGEDAACwOAIbAACAxRHYAAAALI7AVoL+/e9/KyIiQj4+PmrZsqV27tx53e0//PBD3XLLLfLx8VHDhg21du1au/WDBg2Sm5ub3U9sbKxt/ebNm3Otv/aza9cuSdLx48fzXP/VV1+5rC9JOnjwoHr06KGgoCD5+/urefPmSklJsa3PyMjQiBEjVLVqVQUEBKhv3746ffq03RgpKSm655575Ofnp5CQEI0dO1ZXr161ZE+//fabnnzySdWvX1++vr4KCwvTP//5T6WmptqNkddttXTp0gL15Iq+JKlDhw65ah4+fLjdGKXptsrvMePm5qYPP/zQNkZxbyurcfRxPn36tAYNGqSaNWvKz89PsbGxOnLkiDNbKLLC9P7999+rb9++ioiIkJubm1577bVij+kqju77iy++UPfu3VWzZk25ublp5cqVziu+GBzd90svvaTmzZurYsWKCgkJUa9evXTo0KHCF2ZQIpYuXWq8vLzMu+++a77//nszdOhQU6lSJXP69Ok8t9+2bZupUKGCmTlzpjlw4ICZOHGi8fT0NPv27bNtM3DgQBMbG2tOnjxp+/ntt99s6zMzM+3WnTx50gwZMsTUqVPH5OTkGGOMOXbsmJFkPvvsM7vtsrKyXNZXUlKSqVKlihk7dqz59ttvTVJSklm1apXdmMOHDze1a9c2GzduNN98841p1aqVadOmjW391atXze233246d+5s9uzZY9auXWuqVatmxo8fb8me9u3bZ/r06WNWr15tkpKSzMaNG029evVM37597fYlySQkJNjdVpcvX75hT67qyxhj2rdvb4YOHWpXc2pqqm19abutrl69mutxFR8fbwICAsyFCxds4xTntrIaRx/nnJwc06pVK9O2bVuzc+dO83//939m2LBhJiwszFy8eLEkW7uhwva+c+dOM2bMGPOf//zHVK9e3bz66qvFHtMVnNH32rVrzYQJE8zy5cuNJLNixQrnNlEEzui7a9euJiEhwezfv98kJiaav/3tb0W6rxPYSkiLFi3MiBEjbJezs7NNzZo1zUsvvZTn9vfff7+555577Ja1bNnS/OMf/7BdHjhwoOnZs2eBa8jKyjLBwcHmueeesy27Ftj27NlT4HH+zBl9/f3vfzcPPfRQvvs8f/688fT0NB9++KFt2cGDB40ks2PHDmPM7xODu7u7OXXqlG2bN9980wQGBprMzEzL9ZSXZcuWGS8vL3PlyhXbsuJMcq7qq3379uapp57Kd31ZuK2io6PNo48+arfMqv8hFYWjj/OhQ4eMJLN//367MYODg828efOc0EHRFbb3PwsPD8/zP/DijFlSnNH3n1n18eHsvo0x5syZM0aS2bJlS6Fq4ynREpCVlaXdu3erc+fOtmXu7u7q3LmzduzYked1duzYYbe9JHXt2jXX9ps3b1ZISIjq16+vxx57TGfPns23jtWrV+vs2bN65JFHcq3r0aOHQkJCdOedd2r16tUu6ysnJ0effPKJoqKi1LVrV4WEhKhly5Z2p853796tK1eu2I1zyy23KCwszDbOjh071LBhQ4WGhtrtJy0tTd9//73lesrLtS8D9vCw/wa5ESNGqFq1amrRooXeffddmQJ8Mo+r+3r//fdVrVo13X777Ro/frzS09Pt9lOab6vdu3crMTFRgwcPzrWuKLeV1TjjOGdmZkqSfHx87Mb09vbWl19+6egWiqwovbtiTEcrDTU6Q0n1fe2lLlWqVCnU9QhsJeDXX39Vdna23X9IkhQaGqpTp07leZ1Tp07dcPvY2FgtWrRIGzdu1IwZM7RlyxZ169ZN2dnZeY65YMECde3a1e47BwMCAvTKK6/oww8/1CeffKI777xTvXr1KlBoc0ZfZ86c0cWLFzV9+nTFxsZqw4YN6t27t/r06aMtW7bYxvDy8lKlSpXyHSe//VxbZ7We8qrj+eef17Bhw+yWP/fcc1q2bJn++9//qm/fvnr88cc1d+7cfPuxQl8PPviglixZok2bNmn8+PFavHixHnrooRvu59o6K/b0ZwsWLFCDBg3Upk0bu+VFva2sxhnH+dofWOPHj9e5c+eUlZWlGTNm6KefftLJkyed00gRFKV3V4zpaKWhRmcoib5zcnI0cuRI3XHHHbr99tsLdV2+/L0U69+/v+33hg0bqlGjRrr55pu1efNmderUyW7bn376SZ9++qmWLVtmt7xatWp6+umnbZebN2+uEydO6OWXX1aPHj2c20AecnJyJEk9e/bUqFGjJEnR0dHavn273nrrLbVv377EayquwvaUlpame+65R7feequmTp1qt27SpEm232NiYnTp0iW9/PLL+uc//+ncJvJQ0L7+HDobNmyoGjVqqFOnTjp69KhuvvnmEq/7egp7W12+fFkffPCB3e1yjZVuK6vx9PTU8uXLNXjwYFWpUkUVKlRQ586d1a1bt1J5FhIoqBEjRmj//v1FOpPMGbYSUK1aNVWoUCHXuxhPnz6t6tWr53md6tWrF2p7SYqMjFS1atWUlJSUa11CQoKqVq1aoBDWsmXLPMf4K2f0Va1aNXl4eOjWW2+126ZBgwa2d+lVr15dWVlZOn/+fL7j5Lefa+us1tM1Fy5cUGxsrCpWrKgVK1bI09Mz31ql32+rn376yfYUk1X7+mvNkmz3sdJ6W0nSRx99pPT0dMXFxeVb5zUFva2sxlnzV9OmTZWYmKjz58/r5MmTWr9+vc6ePavIyEjHN1FERendFWM6Wmmo0Rmc3fcTTzyhNWvWaNOmTXbPdBUUga0EeHl5qWnTptq4caNtWU5OjjZu3KjWrVvneZ3WrVvbbS9J//3vf/PdXvr9LNrZs2dVo0YNu+XGGCUkJCguLu6GAUCSEhMTc42RF2f05eXlpebNm+d6y/Phw4cVHh4u6feJ3tPT026cQ4cOKSUlxTZO69attW/fPp05c8ZuP4GBgbn+M7ZCT9LvZ9a6dOkiLy8vrV692u71PflJTExU5cqVb/g9dq7sK6+aJdnuY6XxtrpmwYIF6tGjh4KDg/Ot85qC3lZW4+z5KygoSMHBwTpy5Ii++eYb9ezZ07ENFENRenfFmI5WGmp0Bmf1bYzRE088oRUrVujzzz9XnTp1ijwQSsDSpUuNt7e3ee+998yBAwfMsGHDTKVKlWzvjHv44YfNuHHjbNtv27bNeHh4mFmzZpmDBw+aKVOm2L0t/sKFC2bMmDFmx44d5tixY+azzz4zTZo0MfXq1TMZGRl2+/7ss8+MJHPw4MFcdb333nvmgw8+MAcPHjQHDx4006ZNM+7u7ubdd991SV/GGLN8+XLj6elp3nnnHXPkyBEzd+5cU6FCBbN161bbNsOHDzdhYWHm888/N998841p3bq1ad26tW39tY+K6NKli0lMTDTr1683wcHBBf6oiJLuKTU11bRs2dI0bNjQJCUl2X0UxNWrV40xxqxevdrMmzfP7Nu3zxw5csS88cYbxs/Pz0yePNmyt1VSUpJ57rnnzDfffGOOHTtmVq1aZSIjI027du1K7W11zZEjR4ybm5tZt25drrqKe1tZjTOO87Jly8ymTZvM0aNHzcqVK014eLjp06dPifd2I4XtPTMz0+zZs8fs2bPH1KhRw4wZM8bs2bPHHDlypMBjWoEz+r5w4YJtG0lm9uzZZs+ePSY5ObnE+8uPM/p+7LHHTFBQkNm8ebPd3J6enl6o2ghsJWju3LkmLCzMeHl5mRYtWpivvvrKtq59+/Zm4MCBdtsvW7bMREVFGS8vL3PbbbeZTz75xLYuPT3ddOnSxQQHBxtPT08THh5uhg4dmucD/oEHHrD7jLI/e++990yDBg2Mn5+fCQwMNC1atLD7uIyS7uuaBQsWmLp16xofHx/TuHFjs3LlSrv1ly9fNo8//ripXLmy8fPzM7179zYnT5602+b48eOmW7duxtfX11SrVs2MHj3a7iMyrNTTpk2bjKQ8f44dO2aMMWbdunUmOjraBAQEGH9/f9O4cWPz1ltvmezs7AL15Iq+UlJSTLt27UyVKlWMt7e3qVu3rhk7dqzd57AZU7puq2vGjx9vateunefxd8RtZTWOPs6vv/66qVWrlvH09DRhYWFm4sSJN/wYF1cpTO/XPirprz/t27cv8JhW4ei+85vn/nrfcTVH953f3J6QkFCoutz+/2AAAACwKF7DBgAAYHEENgAAAIsjsAEAAFgcgQ0AAMDiCGwAAAAWR2ADAACwOAIbAACAxRHYAAAALI7ABhRThw4dNHLkSFeXAaAUYd5AYRHYAAAALI7ABgAAYHEENsCBzp07p7i4OFWuXFl+fn7q1q2bjhw5YlufnJys7t27q3LlyvL399dtt92mtWvX2q47YMAABQcHy9fXV/Xq1VNCQoKrWgFQQpg3UBAeri4AKEsGDRqkI0eOaPXq1QoMDNS//vUv/e1vf9OBAwfk6empESNGKCsrS1988YX8/f114MABBQQESJImTZqkAwcOaN26dapWrZqSkpJ0+fJlF3cEwNmYN1AQBDbAQa5NuNu2bVObNm0kSe+//75q166tlStX6r777lNKSor69u2rhg0bSpIiIyNt109JSVFMTIyaNWsmSYqIiCjxHgCULOYNFBRPiQIOcvDgQXl4eKhly5a2ZVWrVlX9+vV18OBBSdI///lPvfDCC7rjjjs0ZcoUfffdd7ZtH3vsMS1dulTR0dF65plntH379hLvAUDJYt5AQRHYgBI0ZMgQ/fDDD3r44Ye1b98+NWvWTHPnzpUkdevWTcnJyRo1apROnDihTp06acyYMS6uGICrMW9AIrABDtOgQQNdvXpVX3/9tW3Z2bNndejQId166622ZbVr19bw4cO1fPlyjR49WvPmzbOtCw4O1sCBA7VkyRK99tpreuedd0q0BwAli3kDBcVr2AAHqVevnnr27KmhQ4fq7bffVsWKFTVu3DjddNNN6tmzpyRp5MiR6tatm6KionTu3Dlt2rRJDRo0kCRNnjxZTZs21W233abMzEytWbPGtg5A2cS8gYLiDBvgQAkJCWratKnuvfdetW7dWsYYrV27Vp6enpKk7OxsjRgxQg0aNFBsbKyioqL0xhtvSJK8vLw0fvx4NWrUSO3atVOFChW0dOlSV7YDoAQwb6Ag3IwxxtVFAAAAIH+cYQMAALA4AhsAAIDFEdgAAAAsjsAGAABgcQQ2AAAAiyOwAQAAWByBDQAAwOIIbAAAABZHYAMAALA4AhsAAIDFEdgAAAAsjsAGAABgcf8PFuNnZS48Hk8AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 900x300 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_cross_validation_result(res)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are no major differences over the settings above, for simplicity we can take the best performing model on average over all cell types:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "setting\n",
       "degree 2    0.081324\n",
       "degree 1    0.081365\n",
       "Name: loss, dtype: float64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res.groupby('setting').loss.mean().sort_values()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}