analyze_experiment_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-08T14:13:43.150947782Z",
     "start_time": "2024-02-08T14:13:43.149833927Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2675424/2837410967.py:2: DeprecationWarning: \n",
      "Pyarrow will become a required dependency of pandas in the next major release of pandas (pandas 3.0),\n",
      "(to allow more performant data types, such as the Arrow string type, and better interoperability with other libraries)\n",
      "but was not found to be installed on your system.\n",
      "If this would cause problems for you,\n",
      "please provide us feedback at https://github.com/pandas-dev/pandas/issues/54466\n",
      "        \n",
      "  import pandas as pd\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import json\n",
    "\n",
    "def get_split(replica_file):\n",
    "    replica = json.load(open(replica_file, \"r\"))\n",
    "    return replica[\"py/state\"][\"prepSettings\"][\"split\"]['py/object'].split(\".\")[-1]\n",
    "\n",
    "def get_results(data_folder, prefix):\n",
    "    results = []\n",
    "    for res_dir in [os.path.join(data_folder, x) for x in os.listdir(data_folder) if x.startswith(prefix)]:\n",
    "        for res_file in [os.path.join(res_dir, x) for x in os.listdir(res_dir)]:\n",
    "            if os.path.basename(res_file).startswith(\"results_\"):\n",
    "                res_df = pd.read_table(res_file)\n",
    "                res_df[\"Split\"] = res_df.ReplicaFile.apply(get_split)\n",
    "                results.append(res_df)\n",
    "    return pd.concat(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "659bfe8f-e90c-4a79-8964-ed4a78459322",
   "metadata": {},
   "outputs": [],
   "source": [
    "# regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d1519ad5-7c02-4013-91f8-4270c7e533d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Assessor</th>\n",
       "      <th>ScoreFunc</th>\n",
       "      <th>Score</th>\n",
       "      <th>TargetProperty</th>\n",
       "      <th>TargetTask</th>\n",
       "      <th>ModelFile</th>\n",
       "      <th>Algorithm</th>\n",
       "      <th>AlgorithmParams</th>\n",
       "      <th>ReplicaID</th>\n",
       "      <th>DataSet</th>\n",
       "      <th>ReplicaFile</th>\n",
       "      <th>Split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TestSetAssessor</td>\n",
       "      <td>r2_score</td>\n",
       "      <td>-1.682650</td>\n",
       "      <td>measured log solubility in mols per litre</td>\n",
       "      <td>REGRESSION</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>STFullyConnected</td>\n",
       "      <td>NaN</td>\n",
       "      <td>regression_dnn_107420369</td>\n",
       "      <td>delaney-processed_MorganFP_gpu</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>RandomSplit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TestSetAssessor</td>\n",
       "      <td>root_mean_squared_error</td>\n",
       "      <td>3.308069</td>\n",
       "      <td>measured log solubility in mols per litre</td>\n",
       "      <td>REGRESSION</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>STFullyConnected</td>\n",
       "      <td>NaN</td>\n",
       "      <td>regression_dnn_107420369</td>\n",
       "      <td>delaney-processed_MorganFP_gpu</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>RandomSplit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TestSetAssessor</td>\n",
       "      <td>r2_score</td>\n",
       "      <td>-1.538419</td>\n",
       "      <td>measured log solubility in mols per litre</td>\n",
       "      <td>REGRESSION</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>STFullyConnected</td>\n",
       "      <td>NaN</td>\n",
       "      <td>regression_dnn_3184935163</td>\n",
       "      <td>delaney-processed_MorganFP_gpu</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>ClusterSplit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TestSetAssessor</td>\n",
       "      <td>root_mean_squared_error</td>\n",
       "      <td>3.113120</td>\n",
       "      <td>measured log solubility in mols per litre</td>\n",
       "      <td>REGRESSION</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>STFullyConnected</td>\n",
       "      <td>NaN</td>\n",
       "      <td>regression_dnn_3184935163</td>\n",
       "      <td>delaney-processed_MorganFP_gpu</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>ClusterSplit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TestSetAssessor</td>\n",
       "      <td>r2_score</td>\n",
       "      <td>-1.312194</td>\n",
       "      <td>measured log solubility in mols per litre</td>\n",
       "      <td>REGRESSION</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>STFullyConnected</td>\n",
       "      <td>NaN</td>\n",
       "      <td>regression_dnn_958682846</td>\n",
       "      <td>delaney-processed_MorganFP_gpu</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>RandomSplit</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Assessor                ScoreFunc     Score  \\\n",
       "0  TestSetAssessor                 r2_score -1.682650   \n",
       "1  TestSetAssessor  root_mean_squared_error  3.308069   \n",
       "2  TestSetAssessor                 r2_score -1.538419   \n",
       "3  TestSetAssessor  root_mean_squared_error  3.113120   \n",
       "4  TestSetAssessor                 r2_score -1.312194   \n",
       "\n",
       "                              TargetProperty  TargetTask  \\\n",
       "0  measured log solubility in mols per litre  REGRESSION   \n",
       "1  measured log solubility in mols per litre  REGRESSION   \n",
       "2  measured log solubility in mols per litre  REGRESSION   \n",
       "3  measured log solubility in mols per litre  REGRESSION   \n",
       "4  measured log solubility in mols per litre  REGRESSION   \n",
       "\n",
       "                                           ModelFile         Algorithm  \\\n",
       "0  /home/martin/projects/qsp-bench-dev/data/regre...  STFullyConnected   \n",
       "1  /home/martin/projects/qsp-bench-dev/data/regre...  STFullyConnected   \n",
       "2  /home/martin/projects/qsp-bench-dev/data/regre...  STFullyConnected   \n",
       "3  /home/martin/projects/qsp-bench-dev/data/regre...  STFullyConnected   \n",
       "4  /home/martin/projects/qsp-bench-dev/data/regre...  STFullyConnected   \n",
       "\n",
       "  AlgorithmParams                  ReplicaID                         DataSet  \\\n",
       "0             NaN   regression_dnn_107420369  delaney-processed_MorganFP_gpu   \n",
       "1             NaN   regression_dnn_107420369  delaney-processed_MorganFP_gpu   \n",
       "2             NaN  regression_dnn_3184935163  delaney-processed_MorganFP_gpu   \n",
       "3             NaN  regression_dnn_3184935163  delaney-processed_MorganFP_gpu   \n",
       "4             NaN   regression_dnn_958682846  delaney-processed_MorganFP_gpu   \n",
       "\n",
       "                                         ReplicaFile         Split  \n",
       "0  /home/martin/projects/qsp-bench-dev/data/regre...   RandomSplit  \n",
       "1  /home/martin/projects/qsp-bench-dev/data/regre...   RandomSplit  \n",
       "2  /home/martin/projects/qsp-bench-dev/data/regre...  ClusterSplit  \n",
       "3  /home/martin/projects/qsp-bench-dev/data/regre...  ClusterSplit  \n",
       "4  /home/martin/projects/qsp-bench-dev/data/regre...   RandomSplit  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_reg_results = get_results(\"./data/\", \"regression\")\n",
    "df_reg_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "487c551fa6af5445",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-08T14:14:24.241519711Z",
     "start_time": "2024-02-08T14:13:43.416728417Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# classification (uncomment if ran as well)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "85bfa48f-7229-4e8a-9dda-78ab5d67e9f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_cls_results = get_results(\"./data/\", \"classification\")\n",
    "# df_cls_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d1abe8a68a95280b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-08T14:14:24.242030891Z",
     "start_time": "2024-02-08T14:14:24.176324410Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Assessor</th>\n",
       "      <th>ScoreFunc</th>\n",
       "      <th>Score</th>\n",
       "      <th>TargetProperty</th>\n",
       "      <th>TargetTask</th>\n",
       "      <th>ModelFile</th>\n",
       "      <th>Algorithm</th>\n",
       "      <th>AlgorithmParams</th>\n",
       "      <th>ReplicaID</th>\n",
       "      <th>DataSet</th>\n",
       "      <th>ReplicaFile</th>\n",
       "      <th>Split</th>\n",
       "      <th>ScoreFuncNice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TestSetAssessor</td>\n",
       "      <td>r2_score</td>\n",
       "      <td>-0.000533</td>\n",
       "      <td>y</td>\n",
       "      <td>REGRESSION</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>MedianDistributionAlgorithm</td>\n",
       "      <td>NaN</td>\n",
       "      <td>regression_478163327</td>\n",
       "      <td>freesolv_MorganFP</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>RandomSplit</td>\n",
       "      <td>$R^2$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TestSetAssessor</td>\n",
       "      <td>root_mean_squared_error</td>\n",
       "      <td>0.893743</td>\n",
       "      <td>y</td>\n",
       "      <td>REGRESSION</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>MedianDistributionAlgorithm</td>\n",
       "      <td>NaN</td>\n",
       "      <td>regression_478163327</td>\n",
       "      <td>freesolv_MorganFP</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>RandomSplit</td>\n",
       "      <td>RMSE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>TestSetAssessor</td>\n",
       "      <td>r2_score</td>\n",
       "      <td>-0.006679</td>\n",
       "      <td>y</td>\n",
       "      <td>REGRESSION</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>MedianDistributionAlgorithm</td>\n",
       "      <td>NaN</td>\n",
       "      <td>regression_3184935163</td>\n",
       "      <td>freesolv_MorganFP</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>ClusterSplit</td>\n",
       "      <td>$R^2$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>TestSetAssessor</td>\n",
       "      <td>root_mean_squared_error</td>\n",
       "      <td>0.863126</td>\n",
       "      <td>y</td>\n",
       "      <td>REGRESSION</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>MedianDistributionAlgorithm</td>\n",
       "      <td>NaN</td>\n",
       "      <td>regression_3184935163</td>\n",
       "      <td>freesolv_MorganFP</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>ClusterSplit</td>\n",
       "      <td>RMSE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>TestSetAssessor</td>\n",
       "      <td>r2_score</td>\n",
       "      <td>-0.002740</td>\n",
       "      <td>y</td>\n",
       "      <td>REGRESSION</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>MedianDistributionAlgorithm</td>\n",
       "      <td>NaN</td>\n",
       "      <td>regression_1051802512</td>\n",
       "      <td>freesolv_MorganFP</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>RandomSplit</td>\n",
       "      <td>$R^2$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229</th>\n",
       "      <td>TestSetAssessor</td>\n",
       "      <td>root_mean_squared_error</td>\n",
       "      <td>1.751169</td>\n",
       "      <td>measured log solubility in mols per litre</td>\n",
       "      <td>REGRESSION</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>XGBRegressor</td>\n",
       "      <td>{\"n_jobs\": 1, \"random_state\": 3131575764}</td>\n",
       "      <td>regression_3131575764</td>\n",
       "      <td>delaney-processed_MorganFP</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>ClusterSplit</td>\n",
       "      <td>RMSE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232</th>\n",
       "      <td>TestSetAssessor</td>\n",
       "      <td>r2_score</td>\n",
       "      <td>0.731126</td>\n",
       "      <td>measured log solubility in mols per litre</td>\n",
       "      <td>REGRESSION</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>XGBRegressor</td>\n",
       "      <td>{\"n_jobs\": 1, \"random_state\": 701808367}</td>\n",
       "      <td>regression_701808367</td>\n",
       "      <td>delaney-processed_MorganFP</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>RandomSplit</td>\n",
       "      <td>$R^2$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233</th>\n",
       "      <td>TestSetAssessor</td>\n",
       "      <td>root_mean_squared_error</td>\n",
       "      <td>1.060222</td>\n",
       "      <td>measured log solubility in mols per litre</td>\n",
       "      <td>REGRESSION</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>XGBRegressor</td>\n",
       "      <td>{\"n_jobs\": 1, \"random_state\": 701808367}</td>\n",
       "      <td>regression_701808367</td>\n",
       "      <td>delaney-processed_MorganFP</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>RandomSplit</td>\n",
       "      <td>RMSE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236</th>\n",
       "      <td>TestSetAssessor</td>\n",
       "      <td>r2_score</td>\n",
       "      <td>0.418511</td>\n",
       "      <td>measured log solubility in mols per litre</td>\n",
       "      <td>REGRESSION</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>XGBRegressor</td>\n",
       "      <td>{\"n_jobs\": 1, \"random_state\": 1629748727}</td>\n",
       "      <td>regression_1629748727</td>\n",
       "      <td>delaney-processed_MorganFP</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>ClusterSplit</td>\n",
       "      <td>$R^2$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>TestSetAssessor</td>\n",
       "      <td>root_mean_squared_error</td>\n",
       "      <td>1.469275</td>\n",
       "      <td>measured log solubility in mols per litre</td>\n",
       "      <td>REGRESSION</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>XGBRegressor</td>\n",
       "      <td>{\"n_jobs\": 1, \"random_state\": 1629748727}</td>\n",
       "      <td>regression_1629748727</td>\n",
       "      <td>delaney-processed_MorganFP</td>\n",
       "      <td>/home/martin/projects/qsp-bench-dev/data/regre...</td>\n",
       "      <td>ClusterSplit</td>\n",
       "      <td>RMSE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1440 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Assessor                ScoreFunc     Score  \\\n",
       "2    TestSetAssessor                 r2_score -0.000533   \n",
       "3    TestSetAssessor  root_mean_squared_error  0.893743   \n",
       "6    TestSetAssessor                 r2_score -0.006679   \n",
       "7    TestSetAssessor  root_mean_squared_error  0.863126   \n",
       "10   TestSetAssessor                 r2_score -0.002740   \n",
       "..               ...                      ...       ...   \n",
       "229  TestSetAssessor  root_mean_squared_error  1.751169   \n",
       "232  TestSetAssessor                 r2_score  0.731126   \n",
       "233  TestSetAssessor  root_mean_squared_error  1.060222   \n",
       "236  TestSetAssessor                 r2_score  0.418511   \n",
       "237  TestSetAssessor  root_mean_squared_error  1.469275   \n",
       "\n",
       "                                TargetProperty  TargetTask  \\\n",
       "2                                            y  REGRESSION   \n",
       "3                                            y  REGRESSION   \n",
       "6                                            y  REGRESSION   \n",
       "7                                            y  REGRESSION   \n",
       "10                                           y  REGRESSION   \n",
       "..                                         ...         ...   \n",
       "229  measured log solubility in mols per litre  REGRESSION   \n",
       "232  measured log solubility in mols per litre  REGRESSION   \n",
       "233  measured log solubility in mols per litre  REGRESSION   \n",
       "236  measured log solubility in mols per litre  REGRESSION   \n",
       "237  measured log solubility in mols per litre  REGRESSION   \n",
       "\n",
       "                                             ModelFile  \\\n",
       "2    /home/martin/projects/qsp-bench-dev/data/regre...   \n",
       "3    /home/martin/projects/qsp-bench-dev/data/regre...   \n",
       "6    /home/martin/projects/qsp-bench-dev/data/regre...   \n",
       "7    /home/martin/projects/qsp-bench-dev/data/regre...   \n",
       "10   /home/martin/projects/qsp-bench-dev/data/regre...   \n",
       "..                                                 ...   \n",
       "229  /home/martin/projects/qsp-bench-dev/data/regre...   \n",
       "232  /home/martin/projects/qsp-bench-dev/data/regre...   \n",
       "233  /home/martin/projects/qsp-bench-dev/data/regre...   \n",
       "236  /home/martin/projects/qsp-bench-dev/data/regre...   \n",
       "237  /home/martin/projects/qsp-bench-dev/data/regre...   \n",
       "\n",
       "                       Algorithm                            AlgorithmParams  \\\n",
       "2    MedianDistributionAlgorithm                                        NaN   \n",
       "3    MedianDistributionAlgorithm                                        NaN   \n",
       "6    MedianDistributionAlgorithm                                        NaN   \n",
       "7    MedianDistributionAlgorithm                                        NaN   \n",
       "10   MedianDistributionAlgorithm                                        NaN   \n",
       "..                           ...                                        ...   \n",
       "229                 XGBRegressor  {\"n_jobs\": 1, \"random_state\": 3131575764}   \n",
       "232                 XGBRegressor   {\"n_jobs\": 1, \"random_state\": 701808367}   \n",
       "233                 XGBRegressor   {\"n_jobs\": 1, \"random_state\": 701808367}   \n",
       "236                 XGBRegressor  {\"n_jobs\": 1, \"random_state\": 1629748727}   \n",
       "237                 XGBRegressor  {\"n_jobs\": 1, \"random_state\": 1629748727}   \n",
       "\n",
       "                 ReplicaID                     DataSet  \\\n",
       "2     regression_478163327           freesolv_MorganFP   \n",
       "3     regression_478163327           freesolv_MorganFP   \n",
       "6    regression_3184935163           freesolv_MorganFP   \n",
       "7    regression_3184935163           freesolv_MorganFP   \n",
       "10   regression_1051802512           freesolv_MorganFP   \n",
       "..                     ...                         ...   \n",
       "229  regression_3131575764  delaney-processed_MorganFP   \n",
       "232   regression_701808367  delaney-processed_MorganFP   \n",
       "233   regression_701808367  delaney-processed_MorganFP   \n",
       "236  regression_1629748727  delaney-processed_MorganFP   \n",
       "237  regression_1629748727  delaney-processed_MorganFP   \n",
       "\n",
       "                                           ReplicaFile         Split  \\\n",
       "2    /home/martin/projects/qsp-bench-dev/data/regre...   RandomSplit   \n",
       "3    /home/martin/projects/qsp-bench-dev/data/regre...   RandomSplit   \n",
       "6    /home/martin/projects/qsp-bench-dev/data/regre...  ClusterSplit   \n",
       "7    /home/martin/projects/qsp-bench-dev/data/regre...  ClusterSplit   \n",
       "10   /home/martin/projects/qsp-bench-dev/data/regre...   RandomSplit   \n",
       "..                                                 ...           ...   \n",
       "229  /home/martin/projects/qsp-bench-dev/data/regre...  ClusterSplit   \n",
       "232  /home/martin/projects/qsp-bench-dev/data/regre...   RandomSplit   \n",
       "233  /home/martin/projects/qsp-bench-dev/data/regre...   RandomSplit   \n",
       "236  /home/martin/projects/qsp-bench-dev/data/regre...  ClusterSplit   \n",
       "237  /home/martin/projects/qsp-bench-dev/data/regre...  ClusterSplit   \n",
       "\n",
       "    ScoreFuncNice  \n",
       "2           $R^2$  \n",
       "3            RMSE  \n",
       "6           $R^2$  \n",
       "7            RMSE  \n",
       "10          $R^2$  \n",
       "..            ...  \n",
       "229          RMSE  \n",
       "232         $R^2$  \n",
       "233          RMSE  \n",
       "236         $R^2$  \n",
       "237          RMSE  \n",
       "\n",
       "[1440 rows x 13 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_reg_results[\"ScoreFuncNice\"]  = df_reg_results.ScoreFunc.apply(lambda x: {\"r2_score\": r'$R^2$', \"root_mean_squared_error\": \"RMSE\"}[x])\n",
    "df_reg_results = [\n",
    "    df_reg_results[df_reg_results.Algorithm == \"MedianDistributionAlgorithm\"],\n",
    "    df_reg_results[df_reg_results.Algorithm == \"ChempropMoleculeModel\"],\n",
    "    df_reg_results[df_reg_results.Algorithm == \"XGBRegressor\"],\n",
    "]\n",
    "df_reg_results = pd.concat(df_reg_results)\n",
    "df_reg_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9ca2ed6330c92dcb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-08T14:14:26.434303926Z",
     "start_time": "2024-02-08T14:14:24.176592388Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "from utils.plotting import make_box_plot\n",
    "\n",
    "def make_plot(df, prefix, out_dir):\n",
    "    df[\"Data Set\"] = df.DataSet.apply(lambda x: x.split(\"_\")[0])\n",
    "    # df = df[df.Algorithm != \"STFullyConnected\"].sort_values(by=[\"DataSetName\", \"Algorithm\"])\n",
    "    for split_name, df in df.groupby(\"Split\"):\n",
    "        make_box_plot(\n",
    "            df,\n",
    "            x=\"Data Set\",\n",
    "            y=\"Score\",\n",
    "            hue=\"Algorithm\",\n",
    "            plot_name=f\"{prefix}{split_name}\",\n",
    "            ylim=(-0.5, 1),\n",
    "            out_dir=out_dir,\n",
    "            figsize=(8, 4.5),\n",
    "            colors = [\n",
    "                # \"#DEF9C4\",\n",
    "                # \"#9CDBA6\",\n",
    "                # \"#50B498\",\n",
    "                # \"#468585\",\n",
    "                \"#FABC3F\",\n",
    "                \"#E85C0D\",\n",
    "                \"#C7253E\",\n",
    "            ]\n",
    "        )\n",
    "\n",
    "os.makedirs('./results_exp2', exist_ok=True)\n",
    "make_plot(df_reg_results, \"\", out_dir='./results_exp2')\n",
    "# make_plot(df_cls_results, \"classification\", out_dir='./results_exp2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d5457329-77c1-4df5-a1c0-2928b26db1bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_reg_results.to_csv('./results_exp2/regression.csv', header=True, index=False)\n",
    "# df_cls_results.to_csv('./results_exp2/classification.csv', header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "336ddb2b-631f-4ca3-b0b4-38a37036451a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}