mlp_hyper1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f7fbfe8e-cce0-4ab7-948f-1aa0f94fb992",
   "metadata": {},
   "outputs": [],
   "source": [
    "# %load_ext autotime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ac794bb8-5583-4fc8-bcad-44094343c561",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "540c7fdd-56fd-41e0-a60d-b2aa171d0a2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "# from mei import Size_Distribution_Optics\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7ba7f0b9-5d79-4805-a484-c08748a61bb9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import dask.dataframe as dd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "482b3e17-0e20-4b96-9833-2a7132eb34f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn\n",
    "from sklearn.preprocessing import MinMaxScaler, PowerTransformer, QuantileTransformer\n",
    "from sklearn.metrics import r2_score \n",
    "import joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "410594a8-c22b-49a0-ae50-d4756b7111d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-08-17 19:08:14.510381: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  SSE4.1 SSE4.2 AVX AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Flatten\n",
    "from tensorflow.keras.layers import Dense, Dropout\n",
    "from tensorflow.keras.layers import Activation\n",
    "\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, TerminateOnNaN, ReduceLROnPlateau, EarlyStopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9d7721d5-aa98-421a-b1ed-1ddb9efef636",
   "metadata": {},
   "outputs": [],
   "source": [
    "from kerastuner.tuners import RandomSearch, BayesianOptimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a9bf066a-263a-4970-8967-4a4b8e272184",
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(df):\n",
    "    return (df - df.mean()) / df.std()\n",
    "\n",
    "def normalize1(df):    \n",
    "    mx = df.max()\n",
    "    mn = df.min()\n",
    "    \n",
    "    mx['SSA'] = 1.0\n",
    "    mx['Asym'] = 1.0\n",
    "    \n",
    "    mn['SSA'] = 0.0\n",
    "    mn['Asym'] = 0.0\n",
    "    \n",
    "    new = (df - mn) / (mx - mn)    \n",
    "    return new\n",
    "\n",
    "\n",
    "def load_opt(filename):\n",
    "    with open(filename, \"r\") as fp:\n",
    "        data = json.load(fp)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "69998211-344c-4b57-bbe5-ca7e96cdb005",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Extinction</th>\n",
       "      <th>Scattering</th>\n",
       "      <th>SSA</th>\n",
       "      <th>Asym</th>\n",
       "      <th>coating</th>\n",
       "      <th>core_dia</th>\n",
       "      <th>n_core</th>\n",
       "      <th>k_core</th>\n",
       "      <th>n_shell</th>\n",
       "      <th>k_shell</th>\n",
       "      <th>density</th>\n",
       "      <th>lambda</th>\n",
       "      <th>x</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.371424</td>\n",
       "      <td>1.497649</td>\n",
       "      <td>0.631540</td>\n",
       "      <td>0.898342</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>1.451990</td>\n",
       "      <td>0.022362</td>\n",
       "      <td>1.330928</td>\n",
       "      <td>4.128916e-08</td>\n",
       "      <td>2.6</td>\n",
       "      <td>700.0</td>\n",
       "      <td>17.951958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.203139</td>\n",
       "      <td>1.330449</td>\n",
       "      <td>0.603888</td>\n",
       "      <td>0.882517</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>1.453480</td>\n",
       "      <td>0.023396</td>\n",
       "      <td>1.327650</td>\n",
       "      <td>2.061950e-06</td>\n",
       "      <td>2.6</td>\n",
       "      <td>950.0</td>\n",
       "      <td>16.534698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.173950</td>\n",
       "      <td>1.202943</td>\n",
       "      <td>0.553344</td>\n",
       "      <td>0.947042</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9000.0</td>\n",
       "      <td>1.451020</td>\n",
       "      <td>0.018632</td>\n",
       "      <td>1.332398</td>\n",
       "      <td>9.574458e-09</td>\n",
       "      <td>2.6</td>\n",
       "      <td>600.0</td>\n",
       "      <td>47.123890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.068879</td>\n",
       "      <td>1.123843</td>\n",
       "      <td>0.543213</td>\n",
       "      <td>0.952623</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>1.483700</td>\n",
       "      <td>0.099080</td>\n",
       "      <td>1.389200</td>\n",
       "      <td>9.470000e-08</td>\n",
       "      <td>2.6</td>\n",
       "      <td>210.0</td>\n",
       "      <td>149.599650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.150144</td>\n",
       "      <td>1.163233</td>\n",
       "      <td>0.541003</td>\n",
       "      <td>0.941646</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>1.453000</td>\n",
       "      <td>0.022940</td>\n",
       "      <td>1.329120</td>\n",
       "      <td>3.110181e-07</td>\n",
       "      <td>2.6</td>\n",
       "      <td>850.0</td>\n",
       "      <td>36.959914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599995</th>\n",
       "      <td>0.184309</td>\n",
       "      <td>0.033168</td>\n",
       "      <td>0.179958</td>\n",
       "      <td>0.086705</td>\n",
       "      <td>0.4</td>\n",
       "      <td>700.0</td>\n",
       "      <td>1.394680</td>\n",
       "      <td>0.006191</td>\n",
       "      <td>1.420600</td>\n",
       "      <td>1.426000e-01</td>\n",
       "      <td>1.7</td>\n",
       "      <td>4600.0</td>\n",
       "      <td>0.478068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599996</th>\n",
       "      <td>0.181849</td>\n",
       "      <td>0.010640</td>\n",
       "      <td>0.058508</td>\n",
       "      <td>0.054755</td>\n",
       "      <td>0.4</td>\n",
       "      <td>900.0</td>\n",
       "      <td>1.364826</td>\n",
       "      <td>0.022669</td>\n",
       "      <td>1.276529</td>\n",
       "      <td>1.827067e-01</td>\n",
       "      <td>1.7</td>\n",
       "      <td>7100.0</td>\n",
       "      <td>0.398230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599997</th>\n",
       "      <td>0.265074</td>\n",
       "      <td>0.004225</td>\n",
       "      <td>0.015940</td>\n",
       "      <td>0.016758</td>\n",
       "      <td>0.4</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>1.613500</td>\n",
       "      <td>0.648098</td>\n",
       "      <td>2.010000</td>\n",
       "      <td>6.500000e-01</td>\n",
       "      <td>1.7</td>\n",
       "      <td>50000.0</td>\n",
       "      <td>0.188496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599998</th>\n",
       "      <td>0.193847</td>\n",
       "      <td>0.001301</td>\n",
       "      <td>0.006709</td>\n",
       "      <td>0.009228</td>\n",
       "      <td>0.4</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>1.613500</td>\n",
       "      <td>0.648098</td>\n",
       "      <td>2.020000</td>\n",
       "      <td>6.850000e-01</td>\n",
       "      <td>1.7</td>\n",
       "      <td>90000.0</td>\n",
       "      <td>0.139626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599999</th>\n",
       "      <td>0.819451</td>\n",
       "      <td>0.106342</td>\n",
       "      <td>0.129773</td>\n",
       "      <td>0.083984</td>\n",
       "      <td>0.4</td>\n",
       "      <td>12000.0</td>\n",
       "      <td>1.613500</td>\n",
       "      <td>0.648098</td>\n",
       "      <td>2.020000</td>\n",
       "      <td>6.850000e-01</td>\n",
       "      <td>1.7</td>\n",
       "      <td>90000.0</td>\n",
       "      <td>0.418879</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>600000 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Extinction  Scattering       SSA      Asym  coating  core_dia  \\\n",
       "0         2.371424    1.497649  0.631540  0.898342      0.0    4000.0   \n",
       "1         2.203139    1.330449  0.603888  0.882517      0.0    5000.0   \n",
       "2         2.173950    1.202943  0.553344  0.947042      0.0    9000.0   \n",
       "3         2.068879    1.123843  0.543213  0.952623      0.0   10000.0   \n",
       "4         2.150144    1.163233  0.541003  0.941646      0.0   10000.0   \n",
       "...            ...         ...       ...       ...      ...       ...   \n",
       "599995    0.184309    0.033168  0.179958  0.086705      0.4     700.0   \n",
       "599996    0.181849    0.010640  0.058508  0.054755      0.4     900.0   \n",
       "599997    0.265074    0.004225  0.015940  0.016758      0.4    3000.0   \n",
       "599998    0.193847    0.001301  0.006709  0.009228      0.4    4000.0   \n",
       "599999    0.819451    0.106342  0.129773  0.083984      0.4   12000.0   \n",
       "\n",
       "          n_core    k_core   n_shell       k_shell  density   lambda  \\\n",
       "0       1.451990  0.022362  1.330928  4.128916e-08      2.6    700.0   \n",
       "1       1.453480  0.023396  1.327650  2.061950e-06      2.6    950.0   \n",
       "2       1.451020  0.018632  1.332398  9.574458e-09      2.6    600.0   \n",
       "3       1.483700  0.099080  1.389200  9.470000e-08      2.6    210.0   \n",
       "4       1.453000  0.022940  1.329120  3.110181e-07      2.6    850.0   \n",
       "...          ...       ...       ...           ...      ...      ...   \n",
       "599995  1.394680  0.006191  1.420600  1.426000e-01      1.7   4600.0   \n",
       "599996  1.364826  0.022669  1.276529  1.827067e-01      1.7   7100.0   \n",
       "599997  1.613500  0.648098  2.010000  6.500000e-01      1.7  50000.0   \n",
       "599998  1.613500  0.648098  2.020000  6.850000e-01      1.7  90000.0   \n",
       "599999  1.613500  0.648098  2.020000  6.850000e-01      1.7  90000.0   \n",
       "\n",
       "                 x  \n",
       "0        17.951958  \n",
       "1        16.534698  \n",
       "2        47.123890  \n",
       "3       149.599650  \n",
       "4        36.959914  \n",
       "...            ...  \n",
       "599995    0.478068  \n",
       "599996    0.398230  \n",
       "599997    0.188496  \n",
       "599998    0.139626  \n",
       "599999    0.418879  \n",
       "\n",
       "[600000 rows x 13 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "columns = ['Extinction', 'Scattering', 'SSA', 'Asym', 'coating', 'core_dia', 'n_core', 'k_core', 'n_shell', 'k_shell', 'density', 'lambda']\n",
    "df1 = pd.read_csv('/work/bb1070/b382177/mie/icon/12-05-2023/mie_x1.csv', index_col=0).sort_index().sort_index().iloc[:-1224, :]\n",
    "df2 = pd.read_csv('/work/bb1070/b382177/mie/icon/12-05-2023/mie_x2.csv', index_col=0).sort_index().sort_index().iloc[:-1224, :]\n",
    "df = pd.concat([df1, df2], axis=0).reset_index()\n",
    "df = df[columns].dropna().astype('float')\n",
    "df['x'] = np.pi * df['core_dia'] / df['lambda']\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "fdde746d-fe63-41d2-a5f5-ac522459179a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df[[\"Extinction\", \"SSA\", \"Asym\"]].to_csv('/work/bb1070/b382177/mie/icon/mie_out.txt', float_format='%.3f', sep=' ')\n",
    "# df[['coating', 'x', 'n_core', 'k_core', 'n_shell', 'k_shell', 'lambda']].to_csv('/work/bb1070/b382177/mie/icon/mie_input.txt', float_format='%.3f', sep=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f7585681-f732-4002-a277-e5ae027197df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Extinction</th>\n",
       "      <td>4.427469</td>\n",
       "      <td>3.949925e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Scattering</th>\n",
       "      <td>4.427461</td>\n",
       "      <td>2.459024e-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SSA</th>\n",
       "      <td>1.000367</td>\n",
       "      <td>1.902386e-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Asym</th>\n",
       "      <td>0.980657</td>\n",
       "      <td>1.920404e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>coating</th>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>core_dia</th>\n",
       "      <td>20000.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_core</th>\n",
       "      <td>2.768000</td>\n",
       "      <td>1.066360e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>k_core</th>\n",
       "      <td>1.002280</td>\n",
       "      <td>5.000000e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_shell</th>\n",
       "      <td>2.094000</td>\n",
       "      <td>1.172000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>k_shell</th>\n",
       "      <td>0.730000</td>\n",
       "      <td>1.062778e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>density</th>\n",
       "      <td>2.600000</td>\n",
       "      <td>1.250000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lambda</th>\n",
       "      <td>100000.000000</td>\n",
       "      <td>1.000000e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>x</th>\n",
       "      <td>628.318531</td>\n",
       "      <td>3.141593e-04</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        0             0\n",
       "Extinction       4.427469  3.949925e-07\n",
       "Scattering       4.427461  2.459024e-15\n",
       "SSA              1.000367  1.902386e-11\n",
       "Asym             0.980657  1.920404e-08\n",
       "coating          0.400000  0.000000e+00\n",
       "core_dia     20000.000000  1.000000e+01\n",
       "n_core           2.768000  1.066360e+00\n",
       "k_core           1.002280  5.000000e-09\n",
       "n_shell          2.094000  1.172000e+00\n",
       "k_shell          0.730000  1.062778e-09\n",
       "density          2.600000  1.250000e+00\n",
       "lambda      100000.000000  1.000000e+02\n",
       "x              628.318531  3.141593e-04"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mx = df.max()\n",
    "mn = df.min()\n",
    "\n",
    "pd.concat([pd.DataFrame(mx), pd.DataFrame(mn)], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "af78f502-bd3b-4e14-b514-db822920d5ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Extinction</th>\n",
       "      <th>Scattering</th>\n",
       "      <th>Asym</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.385656</td>\n",
       "      <td>-0.134130</td>\n",
       "      <td>-0.108732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.825163</td>\n",
       "      <td>-0.511151</td>\n",
       "      <td>-0.509331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.483993</td>\n",
       "      <td>1.338700</td>\n",
       "      <td>0.826924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.051317</td>\n",
       "      <td>-0.452069</td>\n",
       "      <td>-0.513292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.518457</td>\n",
       "      <td>0.429787</td>\n",
       "      <td>1.835915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539995</th>\n",
       "      <td>-0.990207</td>\n",
       "      <td>-0.841703</td>\n",
       "      <td>-0.810697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539996</th>\n",
       "      <td>0.220158</td>\n",
       "      <td>0.192439</td>\n",
       "      <td>0.161772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539997</th>\n",
       "      <td>-0.956842</td>\n",
       "      <td>-5.199338</td>\n",
       "      <td>-1.599475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539998</th>\n",
       "      <td>-0.213032</td>\n",
       "      <td>-0.563891</td>\n",
       "      <td>-0.612526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539999</th>\n",
       "      <td>-0.487695</td>\n",
       "      <td>-0.670379</td>\n",
       "      <td>-0.625399</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>540000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Extinction  Scattering      Asym\n",
       "0        -0.385656   -0.134130 -0.108732\n",
       "1        -0.825163   -0.511151 -0.509331\n",
       "2         1.483993    1.338700  0.826924\n",
       "3        -0.051317   -0.452069 -0.513292\n",
       "4         0.518457    0.429787  1.835915\n",
       "...            ...         ...       ...\n",
       "539995   -0.990207   -0.841703 -0.810697\n",
       "539996    0.220158    0.192439  0.161772\n",
       "539997   -0.956842   -5.199338 -1.599475\n",
       "539998   -0.213032   -0.563891 -0.612526\n",
       "539999   -0.487695   -0.670379 -0.625399\n",
       "\n",
       "[540000 rows x 3 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qt = QuantileTransformer(n_quantiles=1000, output_distribution=\"normal\", random_state=0)\n",
    "df_shuffled = df.sample(frac=1, random_state=40).reset_index(drop=True)\n",
    "df_shuffled = normalize1(df_shuffled)\n",
    "\n",
    "out_col = [\"Extinction\", \"Scattering\", \"Asym\"]\n",
    "in_col = ['coating', 'x', 'n_core', 'k_core', 'n_shell', 'k_shell', 'lambda']\n",
    "\n",
    "y = df_shuffled[out_col]\n",
    "y[out_col] = qt.fit_transform(y)\n",
    "x = df_shuffled[in_col]\n",
    "\n",
    "split = 0.9\n",
    "train_size = int(split * len(x))\n",
    "train_x = x.iloc[:train_size, :].reset_index(drop=True)\n",
    "train_y = y.iloc[:train_size, :].reset_index(drop=True)\n",
    "test_x = x.iloc[train_size:, :].reset_index(drop=True)\n",
    "test_y = y.iloc[train_size:, :].reset_index(drop=True)\n",
    "train_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "09625a63-07e3-4417-902a-dd4238debc11",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:There are non-GPU devices in `tf.distribute.Strategy`, not using nccl allreduce.\n",
      "INFO:tensorflow:Using MirroredStrategy with devices ('/job:localhost/replica:0/task:0/device:CPU:0',)\n",
      "INFO:tensorflow:Reloading Tuner from model/mie/tuner0.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-08-17 19:08:25.112309: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
      "2023-08-17 19:08:25.112345: W tensorflow/stream_executor/cuda/cuda_driver.cc:263] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2023-08-17 19:08:25.112364: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (l40003.lvt.dkrz.de): /proc/driver/nvidia/version does not exist\n",
      "2023-08-17 19:08:25.112624: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  SSE4.1 SSE4.2 AVX AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "def create_model(hp):\n",
    "    model = Sequential()\n",
    "    model.add(tf.keras.layers.InputLayer(input_shape=(7,)))\n",
    "    hp_optimizers = hp.Choice('optimizer', values=[\"adam\", \"sgd\", 'rmsprop'])   \n",
    "    hp_units = hp.Choice('units', [8, 16, 32, 64])\n",
    "    activation = hp.Choice('activation', ['gelu', 'relu', 'sigmoid', 'tanh'])\n",
    "    for i in range(hp.Int('num_layers', 1, 5)): \n",
    "        #hp_units = hp.Choice('units_' + str(i), [8, 16, 32, 64])\n",
    "        #activation = hp.Choice('activation_' + str(i), ['gelu', 'relu', 'sigmoid', 'tanh'])\n",
    "        model.add(Dense(units=hp_units, activation=activation))\n",
    "        #model.add(Dropout(hp.Choice('dropout_'+ str(i), values=[0.0, 0.1, 0.2])))\n",
    "        \n",
    "    model.add(Dense(3, name=\"output_layer\"))\n",
    "    model.compile(loss=\"mean_absolute_error\", optimizer=hp_optimizers, metrics=[\"mean_squared_error\"])\n",
    "    # model.compile(loss=\"mse\", optimizer=hp_optimizers, metrics=['mse'])\n",
    "    return model\n",
    "\n",
    "tuner = BayesianOptimization(create_model, \n",
    "                             seed=1234,\n",
    "                             objective=\"val_mean_squared_error\", \n",
    "                             max_trials=50, \n",
    "                             distribution_strategy=tf.distribute.MirroredStrategy(),\n",
    "                             directory='model',\n",
    "                             overwrite=False,\n",
    "                             project_name=\"mie\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6e203f12-06f0-40a2-8f18-a8e005b60e44",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search space summary\n",
      "Default search space size: 4\n",
      "optimizer (Choice)\n",
      "{'default': 'adam', 'conditions': [], 'values': ['adam', 'sgd', 'rms'], 'ordered': False}\n",
      "units (Choice)\n",
      "{'default': 8, 'conditions': [], 'values': [8, 16, 32, 64], 'ordered': True}\n",
      "activation (Choice)\n",
      "{'default': 'gelu', 'conditions': [], 'values': ['gelu', 'relu', 'sigmoid', 'tanh'], 'ordered': False}\n",
      "num_layers (Int)\n",
      "{'default': None, 'conditions': [], 'min_value': 1, 'max_value': 5, 'step': 1, 'sampling': 'linear'}\n"
     ]
    }
   ],
   "source": [
    "tuner.search_space_summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c563d5bc-b58a-43ec-bac8-ba34ca0d6bb8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial 50 Complete [00h 07m 26s]\n",
      "val_mean_squared_error: 0.04747132584452629\n",
      "\n",
      "Best val_mean_squared_error So Far: 0.032091204077005386\n",
      "Total elapsed time: 02h 43m 14s\n",
      "INFO:tensorflow:Oracle triggered exit\n"
     ]
    }
   ],
   "source": [
    "tuner.search(train_x, train_y, epochs=200, batch_size=250, validation_data=(test_x, test_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f56068b5-ed7e-49af-91b2-753ad6505802",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results summary\n",
      "Results in model/mie\n",
      "Showing 10 best trials\n",
      "Objective(name=\"val_mean_squared_error\", direction=\"min\")\n",
      "\n",
      "Trial 29 summary\n",
      "Hyperparameters:\n",
      "optimizer: adam\n",
      "units: 64\n",
      "activation: gelu\n",
      "num_layers: 5\n",
      "Score: 0.032091204077005386\n",
      "\n",
      "Trial 27 summary\n",
      "Hyperparameters:\n",
      "optimizer: adam\n",
      "units: 32\n",
      "activation: tanh\n",
      "num_layers: 3\n",
      "Score: 0.03307025879621506\n",
      "\n",
      "Trial 15 summary\n",
      "Hyperparameters:\n",
      "optimizer: adam\n",
      "units: 64\n",
      "activation: tanh\n",
      "num_layers: 3\n",
      "Score: 0.033729273825883865\n",
      "\n",
      "Trial 37 summary\n",
      "Hyperparameters:\n",
      "optimizer: adam\n",
      "units: 32\n",
      "activation: gelu\n",
      "num_layers: 4\n",
      "Score: 0.03562641143798828\n",
      "\n",
      "Trial 24 summary\n",
      "Hyperparameters:\n",
      "optimizer: adam\n",
      "units: 64\n",
      "activation: gelu\n",
      "num_layers: 4\n",
      "Score: 0.03632396087050438\n",
      "\n",
      "Trial 23 summary\n",
      "Hyperparameters:\n",
      "optimizer: adam\n",
      "units: 64\n",
      "activation: relu\n",
      "num_layers: 5\n",
      "Score: 0.03884287551045418\n",
      "\n",
      "Trial 39 summary\n",
      "Hyperparameters:\n",
      "optimizer: adam\n",
      "units: 16\n",
      "activation: tanh\n",
      "num_layers: 3\n",
      "Score: 0.040453314781188965\n",
      "\n",
      "Trial 45 summary\n",
      "Hyperparameters:\n",
      "optimizer: adam\n",
      "units: 16\n",
      "activation: tanh\n",
      "num_layers: 3\n",
      "Score: 0.04061581939458847\n",
      "\n",
      "Trial 47 summary\n",
      "Hyperparameters:\n",
      "optimizer: adam\n",
      "units: 16\n",
      "activation: tanh\n",
      "num_layers: 3\n",
      "Score: 0.041150156408548355\n",
      "\n",
      "Trial 46 summary\n",
      "Hyperparameters:\n",
      "optimizer: adam\n",
      "units: 16\n",
      "activation: tanh\n",
      "num_layers: 3\n",
      "Score: 0.04248723387718201\n"
     ]
    }
   ],
   "source": [
    "tuner.results_summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c76f356a-e770-46db-b202-9b342aafee9f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'optimizer': 'adam', 'units': 64, 'activation': 'gelu', 'num_layers': 5}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_hp = tuner.get_best_hyperparameters()[0]\n",
    "best_hp.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18813221-fbb4-46f9-9258-01047e3ea536",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ML (based on the latest module pytorch)",
   "language": "python",
   "name": "ml-aim"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}