mlp_hyper2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f7fbfe8e-cce0-4ab7-948f-1aa0f94fb992",
   "metadata": {},
   "outputs": [],
   "source": [
    "# %load_ext autotime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ac794bb8-5583-4fc8-bcad-44094343c561",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "540c7fdd-56fd-41e0-a60d-b2aa171d0a2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "# from mei import Size_Distribution_Optics\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7ba7f0b9-5d79-4805-a484-c08748a61bb9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import dask.dataframe as dd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "482b3e17-0e20-4b96-9833-2a7132eb34f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn\n",
    "from sklearn.preprocessing import MinMaxScaler, PowerTransformer, QuantileTransformer\n",
    "from sklearn.metrics import r2_score \n",
    "import joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "410594a8-c22b-49a0-ae50-d4756b7111d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-08-19 11:05:17.924202: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  SSE4.1 SSE4.2 AVX AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Flatten\n",
    "from tensorflow.keras.layers import Dense, Dropout\n",
    "from tensorflow.keras.layers import Activation\n",
    "\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, TerminateOnNaN, ReduceLROnPlateau, EarlyStopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9d7721d5-aa98-421a-b1ed-1ddb9efef636",
   "metadata": {},
   "outputs": [],
   "source": [
    "import kerastuner as kt\n",
    "from kerastuner.tuners import RandomSearch, BayesianOptimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a9bf066a-263a-4970-8967-4a4b8e272184",
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(df):\n",
    "    return (df - df.mean()) / df.std()\n",
    "\n",
    "def normalize1(df):    \n",
    "    mx = df.max()\n",
    "    mn = df.min()\n",
    "    \n",
    "    mx['SSA'] = 1.0\n",
    "    mx['Asym'] = 1.0\n",
    "    \n",
    "    mn['SSA'] = 0.0\n",
    "    mn['Asym'] = 0.0\n",
    "    \n",
    "    new = (df - mn) / (mx - mn)    \n",
    "    return new\n",
    "\n",
    "\n",
    "def load_opt(filename):\n",
    "    with open(filename, \"r\") as fp:\n",
    "        data = json.load(fp)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "69998211-344c-4b57-bbe5-ca7e96cdb005",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Extinction</th>\n",
       "      <th>Scattering</th>\n",
       "      <th>SSA</th>\n",
       "      <th>Asym</th>\n",
       "      <th>coating</th>\n",
       "      <th>core_dia</th>\n",
       "      <th>n_core</th>\n",
       "      <th>k_core</th>\n",
       "      <th>n_shell</th>\n",
       "      <th>k_shell</th>\n",
       "      <th>density</th>\n",
       "      <th>lambda</th>\n",
       "      <th>x</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.371424</td>\n",
       "      <td>1.497649</td>\n",
       "      <td>0.631540</td>\n",
       "      <td>0.898342</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>1.451990</td>\n",
       "      <td>0.022362</td>\n",
       "      <td>1.330928</td>\n",
       "      <td>4.128916e-08</td>\n",
       "      <td>2.6</td>\n",
       "      <td>700.0</td>\n",
       "      <td>17.951958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.203139</td>\n",
       "      <td>1.330449</td>\n",
       "      <td>0.603888</td>\n",
       "      <td>0.882517</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>1.453480</td>\n",
       "      <td>0.023396</td>\n",
       "      <td>1.327650</td>\n",
       "      <td>2.061950e-06</td>\n",
       "      <td>2.6</td>\n",
       "      <td>950.0</td>\n",
       "      <td>16.534698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.173950</td>\n",
       "      <td>1.202943</td>\n",
       "      <td>0.553344</td>\n",
       "      <td>0.947042</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9000.0</td>\n",
       "      <td>1.451020</td>\n",
       "      <td>0.018632</td>\n",
       "      <td>1.332398</td>\n",
       "      <td>9.574458e-09</td>\n",
       "      <td>2.6</td>\n",
       "      <td>600.0</td>\n",
       "      <td>47.123890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.068879</td>\n",
       "      <td>1.123843</td>\n",
       "      <td>0.543213</td>\n",
       "      <td>0.952623</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>1.483700</td>\n",
       "      <td>0.099080</td>\n",
       "      <td>1.389200</td>\n",
       "      <td>9.470000e-08</td>\n",
       "      <td>2.6</td>\n",
       "      <td>210.0</td>\n",
       "      <td>149.599650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.150144</td>\n",
       "      <td>1.163233</td>\n",
       "      <td>0.541003</td>\n",
       "      <td>0.941646</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>1.453000</td>\n",
       "      <td>0.022940</td>\n",
       "      <td>1.329120</td>\n",
       "      <td>3.110181e-07</td>\n",
       "      <td>2.6</td>\n",
       "      <td>850.0</td>\n",
       "      <td>36.959914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599995</th>\n",
       "      <td>0.184309</td>\n",
       "      <td>0.033168</td>\n",
       "      <td>0.179958</td>\n",
       "      <td>0.086705</td>\n",
       "      <td>0.4</td>\n",
       "      <td>700.0</td>\n",
       "      <td>1.394680</td>\n",
       "      <td>0.006191</td>\n",
       "      <td>1.420600</td>\n",
       "      <td>1.426000e-01</td>\n",
       "      <td>1.7</td>\n",
       "      <td>4600.0</td>\n",
       "      <td>0.478068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599996</th>\n",
       "      <td>0.181849</td>\n",
       "      <td>0.010640</td>\n",
       "      <td>0.058508</td>\n",
       "      <td>0.054755</td>\n",
       "      <td>0.4</td>\n",
       "      <td>900.0</td>\n",
       "      <td>1.364826</td>\n",
       "      <td>0.022669</td>\n",
       "      <td>1.276529</td>\n",
       "      <td>1.827067e-01</td>\n",
       "      <td>1.7</td>\n",
       "      <td>7100.0</td>\n",
       "      <td>0.398230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599997</th>\n",
       "      <td>0.265074</td>\n",
       "      <td>0.004225</td>\n",
       "      <td>0.015940</td>\n",
       "      <td>0.016758</td>\n",
       "      <td>0.4</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>1.613500</td>\n",
       "      <td>0.648098</td>\n",
       "      <td>2.010000</td>\n",
       "      <td>6.500000e-01</td>\n",
       "      <td>1.7</td>\n",
       "      <td>50000.0</td>\n",
       "      <td>0.188496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599998</th>\n",
       "      <td>0.193847</td>\n",
       "      <td>0.001301</td>\n",
       "      <td>0.006709</td>\n",
       "      <td>0.009228</td>\n",
       "      <td>0.4</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>1.613500</td>\n",
       "      <td>0.648098</td>\n",
       "      <td>2.020000</td>\n",
       "      <td>6.850000e-01</td>\n",
       "      <td>1.7</td>\n",
       "      <td>90000.0</td>\n",
       "      <td>0.139626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599999</th>\n",
       "      <td>0.819451</td>\n",
       "      <td>0.106342</td>\n",
       "      <td>0.129773</td>\n",
       "      <td>0.083984</td>\n",
       "      <td>0.4</td>\n",
       "      <td>12000.0</td>\n",
       "      <td>1.613500</td>\n",
       "      <td>0.648098</td>\n",
       "      <td>2.020000</td>\n",
       "      <td>6.850000e-01</td>\n",
       "      <td>1.7</td>\n",
       "      <td>90000.0</td>\n",
       "      <td>0.418879</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>600000 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Extinction  Scattering       SSA      Asym  coating  core_dia  \\\n",
       "0         2.371424    1.497649  0.631540  0.898342      0.0    4000.0   \n",
       "1         2.203139    1.330449  0.603888  0.882517      0.0    5000.0   \n",
       "2         2.173950    1.202943  0.553344  0.947042      0.0    9000.0   \n",
       "3         2.068879    1.123843  0.543213  0.952623      0.0   10000.0   \n",
       "4         2.150144    1.163233  0.541003  0.941646      0.0   10000.0   \n",
       "...            ...         ...       ...       ...      ...       ...   \n",
       "599995    0.184309    0.033168  0.179958  0.086705      0.4     700.0   \n",
       "599996    0.181849    0.010640  0.058508  0.054755      0.4     900.0   \n",
       "599997    0.265074    0.004225  0.015940  0.016758      0.4    3000.0   \n",
       "599998    0.193847    0.001301  0.006709  0.009228      0.4    4000.0   \n",
       "599999    0.819451    0.106342  0.129773  0.083984      0.4   12000.0   \n",
       "\n",
       "          n_core    k_core   n_shell       k_shell  density   lambda  \\\n",
       "0       1.451990  0.022362  1.330928  4.128916e-08      2.6    700.0   \n",
       "1       1.453480  0.023396  1.327650  2.061950e-06      2.6    950.0   \n",
       "2       1.451020  0.018632  1.332398  9.574458e-09      2.6    600.0   \n",
       "3       1.483700  0.099080  1.389200  9.470000e-08      2.6    210.0   \n",
       "4       1.453000  0.022940  1.329120  3.110181e-07      2.6    850.0   \n",
       "...          ...       ...       ...           ...      ...      ...   \n",
       "599995  1.394680  0.006191  1.420600  1.426000e-01      1.7   4600.0   \n",
       "599996  1.364826  0.022669  1.276529  1.827067e-01      1.7   7100.0   \n",
       "599997  1.613500  0.648098  2.010000  6.500000e-01      1.7  50000.0   \n",
       "599998  1.613500  0.648098  2.020000  6.850000e-01      1.7  90000.0   \n",
       "599999  1.613500  0.648098  2.020000  6.850000e-01      1.7  90000.0   \n",
       "\n",
       "                 x  \n",
       "0        17.951958  \n",
       "1        16.534698  \n",
       "2        47.123890  \n",
       "3       149.599650  \n",
       "4        36.959914  \n",
       "...            ...  \n",
       "599995    0.478068  \n",
       "599996    0.398230  \n",
       "599997    0.188496  \n",
       "599998    0.139626  \n",
       "599999    0.418879  \n",
       "\n",
       "[600000 rows x 13 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "columns = ['Extinction', 'Scattering', 'SSA', 'Asym', 'coating', 'core_dia', 'n_core', 'k_core', 'n_shell', 'k_shell', 'density', 'lambda']\n",
    "df1 = pd.read_csv('/work/bb1070/b382177/mie/icon/12-05-2023/mie_x1.csv', index_col=0).sort_index().sort_index().iloc[:-1224, :]\n",
    "df2 = pd.read_csv('/work/bb1070/b382177/mie/icon/12-05-2023/mie_x2.csv', index_col=0).sort_index().sort_index().iloc[:-1224, :]\n",
    "df = pd.concat([df1, df2], axis=0).reset_index()\n",
    "df = df[columns].dropna().astype('float')\n",
    "df['x'] = np.pi * df['core_dia'] / df['lambda']\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "fdde746d-fe63-41d2-a5f5-ac522459179a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df[[\"Extinction\", \"SSA\", \"Asym\"]].to_csv('/work/bb1070/b382177/mie/icon/mie_out.txt', float_format='%.3f', sep=' ')\n",
    "# df[['coating', 'x', 'n_core', 'k_core', 'n_shell', 'k_shell', 'lambda']].to_csv('/work/bb1070/b382177/mie/icon/mie_input.txt', float_format='%.3f', sep=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f7585681-f732-4002-a277-e5ae027197df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Extinction</th>\n",
       "      <td>4.427469</td>\n",
       "      <td>3.949925e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Scattering</th>\n",
       "      <td>4.427461</td>\n",
       "      <td>2.459024e-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SSA</th>\n",
       "      <td>1.000367</td>\n",
       "      <td>1.902386e-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Asym</th>\n",
       "      <td>0.980657</td>\n",
       "      <td>1.920404e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>coating</th>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>core_dia</th>\n",
       "      <td>20000.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_core</th>\n",
       "      <td>2.768000</td>\n",
       "      <td>1.066360e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>k_core</th>\n",
       "      <td>1.002280</td>\n",
       "      <td>5.000000e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_shell</th>\n",
       "      <td>2.094000</td>\n",
       "      <td>1.172000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>k_shell</th>\n",
       "      <td>0.730000</td>\n",
       "      <td>1.062778e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>density</th>\n",
       "      <td>2.600000</td>\n",
       "      <td>1.250000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lambda</th>\n",
       "      <td>100000.000000</td>\n",
       "      <td>1.000000e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>x</th>\n",
       "      <td>628.318531</td>\n",
       "      <td>3.141593e-04</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        0             0\n",
       "Extinction       4.427469  3.949925e-07\n",
       "Scattering       4.427461  2.459024e-15\n",
       "SSA              1.000367  1.902386e-11\n",
       "Asym             0.980657  1.920404e-08\n",
       "coating          0.400000  0.000000e+00\n",
       "core_dia     20000.000000  1.000000e+01\n",
       "n_core           2.768000  1.066360e+00\n",
       "k_core           1.002280  5.000000e-09\n",
       "n_shell          2.094000  1.172000e+00\n",
       "k_shell          0.730000  1.062778e-09\n",
       "density          2.600000  1.250000e+00\n",
       "lambda      100000.000000  1.000000e+02\n",
       "x              628.318531  3.141593e-04"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mx = df.max()\n",
    "mn = df.min()\n",
    "\n",
    "pd.concat([pd.DataFrame(mx), pd.DataFrame(mn)], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "af78f502-bd3b-4e14-b514-db822920d5ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Extinction</th>\n",
       "      <th>Scattering</th>\n",
       "      <th>Asym</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.385656</td>\n",
       "      <td>-0.134130</td>\n",
       "      <td>-0.108732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.825163</td>\n",
       "      <td>-0.511151</td>\n",
       "      <td>-0.509331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.483993</td>\n",
       "      <td>1.338700</td>\n",
       "      <td>0.826924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.051317</td>\n",
       "      <td>-0.452069</td>\n",
       "      <td>-0.513292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.518457</td>\n",
       "      <td>0.429787</td>\n",
       "      <td>1.835915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539995</th>\n",
       "      <td>-0.990207</td>\n",
       "      <td>-0.841703</td>\n",
       "      <td>-0.810697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539996</th>\n",
       "      <td>0.220158</td>\n",
       "      <td>0.192439</td>\n",
       "      <td>0.161772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539997</th>\n",
       "      <td>-0.956842</td>\n",
       "      <td>-5.199338</td>\n",
       "      <td>-1.599475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539998</th>\n",
       "      <td>-0.213032</td>\n",
       "      <td>-0.563891</td>\n",
       "      <td>-0.612526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539999</th>\n",
       "      <td>-0.487695</td>\n",
       "      <td>-0.670379</td>\n",
       "      <td>-0.625399</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>540000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Extinction  Scattering      Asym\n",
       "0        -0.385656   -0.134130 -0.108732\n",
       "1        -0.825163   -0.511151 -0.509331\n",
       "2         1.483993    1.338700  0.826924\n",
       "3        -0.051317   -0.452069 -0.513292\n",
       "4         0.518457    0.429787  1.835915\n",
       "...            ...         ...       ...\n",
       "539995   -0.990207   -0.841703 -0.810697\n",
       "539996    0.220158    0.192439  0.161772\n",
       "539997   -0.956842   -5.199338 -1.599475\n",
       "539998   -0.213032   -0.563891 -0.612526\n",
       "539999   -0.487695   -0.670379 -0.625399\n",
       "\n",
       "[540000 rows x 3 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qt = QuantileTransformer(n_quantiles=1000, output_distribution=\"normal\", random_state=0)\n",
    "df_shuffled = df.sample(frac=1, random_state=40).reset_index(drop=True)\n",
    "df_shuffled = normalize1(df_shuffled)\n",
    "\n",
    "out_col = [\"Extinction\", \"Scattering\", \"Asym\"]\n",
    "in_col = ['coating', 'x', 'n_core', 'k_core', 'n_shell', 'k_shell', 'lambda']\n",
    "\n",
    "y = df_shuffled[out_col]\n",
    "y[out_col] = qt.fit_transform(y)\n",
    "x = df_shuffled[in_col]\n",
    "\n",
    "split = 0.9\n",
    "train_size = int(split * len(x))\n",
    "train_x = x.iloc[:train_size, :].reset_index(drop=True)\n",
    "train_y = y.iloc[:train_size, :].reset_index(drop=True)\n",
    "test_x = x.iloc[train_size:, :].reset_index(drop=True)\n",
    "test_y = y.iloc[train_size:, :].reset_index(drop=True)\n",
    "train_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e66c524d-30ac-4d66-bd4d-512a59a76ad8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model(hp):\n",
    "    hp_units = hp.Choice('units', [32, 64])\n",
    "    activation = 'gelu'\n",
    "    learning_rate = hp.Choice(\"lr\", [1e-1, 1e-2, 1e-3, 1e-4])\n",
    "    hp_optimizers = keras.optimizers.Adam(learning_rate=learning_rate)    \n",
    "\n",
    "    model = Sequential()\n",
    "    model.add(tf.keras.layers.InputLayer(input_shape=(7,)))  \n",
    "    [model.add(Dense(units=hp_units, activation=activation)) for i in range(hp.Int('num_layers', 4, 5))]       \n",
    "    model.add(Dense(3, name=\"output_layer\"))\n",
    "    model.compile(loss=\"mean_absolute_error\", optimizer=hp_optimizers, metrics=[\"mean_squared_error\"])\n",
    "    # model.compile(loss=\"mse\", optimizer=hp_optimizers, metrics=['mse'])\n",
    "    return model\n",
    "\n",
    "class MyTuner(RandomSearch):\n",
    "    def run_trial(self, trial, *args, **kwargs):\n",
    "        # You can add additional HyperParameters for preprocessing and custom training loops\n",
    "        # via overriding `run_trial`\n",
    "        kwargs['batch_size'] = trial.hyperparameters.Choice(\"batch_size\", [128, 256, 512, 1024])\n",
    "        #kwargs['epochs'] = trial.hyperparameters.Int('epochs', 100, 500, step=50)\n",
    "        return super(MyTuner, self).run_trial(trial, *args, **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6e0c3c96-accb-4118-bf6f-949ac67e3503",
   "metadata": {},
   "outputs": [],
   "source": [
    "# class MyHyperModel(kt.HyperModel):\n",
    "#     def build(self, hp):\n",
    "#         hp_units = 64\n",
    "#         activation = 'gelu'\n",
    "#         learning_rate = hp.Float(\"lr\", min_value=1e-4, max_value=1e-2, sampling=\"log\")\n",
    "#         hp_optimizers = keras.optimizers.Adam(learning_rate=learning_rate)    \n",
    "\n",
    "#         model = Sequential()\n",
    "#         model.add(tf.keras.layers.InputLayer(input_shape=(7,)))  \n",
    "#         [model.add(Dense(units=hp_units, activation=activation)) for i in range(hp.Int('num_layers', 4, 5))]       \n",
    "#         model.add(Dense(3, name=\"output_layer\"))\n",
    "#         model.compile(loss=\"mean_absolute_error\", optimizer=hp_optimizers, metrics=[\"mean_squared_error\"])\n",
    "#         # model.compile(loss=\"mse\", optimizer=hp_optimizers, metrics=['mse'])\n",
    "#         return model\n",
    "\n",
    "#     def fit(self, hp, model, *args, **kwargs):\n",
    "#         return model.fit(\n",
    "#             *args,\n",
    "#             batch_size=hp.Choice(\"batch_size\", [64, 128, 256, 512]),\n",
    "#             **kwargs,\n",
    "#         )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "09625a63-07e3-4417-902a-dd4238debc11",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:There are non-GPU devices in `tf.distribute.Strategy`, not using nccl allreduce.\n",
      "INFO:tensorflow:Using MirroredStrategy with devices ('/job:localhost/replica:0/task:0/device:CPU:0',)\n",
      "INFO:tensorflow:Reloading Tuner from model/mie/tuner0.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-08-19 11:05:27.827957: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
      "2023-08-19 11:05:27.827991: W tensorflow/stream_executor/cuda/cuda_driver.cc:263] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2023-08-19 11:05:27.828011: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (l40010.lvt.dkrz.de): /proc/driver/nvidia/version does not exist\n",
      "2023-08-19 11:05:27.828781: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  SSE4.1 SSE4.2 AVX AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "tuner = MyTuner(create_model, \n",
    "                seed=1234,\n",
    "                 objective=\"val_mean_squared_error\", \n",
    "                 max_trials=50, \n",
    "                 distribution_strategy=tf.distribute.MirroredStrategy(),\n",
    "                 directory='model',\n",
    "                 overwrite=False,\n",
    "                 project_name=\"mie\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6e203f12-06f0-40a2-8f18-a8e005b60e44",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search space summary\n",
      "Default search space size: 4\n",
      "units (Choice)\n",
      "{'default': 32, 'conditions': [], 'values': [32, 64], 'ordered': True}\n",
      "lr (Choice)\n",
      "{'default': 0.1, 'conditions': [], 'values': [0.1, 0.01, 0.001, 0.0001], 'ordered': True}\n",
      "num_layers (Int)\n",
      "{'default': None, 'conditions': [], 'min_value': 4, 'max_value': 5, 'step': 1, 'sampling': 'linear'}\n",
      "batch_size (Choice)\n",
      "{'default': 128, 'conditions': [], 'values': [128, 256, 512, 1024], 'ordered': True}\n"
     ]
    }
   ],
   "source": [
    "tuner.search_space_summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c563d5bc-b58a-43ec-bac8-ba34ca0d6bb8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial 50 Complete [00h 14m 30s]\n",
      "val_mean_squared_error: 0.017237532883882523\n",
      "\n",
      "Best val_mean_squared_error So Far: 0.01410246454179287\n",
      "Total elapsed time: 03h 01m 13s\n",
      "INFO:tensorflow:Oracle triggered exit\n"
     ]
    }
   ],
   "source": [
    "es = EarlyStopping(monitor=\"val_loss\", mode=\"min\", verbose=1, patience=50)\n",
    "rlr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5, min_lr=0.00001)\n",
    "callbacks = [es, rlr]\n",
    "\n",
    "tuner.search(train_x, train_y, validation_data=(test_x, test_y), epochs=200,  callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f56068b5-ed7e-49af-91b2-753ad6505802",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results summary\n",
      "Results in model/mie\n",
      "Showing 10 best trials\n",
      "Objective(name=\"val_mean_squared_error\", direction=\"min\")\n",
      "\n",
      "Trial 19 summary\n",
      "Hyperparameters:\n",
      "units: 64\n",
      "lr: 0.01\n",
      "num_layers: 4\n",
      "batch_size: 128\n",
      "Score: 0.01410246454179287\n",
      "\n",
      "Trial 49 summary\n",
      "Hyperparameters:\n",
      "units: 64\n",
      "lr: 0.01\n",
      "num_layers: 5\n",
      "batch_size: 512\n",
      "Score: 0.017237532883882523\n",
      "\n",
      "Trial 21 summary\n",
      "Hyperparameters:\n",
      "units: 64\n",
      "lr: 0.01\n",
      "num_layers: 5\n",
      "batch_size: 256\n",
      "Score: 0.017358968034386635\n",
      "\n",
      "Trial 45 summary\n",
      "Hyperparameters:\n",
      "units: 32\n",
      "lr: 0.01\n",
      "num_layers: 4\n",
      "batch_size: 128\n",
      "Score: 0.017747187986969948\n",
      "\n",
      "Trial 46 summary\n",
      "Hyperparameters:\n",
      "units: 64\n",
      "lr: 0.001\n",
      "num_layers: 5\n",
      "batch_size: 128\n",
      "Score: 0.017767425626516342\n",
      "\n",
      "Trial 41 summary\n",
      "Hyperparameters:\n",
      "units: 64\n",
      "lr: 0.01\n",
      "num_layers: 4\n",
      "batch_size: 256\n",
      "Score: 0.018030311912298203\n",
      "\n",
      "Trial 24 summary\n",
      "Hyperparameters:\n",
      "units: 32\n",
      "lr: 0.01\n",
      "num_layers: 5\n",
      "batch_size: 1024\n",
      "Score: 0.01996482163667679\n",
      "\n",
      "Trial 25 summary\n",
      "Hyperparameters:\n",
      "units: 32\n",
      "lr: 0.01\n",
      "num_layers: 4\n",
      "batch_size: 256\n",
      "Score: 0.022220296785235405\n",
      "\n",
      "Trial 22 summary\n",
      "Hyperparameters:\n",
      "units: 32\n",
      "lr: 0.01\n",
      "num_layers: 5\n",
      "batch_size: 128\n",
      "Score: 0.02227042056620121\n",
      "\n",
      "Trial 05 summary\n",
      "Hyperparameters:\n",
      "units: 64\n",
      "lr: 0.01\n",
      "num_layers: 4\n",
      "batch_size: 512\n",
      "Score: 0.022697189822793007\n"
     ]
    }
   ],
   "source": [
    "tuner.results_summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c76f356a-e770-46db-b202-9b342aafee9f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'units': 64, 'lr': 0.01, 'num_layers': 4, 'batch_size': 128}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_hp = tuner.get_best_hyperparameters()[0]\n",
    "best_hp.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18813221-fbb4-46f9-9258-01047e3ea536",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ML (based on the latest module pytorch)",
   "language": "python",
   "name": "ml-aim"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}