diff --git a/BETA_E_Model_T&T.ipynb b/BETA_E_Model_T&T.ipynb index f66b217..f4a2322 100644 --- a/BETA_E_Model_T&T.ipynb +++ b/BETA_E_Model_T&T.ipynb @@ -3561,7 +3561,4021 @@ "Epoch 7/12\n", "256/256 [==============================] - 51s 184ms/step - loss: 0.5474 - accuracy: 0.8428 - val_loss: 0.3377 - val_accuracy: 0.9199 - lr: 0.0088 - momentum: 0.8915\n", "Epoch 8/12\n", - " 71/256 [=======>......................] - ETA: 26s - loss: 0.5399 - accuracy: 0.8363" + "256/256 [==============================] - 47s 182ms/step - loss: 0.5264 - accuracy: 0.8350 - val_loss: 1.1843 - val_accuracy: 0.6667 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 9/12\n", + "256/256 [==============================] - 51s 198ms/step - loss: 0.5311 - accuracy: 0.8145 - val_loss: 0.2925 - val_accuracy: 0.9247 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 10/12\n", + "256/256 [==============================] - 49s 190ms/step - loss: 0.4503 - accuracy: 0.8555 - val_loss: 0.5356 - val_accuracy: 0.8462 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 11/12\n", + "256/256 [==============================] - 50s 194ms/step - loss: 0.4070 - accuracy: 0.8491 - val_loss: 0.3295 - val_accuracy: 0.9054 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 12/12\n", + "207/256 [=======================>......] - ETA: 7s - loss: 0.3178 - accuracy: 0.8979\u001b[0;31m\n", + "Pausing training due to high GPU temperature! (for [60]sec)\u001b[0m\n", + "\u001b[0;33mResuming training...\u001b[0m\n", + "256/256 [==============================] - 109s 428ms/step - loss: 0.3238 - accuracy: 0.8989 - val_loss: 0.2795 - val_accuracy: 0.9103 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-009-0.9247.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9247\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2925\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model accuracy from \u001b[0m\u001b[0;32m0.9230769276618958 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.9246794581413269\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model loss from \u001b[0m\u001b[0;32m0.4047916531562805 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.2925480306148529\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m400.69 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m358.11 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m42.58 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [2] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m3\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 12)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 13/18\n", + "256/256 [==============================] - 55s 202ms/step - loss: 0.4700 - accuracy: 0.8335 - val_loss: 0.3299 - val_accuracy: 0.9103 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 14/18\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.4631 - accuracy: 0.8262 - val_loss: 0.3475 - val_accuracy: 0.9087 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 15/18\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.4167 - accuracy: 0.8574 - val_loss: 0.3250 - val_accuracy: 0.9151 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 16/18\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.4224 - accuracy: 0.8555 - val_loss: 0.3555 - val_accuracy: 0.8926 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 17/18\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.3426 - accuracy: 0.8823 - val_loss: 0.2659 - val_accuracy: 0.9247 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 18/18\n", + "256/256 [==============================] - 45s 174ms/step - loss: 0.2764 - accuracy: 0.9033 - val_loss: 0.2557 - val_accuracy: 0.9247 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-017-0.9247.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9247\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2659\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9246794581413269. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model loss from \u001b[0m\u001b[0;32m0.2925480306148529 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.26587748527526855\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m313.02 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m280.98 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m32.03 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [3] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m4\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 18)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 19/24\n", + "256/256 [==============================] - 49s 179ms/step - loss: 0.3731 - accuracy: 0.8628 - val_loss: 0.2503 - val_accuracy: 0.9263 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 20/24\n", + "256/256 [==============================] - 44s 172ms/step - loss: 0.4015 - accuracy: 0.8457 - val_loss: 0.2964 - val_accuracy: 0.9215 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 21/24\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.4235 - accuracy: 0.8433 - val_loss: 0.2734 - val_accuracy: 0.9295 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 22/24\n", + "256/256 [==============================] - 44s 172ms/step - loss: 0.3687 - accuracy: 0.8643 - val_loss: 0.3075 - val_accuracy: 0.9279 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 23/24\n", + "256/256 [==============================] - 44s 171ms/step - loss: 0.3025 - accuracy: 0.8975 - val_loss: 0.3312 - val_accuracy: 0.8494 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 24/24\n", + "256/256 [==============================] - 44s 172ms/step - loss: 0.2619 - accuracy: 0.9121 - val_loss: 0.2756 - val_accuracy: 0.8718 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-021-0.9295.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9295\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2733\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model accuracy from \u001b[0m\u001b[0;32m0.9246794581413269 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.9294871687889099\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.26587748527526855. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m304.80 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m271.53 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m33.27 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [4] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m5\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 24)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 25/30\n", + "256/256 [==============================] - 49s 180ms/step - loss: 0.4056 - accuracy: 0.8521 - val_loss: 0.2638 - val_accuracy: 0.9231 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 26/30\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.4220 - accuracy: 0.8540 - val_loss: 0.4272 - val_accuracy: 0.8510 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 27/30\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.4353 - accuracy: 0.8315 - val_loss: 0.3853 - val_accuracy: 0.9231 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 28/30\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.4237 - accuracy: 0.8442 - val_loss: 0.2622 - val_accuracy: 0.9311 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 29/30\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.3540 - accuracy: 0.8809 - val_loss: 0.2614 - val_accuracy: 0.9359 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 30/30\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2650 - accuracy: 0.9170 - val_loss: 0.2279 - val_accuracy: 0.9295 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-029-0.9359.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9359\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2614\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model accuracy from \u001b[0m\u001b[0;32m0.9294871687889099 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.9358974099159241\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model loss from \u001b[0m\u001b[0;32m0.26587748527526855 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.26140671968460083\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m312.07 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m277.58 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m34.48 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [5] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m6\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 30)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 31/36\n", + "256/256 [==============================] - 55s 200ms/step - loss: 0.3614 - accuracy: 0.8853 - val_loss: 0.3434 - val_accuracy: 0.9247 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 32/36\n", + "256/256 [==============================] - 50s 194ms/step - loss: 0.3995 - accuracy: 0.8574 - val_loss: 0.4120 - val_accuracy: 0.8846 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 33/36\n", + "256/256 [==============================] - 50s 194ms/step - loss: 0.3560 - accuracy: 0.8770 - val_loss: 0.3237 - val_accuracy: 0.8910 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 34/36\n", + "256/256 [==============================] - 50s 193ms/step - loss: 0.3332 - accuracy: 0.8862 - val_loss: 0.3489 - val_accuracy: 0.8702 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 35/36\n", + "256/256 [==============================] - 49s 192ms/step - loss: 0.2828 - accuracy: 0.9077 - val_loss: 0.2354 - val_accuracy: 0.9087 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 36/36\n", + "256/256 [==============================] - 50s 196ms/step - loss: 0.2543 - accuracy: 0.9087 - val_loss: 0.2251 - val_accuracy: 0.9327 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-036-0.9327.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9327\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2251\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9358974099159241. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model loss from \u001b[0m\u001b[0;32m0.26140671968460083 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.22506119310855865\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m338.91 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m304.51 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m34.40 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [6] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m7\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 36)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 37/42\n", + "256/256 [==============================] - 56s 202ms/step - loss: 0.3474 - accuracy: 0.8711 - val_loss: 0.3355 - val_accuracy: 0.9343 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 38/42\n", + "256/256 [==============================] - 50s 196ms/step - loss: 0.3520 - accuracy: 0.8970 - val_loss: 0.3168 - val_accuracy: 0.8814 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 39/42\n", + "256/256 [==============================] - 49s 189ms/step - loss: 0.3267 - accuracy: 0.8945 - val_loss: 0.3317 - val_accuracy: 0.9263 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 40/42\n", + "256/256 [==============================] - 49s 191ms/step - loss: 0.3790 - accuracy: 0.8872 - val_loss: 0.3114 - val_accuracy: 0.9103 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 41/42\n", + "256/256 [==============================] - 50s 195ms/step - loss: 0.2749 - accuracy: 0.9233 - val_loss: 0.2513 - val_accuracy: 0.9103 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 42/42\n", + "256/256 [==============================] - 49s 192ms/step - loss: 0.2435 - accuracy: 0.9312 - val_loss: 0.2553 - val_accuracy: 0.9103 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-037-0.9343.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9343\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3355\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9358974099159241. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.22506119310855865. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m343.71 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m304.46 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m39.26 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [7] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m8\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 42)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 43/48\n", + "256/256 [==============================] - 57s 204ms/step - loss: 0.3539 - accuracy: 0.8828 - val_loss: 0.2467 - val_accuracy: 0.9375 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 44/48\n", + "256/256 [==============================] - 50s 195ms/step - loss: 0.4044 - accuracy: 0.8677 - val_loss: 0.3088 - val_accuracy: 0.9199 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 45/48\n", + "256/256 [==============================] - 49s 192ms/step - loss: 0.3713 - accuracy: 0.8667 - val_loss: 0.2881 - val_accuracy: 0.9375 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 46/48\n", + "256/256 [==============================] - 49s 191ms/step - loss: 0.3581 - accuracy: 0.8765 - val_loss: 0.3279 - val_accuracy: 0.9135 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 47/48\n", + "256/256 [==============================] - 49s 192ms/step - loss: 0.2639 - accuracy: 0.9263 - val_loss: 0.2231 - val_accuracy: 0.9295 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 48/48\n", + "256/256 [==============================] - 49s 193ms/step - loss: 0.2355 - accuracy: 0.9238 - val_loss: 0.2298 - val_accuracy: 0.9327 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-043-0.9375.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9375\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2467\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model accuracy from \u001b[0m\u001b[0;32m0.9358974099159241 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.9375\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.22506119310855865. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m345.83 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m305.03 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m40.80 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [8] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m9\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 48)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 49/54\n", + "256/256 [==============================] - 55s 199ms/step - loss: 0.3562 - accuracy: 0.8745 - val_loss: 0.2796 - val_accuracy: 0.8974 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 50/54\n", + "256/256 [==============================] - 51s 200ms/step - loss: 0.3632 - accuracy: 0.8892 - val_loss: 0.2843 - val_accuracy: 0.9327 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 51/54\n", + "256/256 [==============================] - 50s 194ms/step - loss: 0.3904 - accuracy: 0.8652 - val_loss: 0.3229 - val_accuracy: 0.8894 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 52/54\n", + "256/256 [==============================] - 50s 195ms/step - loss: 0.3412 - accuracy: 0.8926 - val_loss: 0.2218 - val_accuracy: 0.9487 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 53/54\n", + "256/256 [==============================] - 50s 193ms/step - loss: 0.2635 - accuracy: 0.9204 - val_loss: 0.2349 - val_accuracy: 0.9327 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 54/54\n", + "256/256 [==============================] - 49s 190ms/step - loss: 0.2201 - accuracy: 0.9331 - val_loss: 0.2753 - val_accuracy: 0.8910 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-052-0.9487.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9487\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2218\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model accuracy from \u001b[0m\u001b[0;32m0.9375 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.9487179517745972\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model loss from \u001b[0m\u001b[0;32m0.22506119310855865 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.22177116572856903\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m349.93 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m305.99 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m43.93 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [9] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m10\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 54)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 55/60\n", + "256/256 [==============================] - 57s 205ms/step - loss: 0.3582 - accuracy: 0.8853 - val_loss: 1.1253 - val_accuracy: 0.6955 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 56/60\n", + "256/256 [==============================] - 51s 196ms/step - loss: 0.3713 - accuracy: 0.8823 - val_loss: 0.4083 - val_accuracy: 0.7885 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 57/60\n", + "256/256 [==============================] - 52s 201ms/step - loss: 0.3291 - accuracy: 0.8848 - val_loss: 0.3009 - val_accuracy: 0.8606 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 58/60\n", + "256/256 [==============================] - 50s 195ms/step - loss: 0.3178 - accuracy: 0.9058 - val_loss: 0.4083 - val_accuracy: 0.8590 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 59/60\n", + "256/256 [==============================] - 51s 199ms/step - loss: 0.2571 - accuracy: 0.9307 - val_loss: 0.2562 - val_accuracy: 0.9038 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 60/60\n", + "256/256 [==============================] - 51s 198ms/step - loss: 0.2568 - accuracy: 0.9209 - val_loss: 0.2685 - val_accuracy: 0.9038 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-059-0.9038.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9038\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2562\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.22177116572856903. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m350.67 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m311.58 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m39.08 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [10] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m11\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 60)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 61/66\n", + "256/256 [==============================] - 57s 205ms/step - loss: 0.3156 - accuracy: 0.8970 - val_loss: 0.2816 - val_accuracy: 0.9119 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 62/66\n", + "256/256 [==============================] - 52s 202ms/step - loss: 0.3533 - accuracy: 0.8877 - val_loss: 0.2945 - val_accuracy: 0.9183 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 63/66\n", + "256/256 [==============================] - 50s 196ms/step - loss: 0.3629 - accuracy: 0.8882 - val_loss: 0.2772 - val_accuracy: 0.9103 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 64/66\n", + "256/256 [==============================] - 50s 195ms/step - loss: 0.3021 - accuracy: 0.9136 - val_loss: 0.3078 - val_accuracy: 0.9054 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 65/66\n", + "256/256 [==============================] - 50s 193ms/step - loss: 0.2472 - accuracy: 0.9219 - val_loss: 0.4162 - val_accuracy: 0.8189 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 66/66\n", + "256/256 [==============================] - 51s 200ms/step - loss: 0.2013 - accuracy: 0.9517 - val_loss: 0.2229 - val_accuracy: 0.9263 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-066-0.9263.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9263\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2229\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.22177116572856903. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m350.09 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m311.16 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m38.92 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [11] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m12\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 66)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 67/72\n", + "256/256 [==============================] - 55s 196ms/step - loss: 0.2781 - accuracy: 0.9111 - val_loss: 0.5012 - val_accuracy: 0.8237 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 68/72\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.3271 - accuracy: 0.9038 - val_loss: 0.2999 - val_accuracy: 0.9279 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 69/72\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.3290 - accuracy: 0.9087 - val_loss: 0.3948 - val_accuracy: 0.9279 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 70/72\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.3285 - accuracy: 0.8979 - val_loss: 0.3564 - val_accuracy: 0.9263 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 71/72\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2478 - accuracy: 0.9302 - val_loss: 0.2477 - val_accuracy: 0.9279 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 72/72\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1993 - accuracy: 0.9458 - val_loss: 0.2491 - val_accuracy: 0.9327 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-072-0.9327.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9327\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2491\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.22177116572856903. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m323.09 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m283.75 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m39.34 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [12] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m13\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 72)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 73/78\n", + "256/256 [==============================] - 50s 183ms/step - loss: 0.2939 - accuracy: 0.9072 - val_loss: 0.3677 - val_accuracy: 0.9343 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 74/78\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.3566 - accuracy: 0.8843 - val_loss: 0.3543 - val_accuracy: 0.9247 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 75/78\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.3509 - accuracy: 0.8916 - val_loss: 0.3390 - val_accuracy: 0.9167 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 76/78\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.3289 - accuracy: 0.9097 - val_loss: 0.3772 - val_accuracy: 0.9135 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 77/78\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2213 - accuracy: 0.9404 - val_loss: 0.2276 - val_accuracy: 0.9423 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 78/78\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1726 - accuracy: 0.9600 - val_loss: 0.2253 - val_accuracy: 0.9407 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-077-0.9423.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9423\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2276\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.22177116572856903. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m312.50 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m279.87 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m32.63 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [13] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m14\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 78)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 79/84\n", + "256/256 [==============================] - 50s 184ms/step - loss: 0.2854 - accuracy: 0.9194 - val_loss: 0.3002 - val_accuracy: 0.9263 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 80/84\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.3165 - accuracy: 0.8979 - val_loss: 0.4973 - val_accuracy: 0.8077 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 81/84\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.3448 - accuracy: 0.8999 - val_loss: 0.4124 - val_accuracy: 0.9231 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 82/84\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.3198 - accuracy: 0.9214 - val_loss: 0.2585 - val_accuracy: 0.9279 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 83/84\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.2571 - accuracy: 0.9268 - val_loss: 0.2274 - val_accuracy: 0.9311 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 84/84\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2081 - accuracy: 0.9497 - val_loss: 0.2196 - val_accuracy: 0.9359 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-084-0.9359.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9359\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2196\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model loss from \u001b[0m\u001b[0;32m0.22177116572856903 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.2195880115032196\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m316.10 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m281.32 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m34.78 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [14] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m15\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 84)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 85/90\n", + "256/256 [==============================] - 50s 183ms/step - loss: 0.2659 - accuracy: 0.9204 - val_loss: 0.2711 - val_accuracy: 0.9407 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 86/90\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.3555 - accuracy: 0.8950 - val_loss: 0.4089 - val_accuracy: 0.8333 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 87/90\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.3097 - accuracy: 0.9155 - val_loss: 0.3248 - val_accuracy: 0.9151 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 88/90\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.2897 - accuracy: 0.9131 - val_loss: 0.2811 - val_accuracy: 0.9135 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 89/90\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2093 - accuracy: 0.9497 - val_loss: 0.2760 - val_accuracy: 0.9247 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 90/90\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1866 - accuracy: 0.9536 - val_loss: 0.2868 - val_accuracy: 0.9151 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-085-0.9407.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9407\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2711\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.2195880115032196. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m312.70 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m279.38 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m33.32 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [15] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m16\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 90)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 91/96\n", + "256/256 [==============================] - 50s 183ms/step - loss: 0.3088 - accuracy: 0.9058 - val_loss: 0.2267 - val_accuracy: 0.9359 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 92/96\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.3698 - accuracy: 0.8774 - val_loss: 0.3892 - val_accuracy: 0.9215 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 93/96\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.3414 - accuracy: 0.9053 - val_loss: 0.4820 - val_accuracy: 0.8638 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 94/96\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.3151 - accuracy: 0.9082 - val_loss: 0.2973 - val_accuracy: 0.9439 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 95/96\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2220 - accuracy: 0.9453 - val_loss: 0.2623 - val_accuracy: 0.9215 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 96/96\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2062 - accuracy: 0.9409 - val_loss: 0.2417 - val_accuracy: 0.9199 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-094-0.9439.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9439\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2973\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.2195880115032196. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m313.30 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m280.20 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m33.10 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [16] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m17\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 96)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 97/102\n", + "256/256 [==============================] - 51s 184ms/step - loss: 0.3256 - accuracy: 0.9087 - val_loss: 0.2532 - val_accuracy: 0.9247 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 98/102\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.3436 - accuracy: 0.8960 - val_loss: 0.2876 - val_accuracy: 0.9199 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 99/102\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.3054 - accuracy: 0.9033 - val_loss: 0.2421 - val_accuracy: 0.9343 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 100/102\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.3065 - accuracy: 0.9214 - val_loss: 0.4420 - val_accuracy: 0.9375 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 101/102\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.2821 - accuracy: 0.9214 - val_loss: 0.2932 - val_accuracy: 0.9375 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 102/102\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1880 - accuracy: 0.9526 - val_loss: 0.2346 - val_accuracy: 0.9423 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-102-0.9423.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9423\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2346\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.2195880115032196. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m314.92 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m281.14 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m33.79 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [17] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m18\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 102)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33m└───Shuffling data...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 103/108\n", + "256/256 [==============================] - 51s 184ms/step - loss: 0.2700 - accuracy: 0.9189 - val_loss: 0.2869 - val_accuracy: 0.9311 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 104/108\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.3335 - accuracy: 0.9072 - val_loss: 0.4121 - val_accuracy: 0.9103 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 105/108\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.3382 - accuracy: 0.9150 - val_loss: 0.3712 - val_accuracy: 0.9327 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 106/108\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2884 - accuracy: 0.9351 - val_loss: 0.3762 - val_accuracy: 0.8590 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 107/108\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.2295 - accuracy: 0.9419 - val_loss: 0.2174 - val_accuracy: 0.9423 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 108/108\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1938 - accuracy: 0.9492 - val_loss: 0.2403 - val_accuracy: 0.9423 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-107-0.9423.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9423\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2174\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model loss from \u001b[0m\u001b[0;32m0.2195880115032196 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.21743160486221313\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m321.67 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m282.30 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m39.36 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [18] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m19\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 108)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 109/114\n", + "256/256 [==============================] - 51s 185ms/step - loss: 0.2896 - accuracy: 0.9160 - val_loss: 0.3929 - val_accuracy: 0.8990 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 110/114\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.3225 - accuracy: 0.9116 - val_loss: 0.2669 - val_accuracy: 0.9263 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 111/114\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2926 - accuracy: 0.9229 - val_loss: 0.4608 - val_accuracy: 0.8878 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 112/114\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2669 - accuracy: 0.9209 - val_loss: 0.2904 - val_accuracy: 0.9247 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 113/114\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2234 - accuracy: 0.9473 - val_loss: 0.4278 - val_accuracy: 0.9247 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 114/114\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1819 - accuracy: 0.9546 - val_loss: 0.2227 - val_accuracy: 0.9375 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-114-0.9375.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9375\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2227\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.21743160486221313. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m316.72 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m282.49 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m34.24 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [19] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m20\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 114)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 115/120\n", + "256/256 [==============================] - 51s 185ms/step - loss: 0.2638 - accuracy: 0.9160 - val_loss: 0.2409 - val_accuracy: 0.9391 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 116/120\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.3396 - accuracy: 0.8979 - val_loss: 0.2746 - val_accuracy: 0.9199 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 117/120\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2945 - accuracy: 0.9136 - val_loss: 0.2682 - val_accuracy: 0.9167 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 118/120\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2795 - accuracy: 0.9316 - val_loss: 0.3202 - val_accuracy: 0.9295 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 119/120\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2383 - accuracy: 0.9355 - val_loss: 0.2700 - val_accuracy: 0.9311 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 120/120\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1877 - accuracy: 0.9478 - val_loss: 0.3409 - val_accuracy: 0.9327 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-115-0.9391.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9391\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2409\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.21743160486221313. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m316.52 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m281.61 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m34.91 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [20] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m21\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 120)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 121/126\n", + "256/256 [==============================] - 51s 185ms/step - loss: 0.3039 - accuracy: 0.9131 - val_loss: 0.3509 - val_accuracy: 0.9375 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 122/126\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.3234 - accuracy: 0.9092 - val_loss: 0.4477 - val_accuracy: 0.8013 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 123/126\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.3611 - accuracy: 0.8965 - val_loss: 0.3186 - val_accuracy: 0.8910 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 124/126\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2612 - accuracy: 0.9375 - val_loss: 0.3147 - val_accuracy: 0.9199 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 125/126\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2596 - accuracy: 0.9287 - val_loss: 0.2446 - val_accuracy: 0.9247 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 126/126\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1895 - accuracy: 0.9585 - val_loss: 0.2598 - val_accuracy: 0.9279 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-121-0.9375.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9375\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3509\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.21743160486221313. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m317.91 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m282.50 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m35.41 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [21] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m22\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 126)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 127/132\n", + "256/256 [==============================] - 51s 186ms/step - loss: 0.3124 - accuracy: 0.9150 - val_loss: 0.3509 - val_accuracy: 0.9263 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 128/132\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.3525 - accuracy: 0.8901 - val_loss: 0.2671 - val_accuracy: 0.9119 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 129/132\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.3308 - accuracy: 0.9009 - val_loss: 0.3931 - val_accuracy: 0.8333 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 130/132\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.2898 - accuracy: 0.9199 - val_loss: 0.2497 - val_accuracy: 0.9343 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 131/132\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2178 - accuracy: 0.9434 - val_loss: 0.3649 - val_accuracy: 0.9263 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 132/132\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1691 - accuracy: 0.9561 - val_loss: 0.2934 - val_accuracy: 0.9295 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-130-0.9343.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9343\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2497\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.21743160486221313. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m318.28 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m282.45 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m35.83 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [22] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m23\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 132)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Learning the patterns]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 133/138\n", + "256/256 [==============================] - 51s 185ms/step - loss: 0.2837 - accuracy: 0.9141 - val_loss: 0.3335 - val_accuracy: 0.9311 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 134/138\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.2763 - accuracy: 0.9194 - val_loss: 0.2933 - val_accuracy: 0.9359 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 135/138\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2699 - accuracy: 0.9297 - val_loss: 0.2995 - val_accuracy: 0.9071 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 136/138\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2943 - accuracy: 0.9238 - val_loss: 0.2386 - val_accuracy: 0.9215 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 137/138\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2269 - accuracy: 0.9380 - val_loss: 0.2149 - val_accuracy: 0.9375 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 138/138\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.1804 - accuracy: 0.9580 - val_loss: 0.2456 - val_accuracy: 0.9391 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-138-0.9391.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9391\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2455\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.21743160486221313. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m317.84 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m282.05 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m35.79 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [23] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m24\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 138)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0147\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 139/144\n", + "256/256 [==============================] - 50s 180ms/step - loss: 0.2347 - accuracy: 0.9297 - val_loss: 0.3001 - val_accuracy: 0.9295 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 140/144\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2650 - accuracy: 0.9229 - val_loss: 0.2768 - val_accuracy: 0.9359 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 141/144\n", + "256/256 [==============================] - 45s 174ms/step - loss: 0.3086 - accuracy: 0.9121 - val_loss: 0.2986 - val_accuracy: 0.9231 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 142/144\n", + "256/256 [==============================] - 44s 173ms/step - loss: 0.2620 - accuracy: 0.9331 - val_loss: 0.3041 - val_accuracy: 0.9103 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 143/144\n", + "256/256 [==============================] - 45s 174ms/step - loss: 0.2440 - accuracy: 0.9375 - val_loss: 0.2865 - val_accuracy: 0.9167 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 144/144\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2283 - accuracy: 0.9385 - val_loss: 0.3130 - val_accuracy: 0.9263 - lr: 5.8800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-140-0.9359.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9359\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2768\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.21743160486221313. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m310.50 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m274.43 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m36.07 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [24] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m25\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 144)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01464\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 145/150\n", + "256/256 [==============================] - 50s 181ms/step - loss: 0.3185 - accuracy: 0.9077 - val_loss: 0.2748 - val_accuracy: 0.9263 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 146/150\n", + "256/256 [==============================] - 45s 174ms/step - loss: 0.3295 - accuracy: 0.9131 - val_loss: 0.3341 - val_accuracy: 0.9199 - lr: 0.0146 - momentum: 0.8506\n", + "Epoch 147/150\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.3034 - accuracy: 0.9287 - val_loss: 0.4131 - val_accuracy: 0.8974 - lr: 0.0119 - momentum: 0.8688\n", + "Epoch 148/150\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2569 - accuracy: 0.9287 - val_loss: 0.3959 - val_accuracy: 0.8958 - lr: 0.0068 - momentum: 0.9037\n", + "Epoch 149/150\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.2157 - accuracy: 0.9395 - val_loss: 0.3447 - val_accuracy: 0.9327 - lr: 0.0020 - momentum: 0.9367\n", + "Epoch 150/150\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1528 - accuracy: 0.9604 - val_loss: 0.3013 - val_accuracy: 0.9359 - lr: 5.8560e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9359\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3013\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.21743160486221313. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m311.94 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m276.41 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m35.53 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [25] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m26\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 150)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01458\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 151/156\n", + "256/256 [==============================] - 49s 178ms/step - loss: 0.3113 - accuracy: 0.9087 - val_loss: 0.2813 - val_accuracy: 0.9327 - lr: 0.0088 - momentum: 0.8915\n", + "Epoch 152/156\n", + "256/256 [==============================] - 45s 174ms/step - loss: 0.2978 - accuracy: 0.9126 - val_loss: 0.3443 - val_accuracy: 0.9006 - lr: 0.0145 - momentum: 0.8506\n", + "Epoch 153/156\n", + "256/256 [==============================] - 45s 174ms/step - loss: 0.3167 - accuracy: 0.9155 - val_loss: 0.3006 - val_accuracy: 0.9006 - lr: 0.0118 - momentum: 0.8688\n", + "Epoch 154/156\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2776 - accuracy: 0.9355 - val_loss: 0.2325 - val_accuracy: 0.9375 - lr: 0.0067 - momentum: 0.9037\n", + "Epoch 155/156\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2035 - accuracy: 0.9531 - val_loss: 0.2372 - val_accuracy: 0.9311 - lr: 0.0019 - momentum: 0.9367\n", + "Epoch 156/156\n", + "256/256 [==============================] - 45s 174ms/step - loss: 0.1659 - accuracy: 0.9580 - val_loss: 0.2305 - val_accuracy: 0.9279 - lr: 5.8320e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9279\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2304\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.21743160486221313. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m309.28 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m273.82 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m35.46 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [26] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m27\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 156)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01452\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 157/162\n", + "256/256 [==============================] - 49s 179ms/step - loss: 0.2430 - accuracy: 0.9316 - val_loss: 0.2585 - val_accuracy: 0.9215 - lr: 0.0087 - momentum: 0.8915\n", + "Epoch 158/162\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2832 - accuracy: 0.9243 - val_loss: 0.3427 - val_accuracy: 0.9183 - lr: 0.0144 - momentum: 0.8506\n", + "Epoch 159/162\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.3188 - accuracy: 0.9102 - val_loss: 0.5467 - val_accuracy: 0.8894 - lr: 0.0118 - momentum: 0.8688\n", + "Epoch 160/162\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.3255 - accuracy: 0.9131 - val_loss: 0.3769 - val_accuracy: 0.9279 - lr: 0.0067 - momentum: 0.9037\n", + "Epoch 161/162\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2106 - accuracy: 0.9546 - val_loss: 0.2703 - val_accuracy: 0.9407 - lr: 0.0019 - momentum: 0.9367\n", + "Epoch 162/162\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.1778 - accuracy: 0.9536 - val_loss: 0.2150 - val_accuracy: 0.9215 - lr: 5.8080e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-161-0.9407.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9407\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2703\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.21743160486221313. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m312.10 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m275.52 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m36.58 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [27] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m28\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 162)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01446\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 163/168\n", + "256/256 [==============================] - 50s 181ms/step - loss: 0.2903 - accuracy: 0.9170 - val_loss: 0.2352 - val_accuracy: 0.9391 - lr: 0.0087 - momentum: 0.8915\n", + "Epoch 164/168\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.3076 - accuracy: 0.9126 - val_loss: 0.3734 - val_accuracy: 0.8942 - lr: 0.0144 - momentum: 0.8506\n", + "Epoch 165/168\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2687 - accuracy: 0.9248 - val_loss: 0.5690 - val_accuracy: 0.8958 - lr: 0.0117 - momentum: 0.8688\n", + "Epoch 166/168\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2571 - accuracy: 0.9385 - val_loss: 1.7227 - val_accuracy: 0.7147 - lr: 0.0067 - momentum: 0.9037\n", + "Epoch 167/168\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2202 - accuracy: 0.9409 - val_loss: 0.2906 - val_accuracy: 0.9359 - lr: 0.0019 - momentum: 0.9367\n", + "Epoch 168/168\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.1599 - accuracy: 0.9565 - val_loss: 0.3119 - val_accuracy: 0.9343 - lr: 5.7840e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9343\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3119\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.21743160486221313. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m311.90 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m275.90 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m36.00 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [28] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m29\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 168)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0144\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 169/174\n", + "256/256 [==============================] - 49s 179ms/step - loss: 0.2573 - accuracy: 0.9277 - val_loss: 0.2854 - val_accuracy: 0.9295 - lr: 0.0087 - momentum: 0.8915\n", + "Epoch 170/174\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2967 - accuracy: 0.9146 - val_loss: 0.3175 - val_accuracy: 0.9375 - lr: 0.0143 - momentum: 0.8506\n", + "Epoch 171/174\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2875 - accuracy: 0.9219 - val_loss: 0.3138 - val_accuracy: 0.9343 - lr: 0.0117 - momentum: 0.8688\n", + "Epoch 172/174\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2320 - accuracy: 0.9395 - val_loss: 0.4648 - val_accuracy: 0.9343 - lr: 0.0067 - momentum: 0.9037\n", + "Epoch 173/174\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2052 - accuracy: 0.9531 - val_loss: 0.2382 - val_accuracy: 0.9439 - lr: 0.0019 - momentum: 0.9367\n", + "Epoch 174/174\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1521 - accuracy: 0.9683 - val_loss: 0.2314 - val_accuracy: 0.9423 - lr: 5.7600e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9423\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2314\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.21743160486221313. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m312.51 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m276.07 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m36.44 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [29] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m30\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 174)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01434\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 175/180\n", + "256/256 [==============================] - 49s 179ms/step - loss: 0.2609 - accuracy: 0.9287 - val_loss: 0.2654 - val_accuracy: 0.9407 - lr: 0.0086 - momentum: 0.8915\n", + "Epoch 176/180\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2647 - accuracy: 0.9385 - val_loss: 0.2827 - val_accuracy: 0.9054 - lr: 0.0143 - momentum: 0.8506\n", + "Epoch 177/180\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2902 - accuracy: 0.9131 - val_loss: 0.3556 - val_accuracy: 0.9263 - lr: 0.0116 - momentum: 0.8688\n", + "Epoch 178/180\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2357 - accuracy: 0.9448 - val_loss: 0.2267 - val_accuracy: 0.9359 - lr: 0.0066 - momentum: 0.9037\n", + "Epoch 179/180\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.1682 - accuracy: 0.9644 - val_loss: 0.2848 - val_accuracy: 0.9327 - lr: 0.0019 - momentum: 0.9367\n", + "Epoch 180/180\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.1863 - accuracy: 0.9585 - val_loss: 0.2182 - val_accuracy: 0.9375 - lr: 5.7360e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9375\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2182\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.21743160486221313. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m311.12 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m274.94 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m36.18 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [30] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m31\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 180)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01428\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 181/186\n", + "256/256 [==============================] - 49s 178ms/step - loss: 0.2376 - accuracy: 0.9355 - val_loss: 0.4749 - val_accuracy: 0.9183 - lr: 0.0086 - momentum: 0.8915\n", + "Epoch 182/186\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2735 - accuracy: 0.9219 - val_loss: 0.6848 - val_accuracy: 0.8734 - lr: 0.0142 - momentum: 0.8506\n", + "Epoch 183/186\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2613 - accuracy: 0.9326 - val_loss: 0.3324 - val_accuracy: 0.9359 - lr: 0.0116 - momentum: 0.8688\n", + "Epoch 184/186\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2152 - accuracy: 0.9414 - val_loss: 0.2848 - val_accuracy: 0.9423 - lr: 0.0066 - momentum: 0.9037\n", + "Epoch 185/186\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1656 - accuracy: 0.9634 - val_loss: 0.3078 - val_accuracy: 0.8910 - lr: 0.0019 - momentum: 0.9367\n", + "Epoch 186/186\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1400 - accuracy: 0.9702 - val_loss: 0.2298 - val_accuracy: 0.9311 - lr: 5.7120e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9311\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2298\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.21743160486221313. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m312.11 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m275.49 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m36.62 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [31] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m32\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 186)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01422\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 187/192\n", + "256/256 [==============================] - 50s 180ms/step - loss: 0.2385 - accuracy: 0.9321 - val_loss: 0.2560 - val_accuracy: 0.9295 - lr: 0.0086 - momentum: 0.8915\n", + "Epoch 188/192\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2592 - accuracy: 0.9258 - val_loss: 0.7048 - val_accuracy: 0.6458 - lr: 0.0141 - momentum: 0.8506\n", + "Epoch 189/192\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2519 - accuracy: 0.9346 - val_loss: 0.2099 - val_accuracy: 0.9471 - lr: 0.0115 - momentum: 0.8688\n", + "Epoch 190/192\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2154 - accuracy: 0.9497 - val_loss: 0.3195 - val_accuracy: 0.8878 - lr: 0.0066 - momentum: 0.9037\n", + "Epoch 191/192\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.1856 - accuracy: 0.9546 - val_loss: 0.2592 - val_accuracy: 0.9407 - lr: 0.0019 - momentum: 0.9367\n", + "Epoch 192/192\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1447 - accuracy: 0.9658 - val_loss: 0.2122 - val_accuracy: 0.9439 - lr: 5.6880e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-189-0.9471.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9471\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2099\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9487179517745972. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model loss from \u001b[0m\u001b[0;32m0.21743160486221313 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.20990584790706635\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m315.82 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m276.17 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m39.65 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [32] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m33\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 192)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01416\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 193/198\n", + "256/256 [==============================] - 50s 181ms/step - loss: 0.2852 - accuracy: 0.9233 - val_loss: 0.2252 - val_accuracy: 0.9551 - lr: 0.0085 - momentum: 0.8915\n", + "Epoch 194/198\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2926 - accuracy: 0.9175 - val_loss: 0.3387 - val_accuracy: 0.8702 - lr: 0.0141 - momentum: 0.8506\n", + "Epoch 195/198\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.3013 - accuracy: 0.9111 - val_loss: 0.2416 - val_accuracy: 0.9535 - lr: 0.0115 - momentum: 0.8688\n", + "Epoch 196/198\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2782 - accuracy: 0.9297 - val_loss: 0.2526 - val_accuracy: 0.9327 - lr: 0.0066 - momentum: 0.9037\n", + "Epoch 197/198\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.1860 - accuracy: 0.9556 - val_loss: 0.2367 - val_accuracy: 0.9375 - lr: 0.0019 - momentum: 0.9367\n", + "Epoch 198/198\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.1499 - accuracy: 0.9644 - val_loss: 0.2806 - val_accuracy: 0.9343 - lr: 5.6640e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-193-0.9551.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9551\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2252\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model accuracy from \u001b[0m\u001b[0;32m0.9487179517745972 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.9551281929016113\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20990584790706635. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m316.28 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m276.31 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m39.97 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [33] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m34\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 198)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0141\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 199/204\n", + "256/256 [==============================] - 50s 182ms/step - loss: 0.2587 - accuracy: 0.9248 - val_loss: 0.2475 - val_accuracy: 0.9215 - lr: 0.0085 - momentum: 0.8915\n", + "Epoch 200/204\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.2758 - accuracy: 0.9204 - val_loss: 0.2672 - val_accuracy: 0.9407 - lr: 0.0140 - momentum: 0.8506\n", + "Epoch 201/204\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2505 - accuracy: 0.9341 - val_loss: 0.3928 - val_accuracy: 0.8574 - lr: 0.0114 - momentum: 0.8688\n", + "Epoch 202/204\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2604 - accuracy: 0.9316 - val_loss: 0.2649 - val_accuracy: 0.9295 - lr: 0.0065 - momentum: 0.9037\n", + "Epoch 203/204\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.1986 - accuracy: 0.9551 - val_loss: 0.2377 - val_accuracy: 0.9391 - lr: 0.0019 - momentum: 0.9367\n", + "Epoch 204/204\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.1519 - accuracy: 0.9624 - val_loss: 0.2117 - val_accuracy: 0.9407 - lr: 5.6400e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9407\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2117\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9551281929016113. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20990584790706635. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m314.19 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m276.11 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m38.07 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [34] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m35\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 204)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01404\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 205/210\n", + "256/256 [==============================] - 50s 179ms/step - loss: 0.2498 - accuracy: 0.9204 - val_loss: 0.3110 - val_accuracy: 0.9183 - lr: 0.0085 - momentum: 0.8915\n", + "Epoch 206/210\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.3439 - accuracy: 0.8901 - val_loss: 0.3838 - val_accuracy: 0.9167 - lr: 0.0140 - momentum: 0.8506\n", + "Epoch 207/210\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.3214 - accuracy: 0.9214 - val_loss: 0.3202 - val_accuracy: 0.8622 - lr: 0.0114 - momentum: 0.8688\n", + "Epoch 208/210\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2891 - accuracy: 0.9170 - val_loss: 0.4629 - val_accuracy: 0.9038 - lr: 0.0065 - momentum: 0.9037\n", + "Epoch 209/210\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2779 - accuracy: 0.9424 - val_loss: 0.4567 - val_accuracy: 0.9247 - lr: 0.0019 - momentum: 0.9367\n", + "Epoch 210/210\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2118 - accuracy: 0.9570 - val_loss: 0.4585 - val_accuracy: 0.9183 - lr: 5.6160e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9199\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.4585\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9551281929016113. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20990584790706635. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m314.31 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m276.11 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m38.20 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [35] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m36\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 210)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33m└───Shuffling data...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01398\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 211/216\n", + "256/256 [==============================] - 49s 179ms/step - loss: 0.3015 - accuracy: 0.9155 - val_loss: 0.7072 - val_accuracy: 0.8381 - lr: 0.0084 - momentum: 0.8915\n", + "Epoch 212/216\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.3203 - accuracy: 0.9116 - val_loss: 0.2684 - val_accuracy: 0.9455 - lr: 0.0139 - momentum: 0.8506\n", + "Epoch 213/216\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2697 - accuracy: 0.9307 - val_loss: 0.3074 - val_accuracy: 0.9407 - lr: 0.0113 - momentum: 0.8688\n", + "Epoch 214/216\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2326 - accuracy: 0.9404 - val_loss: 0.2609 - val_accuracy: 0.9423 - lr: 0.0065 - momentum: 0.9037\n", + "Epoch 215/216\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2165 - accuracy: 0.9458 - val_loss: 0.4155 - val_accuracy: 0.9359 - lr: 0.0019 - momentum: 0.9367\n", + "Epoch 216/216\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.1497 - accuracy: 0.9663 - val_loss: 0.4218 - val_accuracy: 0.9295 - lr: 5.5920e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9295\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.4218\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9551281929016113. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20990584790706635. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m316.30 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m276.30 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m40.00 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [36] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m37\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 216)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01392\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 217/222\n", + "256/256 [==============================] - 50s 179ms/step - loss: 0.2571 - accuracy: 0.9263 - val_loss: 0.3335 - val_accuracy: 0.9359 - lr: 0.0084 - momentum: 0.8915\n", + "Epoch 218/222\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2999 - accuracy: 0.9131 - val_loss: 0.2366 - val_accuracy: 0.9423 - lr: 0.0138 - momentum: 0.8506\n", + "Epoch 219/222\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2779 - accuracy: 0.9331 - val_loss: 0.2879 - val_accuracy: 0.8974 - lr: 0.0113 - momentum: 0.8688\n", + "Epoch 220/222\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2349 - accuracy: 0.9399 - val_loss: 0.3135 - val_accuracy: 0.8990 - lr: 0.0064 - momentum: 0.9037\n", + "Epoch 221/222\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2008 - accuracy: 0.9546 - val_loss: 0.2773 - val_accuracy: 0.9247 - lr: 0.0019 - momentum: 0.9367\n", + "Epoch 222/222\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1394 - accuracy: 0.9683 - val_loss: 0.3434 - val_accuracy: 0.9279 - lr: 5.5680e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9279\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3434\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9551281929016113. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20990584790706635. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m314.11 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m276.01 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m38.11 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [37] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m38\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 222)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01386\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 223/228\n", + "256/256 [==============================] - 50s 180ms/step - loss: 0.2307 - accuracy: 0.9307 - val_loss: 0.2328 - val_accuracy: 0.9375 - lr: 0.0083 - momentum: 0.8915\n", + "Epoch 224/228\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2818 - accuracy: 0.9204 - val_loss: 0.4113 - val_accuracy: 0.8718 - lr: 0.0138 - momentum: 0.8506\n", + "Epoch 225/228\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.3218 - accuracy: 0.9136 - val_loss: 0.2276 - val_accuracy: 0.9279 - lr: 0.0113 - momentum: 0.8688\n", + "Epoch 226/228\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1970 - accuracy: 0.9492 - val_loss: 0.3562 - val_accuracy: 0.8638 - lr: 0.0064 - momentum: 0.9037\n", + "Epoch 227/228\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1782 - accuracy: 0.9639 - val_loss: 0.2323 - val_accuracy: 0.9295 - lr: 0.0018 - momentum: 0.9367\n", + "Epoch 228/228\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1440 - accuracy: 0.9648 - val_loss: 0.2243 - val_accuracy: 0.9375 - lr: 5.5440e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9375\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2243\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9551281929016113. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20990584790706635. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m313.88 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m275.76 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m38.13 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [38] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m39\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 228)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0138\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 229/234\n", + "256/256 [==============================] - 50s 180ms/step - loss: 0.2119 - accuracy: 0.9375 - val_loss: 0.2668 - val_accuracy: 0.9423 - lr: 0.0083 - momentum: 0.8915\n", + "Epoch 230/234\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2587 - accuracy: 0.9321 - val_loss: 0.5260 - val_accuracy: 0.8157 - lr: 0.0137 - momentum: 0.8506\n", + "Epoch 231/234\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2948 - accuracy: 0.9136 - val_loss: 0.2672 - val_accuracy: 0.9343 - lr: 0.0112 - momentum: 0.8688\n", + "Epoch 232/234\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1842 - accuracy: 0.9541 - val_loss: 0.2506 - val_accuracy: 0.8990 - lr: 0.0064 - momentum: 0.9037\n", + "Epoch 233/234\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1561 - accuracy: 0.9595 - val_loss: 0.3073 - val_accuracy: 0.9327 - lr: 0.0018 - momentum: 0.9367\n", + "Epoch 234/234\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1629 - accuracy: 0.9688 - val_loss: 0.2668 - val_accuracy: 0.9279 - lr: 5.5200e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9279\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2668\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9551281929016113. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20990584790706635. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m314.90 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m276.19 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m38.71 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [39] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m40\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 234)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01374\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 235/240\n", + "256/256 [==============================] - 50s 179ms/step - loss: 0.2425 - accuracy: 0.9351 - val_loss: 0.2922 - val_accuracy: 0.9359 - lr: 0.0083 - momentum: 0.8915\n", + "Epoch 236/240\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2727 - accuracy: 0.9082 - val_loss: 0.5939 - val_accuracy: 0.7740 - lr: 0.0137 - momentum: 0.8506\n", + "Epoch 237/240\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2865 - accuracy: 0.9287 - val_loss: 0.3467 - val_accuracy: 0.9231 - lr: 0.0112 - momentum: 0.8688\n", + "Epoch 238/240\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2779 - accuracy: 0.9360 - val_loss: 0.3094 - val_accuracy: 0.9119 - lr: 0.0064 - momentum: 0.9037\n", + "Epoch 239/240\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1906 - accuracy: 0.9619 - val_loss: 0.3536 - val_accuracy: 0.9263 - lr: 0.0018 - momentum: 0.9367\n", + "Epoch 240/240\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1518 - accuracy: 0.9673 - val_loss: 0.2827 - val_accuracy: 0.9231 - lr: 5.4960e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9231\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2827\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9551281929016113. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20990584790706635. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m315.31 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m275.95 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m39.36 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [40] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m41\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 240)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01368\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 241/246\n", + "256/256 [==============================] - 50s 180ms/step - loss: 0.2409 - accuracy: 0.9297 - val_loss: 0.3638 - val_accuracy: 0.9006 - lr: 0.0082 - momentum: 0.8915\n", + "Epoch 242/246\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2893 - accuracy: 0.9214 - val_loss: 0.4081 - val_accuracy: 0.8462 - lr: 0.0136 - momentum: 0.8506\n", + "Epoch 243/246\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2682 - accuracy: 0.9346 - val_loss: 0.3090 - val_accuracy: 0.8910 - lr: 0.0111 - momentum: 0.8688\n", + "Epoch 244/246\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2327 - accuracy: 0.9502 - val_loss: 0.3252 - val_accuracy: 0.9022 - lr: 0.0063 - momentum: 0.9037\n", + "Epoch 245/246\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2007 - accuracy: 0.9521 - val_loss: 0.2587 - val_accuracy: 0.9006 - lr: 0.0018 - momentum: 0.9367\n", + "Epoch 246/246\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.1625 - accuracy: 0.9653 - val_loss: 0.2111 - val_accuracy: 0.9231 - lr: 5.4720e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9231\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2111\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9551281929016113. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20990584790706635. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m315.11 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m275.97 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m39.14 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [41] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m42\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 246)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01362\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 247/252\n", + "256/256 [==============================] - 50s 180ms/step - loss: 0.2656 - accuracy: 0.9224 - val_loss: 0.2411 - val_accuracy: 0.9343 - lr: 0.0082 - momentum: 0.8915\n", + "Epoch 248/252\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2665 - accuracy: 0.9355 - val_loss: 0.3326 - val_accuracy: 0.9343 - lr: 0.0135 - momentum: 0.8506\n", + "Epoch 249/252\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2297 - accuracy: 0.9478 - val_loss: 0.3011 - val_accuracy: 0.9423 - lr: 0.0111 - momentum: 0.8688\n", + "Epoch 250/252\n", + "256/256 [==============================] - 45s 175ms/step - loss: 0.2331 - accuracy: 0.9424 - val_loss: 0.2626 - val_accuracy: 0.9407 - lr: 0.0063 - momentum: 0.9037\n", + "Epoch 251/252\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1532 - accuracy: 0.9619 - val_loss: 0.2534 - val_accuracy: 0.9375 - lr: 0.0018 - momentum: 0.9367\n", + "Epoch 252/252\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1258 - accuracy: 0.9717 - val_loss: 0.2721 - val_accuracy: 0.9375 - lr: 5.4480e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9375\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2721\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9551281929016113. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20990584790706635. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m315.89 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m276.09 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m39.81 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [42] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m43\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 252)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01356\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 253/258\n", + "256/256 [==============================] - 50s 180ms/step - loss: 0.2579 - accuracy: 0.9268 - val_loss: 0.2668 - val_accuracy: 0.9359 - lr: 0.0082 - momentum: 0.8915\n", + "Epoch 254/258\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2815 - accuracy: 0.9316 - val_loss: 0.3802 - val_accuracy: 0.9375 - lr: 0.0135 - momentum: 0.8506\n", + "Epoch 255/258\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2712 - accuracy: 0.9287 - val_loss: 0.2258 - val_accuracy: 0.9407 - lr: 0.0110 - momentum: 0.8688\n", + "Epoch 256/258\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2200 - accuracy: 0.9517 - val_loss: 0.2347 - val_accuracy: 0.9455 - lr: 0.0063 - momentum: 0.9037\n", + "Epoch 257/258\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1762 - accuracy: 0.9595 - val_loss: 0.2185 - val_accuracy: 0.9487 - lr: 0.0018 - momentum: 0.9367\n", + "Epoch 258/258\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.1379 - accuracy: 0.9673 - val_loss: 0.3341 - val_accuracy: 0.9375 - lr: 5.4240e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9375\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3341\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9551281929016113. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20990584790706635. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m316.30 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m277.32 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m38.98 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [43] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m44\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 258)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0135\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 259/264\n", + "256/256 [==============================] - 50s 179ms/step - loss: 0.2152 - accuracy: 0.9424 - val_loss: 0.4052 - val_accuracy: 0.9375 - lr: 0.0081 - momentum: 0.8915\n", + "Epoch 260/264\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2398 - accuracy: 0.9346 - val_loss: 0.2218 - val_accuracy: 0.9567 - lr: 0.0134 - momentum: 0.8506\n", + "Epoch 261/264\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2608 - accuracy: 0.9302 - val_loss: 0.2922 - val_accuracy: 0.9327 - lr: 0.0110 - momentum: 0.8688\n", + "Epoch 262/264\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2365 - accuracy: 0.9497 - val_loss: 0.3023 - val_accuracy: 0.9247 - lr: 0.0062 - momentum: 0.9037\n", + "Epoch 263/264\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1514 - accuracy: 0.9692 - val_loss: 0.3830 - val_accuracy: 0.9311 - lr: 0.0018 - momentum: 0.9367\n", + "Epoch 264/264\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1281 - accuracy: 0.9736 - val_loss: 0.4081 - val_accuracy: 0.9263 - lr: 5.4000e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-260-0.9567.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9567\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2218\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model accuracy from \u001b[0m\u001b[0;32m0.9551281929016113 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.9567307829856873\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20990584790706635. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m319.99 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m277.29 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m42.70 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [44] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m45\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 264)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01344\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 265/270\n", + "256/256 [==============================] - 50s 182ms/step - loss: 0.2583 - accuracy: 0.9321 - val_loss: 0.2134 - val_accuracy: 0.9551 - lr: 0.0081 - momentum: 0.8915\n", + "Epoch 266/270\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2626 - accuracy: 0.9414 - val_loss: 0.3394 - val_accuracy: 0.8830 - lr: 0.0134 - momentum: 0.8506\n", + "Epoch 267/270\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2253 - accuracy: 0.9497 - val_loss: 0.2738 - val_accuracy: 0.9375 - lr: 0.0109 - momentum: 0.8688\n", + "Epoch 268/270\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.1872 - accuracy: 0.9551 - val_loss: 0.2326 - val_accuracy: 0.9439 - lr: 0.0062 - momentum: 0.9037\n", + "Epoch 269/270\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1712 - accuracy: 0.9619 - val_loss: 0.3063 - val_accuracy: 0.8814 - lr: 0.0018 - momentum: 0.9367\n", + "Epoch 270/270\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1562 - accuracy: 0.9609 - val_loss: 0.2072 - val_accuracy: 0.9407 - lr: 5.3760e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-265-0.9551.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9551\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2134\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20990584790706635. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m318.47 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m277.75 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m40.72 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [45] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m46\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 270)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01338\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 271/276\n", + "256/256 [==============================] - 51s 183ms/step - loss: 0.2065 - accuracy: 0.9492 - val_loss: 0.2984 - val_accuracy: 0.9215 - lr: 0.0081 - momentum: 0.8915\n", + "Epoch 272/276\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2423 - accuracy: 0.9321 - val_loss: 0.3009 - val_accuracy: 0.9375 - lr: 0.0133 - momentum: 0.8506\n", + "Epoch 273/276\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2357 - accuracy: 0.9390 - val_loss: 0.2944 - val_accuracy: 0.9359 - lr: 0.0109 - momentum: 0.8688\n", + "Epoch 274/276\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2622 - accuracy: 0.9468 - val_loss: 0.3181 - val_accuracy: 0.9391 - lr: 0.0062 - momentum: 0.9037\n", + "Epoch 275/276\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1550 - accuracy: 0.9668 - val_loss: 0.4649 - val_accuracy: 0.9215 - lr: 0.0018 - momentum: 0.9367\n", + "Epoch 276/276\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.1352 - accuracy: 0.9678 - val_loss: 0.3932 - val_accuracy: 0.9295 - lr: 5.3520e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9295\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3932\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20990584790706635. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m320.07 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m279.36 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m40.70 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [46] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m47\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 276)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01332\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 277/282\n", + "256/256 [==============================] - 50s 180ms/step - loss: 0.1981 - accuracy: 0.9482 - val_loss: 0.3007 - val_accuracy: 0.9295 - lr: 0.0080 - momentum: 0.8915\n", + "Epoch 278/282\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2820 - accuracy: 0.9189 - val_loss: 0.2647 - val_accuracy: 0.9311 - lr: 0.0132 - momentum: 0.8506\n", + "Epoch 279/282\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2488 - accuracy: 0.9272 - val_loss: 0.3855 - val_accuracy: 0.9471 - lr: 0.0108 - momentum: 0.8688\n", + "Epoch 280/282\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.3068 - accuracy: 0.9248 - val_loss: 0.3390 - val_accuracy: 0.9006 - lr: 0.0062 - momentum: 0.9037\n", + "Epoch 281/282\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2510 - accuracy: 0.9458 - val_loss: 0.3447 - val_accuracy: 0.9022 - lr: 0.0018 - momentum: 0.9367\n", + "Epoch 282/282\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.1883 - accuracy: 0.9639 - val_loss: 0.3202 - val_accuracy: 0.9343 - lr: 5.3280e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9343\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3202\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20990584790706635. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m319.08 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m278.12 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m40.96 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [47] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m48\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 282)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01326\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 283/288\n", + "256/256 [==============================] - 50s 180ms/step - loss: 0.2534 - accuracy: 0.9224 - val_loss: 0.2646 - val_accuracy: 0.9183 - lr: 0.0080 - momentum: 0.8915\n", + "Epoch 284/288\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2299 - accuracy: 0.9365 - val_loss: 0.2172 - val_accuracy: 0.9391 - lr: 0.0132 - momentum: 0.8506\n", + "Epoch 285/288\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.2070 - accuracy: 0.9473 - val_loss: 0.2389 - val_accuracy: 0.9327 - lr: 0.0108 - momentum: 0.8688\n", + "Epoch 286/288\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2152 - accuracy: 0.9497 - val_loss: 0.2141 - val_accuracy: 0.9455 - lr: 0.0061 - momentum: 0.9037\n", + "Epoch 287/288\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.1948 - accuracy: 0.9541 - val_loss: 0.2333 - val_accuracy: 0.9327 - lr: 0.0018 - momentum: 0.9367\n", + "Epoch 288/288\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.1655 - accuracy: 0.9590 - val_loss: 0.2491 - val_accuracy: 0.9327 - lr: 5.3040e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9327\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2491\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20990584790706635. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m318.92 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m277.33 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m41.59 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [48] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m49\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 288)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0132\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 289/294\n", + "256/256 [==============================] - 50s 180ms/step - loss: 0.2384 - accuracy: 0.9297 - val_loss: 0.2766 - val_accuracy: 0.9359 - lr: 0.0079 - momentum: 0.8915\n", + "Epoch 290/294\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2199 - accuracy: 0.9360 - val_loss: 0.2895 - val_accuracy: 0.9359 - lr: 0.0131 - momentum: 0.8506\n", + "Epoch 291/294\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2179 - accuracy: 0.9390 - val_loss: 0.2628 - val_accuracy: 0.9311 - lr: 0.0107 - momentum: 0.8688\n", + "Epoch 292/294\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.1770 - accuracy: 0.9595 - val_loss: 0.2369 - val_accuracy: 0.9391 - lr: 0.0061 - momentum: 0.9037\n", + "Epoch 293/294\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1705 - accuracy: 0.9575 - val_loss: 0.2280 - val_accuracy: 0.9391 - lr: 0.0018 - momentum: 0.9367\n", + "Epoch 294/294\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1195 - accuracy: 0.9751 - val_loss: 0.2580 - val_accuracy: 0.9391 - lr: 5.2800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9391\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2580\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20990584790706635. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m318.72 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m277.14 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m41.58 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [49] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m50\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 294)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01314\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 295/300\n", + "256/256 [==============================] - 50s 180ms/step - loss: 0.2363 - accuracy: 0.9331 - val_loss: 0.2041 - val_accuracy: 0.9423 - lr: 0.0079 - momentum: 0.8915\n", + "Epoch 296/300\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2320 - accuracy: 0.9438 - val_loss: 0.2156 - val_accuracy: 0.9263 - lr: 0.0131 - momentum: 0.8506\n", + "Epoch 297/300\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2299 - accuracy: 0.9336 - val_loss: 0.3079 - val_accuracy: 0.9183 - lr: 0.0107 - momentum: 0.8688\n", + "Epoch 298/300\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2252 - accuracy: 0.9497 - val_loss: 0.3596 - val_accuracy: 0.9423 - lr: 0.0061 - momentum: 0.9037\n", + "Epoch 299/300\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1734 - accuracy: 0.9561 - val_loss: 0.2186 - val_accuracy: 0.9135 - lr: 0.0018 - momentum: 0.9367\n", + "Epoch 300/300\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.1446 - accuracy: 0.9653 - val_loss: 0.2723 - val_accuracy: 0.9375 - lr: 5.2560e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;31mERROR: Failed to load weights. Error: max() arg is an empty sequence\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9375\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2724\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20990584790706635. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m318.08 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m276.85 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m41.23 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [50] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m51\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 300)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01308\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 301/306\n", + "256/256 [==============================] - 51s 183ms/step - loss: 0.2388 - accuracy: 0.9302 - val_loss: 0.2832 - val_accuracy: 0.9263 - lr: 0.0079 - momentum: 0.8915\n", + "Epoch 302/306\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2805 - accuracy: 0.9116 - val_loss: 0.4591 - val_accuracy: 0.8974 - lr: 0.0130 - momentum: 0.8506\n", + "Epoch 303/306\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2742 - accuracy: 0.9312 - val_loss: 0.3353 - val_accuracy: 0.9263 - lr: 0.0106 - momentum: 0.8688\n", + "Epoch 304/306\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2330 - accuracy: 0.9390 - val_loss: 0.4355 - val_accuracy: 0.9087 - lr: 0.0061 - momentum: 0.9037\n", + "Epoch 305/306\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1988 - accuracy: 0.9565 - val_loss: 0.5020 - val_accuracy: 0.9119 - lr: 0.0017 - momentum: 0.9367\n", + "Epoch 306/306\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1722 - accuracy: 0.9629 - val_loss: 0.3481 - val_accuracy: 0.9295 - lr: 5.2320e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9295\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3481\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20990584790706635. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m320.33 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m278.49 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m41.83 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [51] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m52\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 306)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01302\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 307/312\n", + "256/256 [==============================] - 51s 184ms/step - loss: 0.2609 - accuracy: 0.9229 - val_loss: 0.3252 - val_accuracy: 0.9311 - lr: 0.0078 - momentum: 0.8915\n", + "Epoch 308/312\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.2811 - accuracy: 0.9346 - val_loss: 0.5813 - val_accuracy: 0.8622 - lr: 0.0129 - momentum: 0.8506\n", + "Epoch 309/312\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2753 - accuracy: 0.9316 - val_loss: 0.2384 - val_accuracy: 0.9503 - lr: 0.0106 - momentum: 0.8688\n", + "Epoch 310/312\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2208 - accuracy: 0.9458 - val_loss: 0.3539 - val_accuracy: 0.9135 - lr: 0.0060 - momentum: 0.9037\n", + "Epoch 311/312\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1831 - accuracy: 0.9575 - val_loss: 0.2050 - val_accuracy: 0.9519 - lr: 0.0017 - momentum: 0.9367\n", + "Epoch 312/312\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.1277 - accuracy: 0.9702 - val_loss: 0.2321 - val_accuracy: 0.9439 - lr: 5.2080e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-311-0.9519.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9519\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2050\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model loss from \u001b[0m\u001b[0;32m0.20990584790706635 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.20500072836875916\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m325.79 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m279.91 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m45.89 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [52] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m53\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 312)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01296\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 313/318\n", + "256/256 [==============================] - 50s 182ms/step - loss: 0.2367 - accuracy: 0.9307 - val_loss: 0.2165 - val_accuracy: 0.9423 - lr: 0.0078 - momentum: 0.8915\n", + "Epoch 314/318\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2548 - accuracy: 0.9351 - val_loss: 0.2510 - val_accuracy: 0.9391 - lr: 0.0129 - momentum: 0.8506\n", + "Epoch 315/318\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2100 - accuracy: 0.9438 - val_loss: 0.2889 - val_accuracy: 0.9391 - lr: 0.0105 - momentum: 0.8688\n", + "Epoch 316/318\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2010 - accuracy: 0.9492 - val_loss: 0.2971 - val_accuracy: 0.9359 - lr: 0.0060 - momentum: 0.9037\n", + "Epoch 317/318\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1639 - accuracy: 0.9683 - val_loss: 0.2362 - val_accuracy: 0.9487 - lr: 0.0017 - momentum: 0.9367\n", + "Epoch 318/318\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.1011 - accuracy: 0.9800 - val_loss: 0.3076 - val_accuracy: 0.9439 - lr: 5.1840e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9439\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3076\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m321.77 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m278.95 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m42.81 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [53] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m54\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 318)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33m└───Shuffling data...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0129\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 319/324\n", + "256/256 [==============================] - 50s 180ms/step - loss: 0.1977 - accuracy: 0.9463 - val_loss: 0.3348 - val_accuracy: 0.9343 - lr: 0.0078 - momentum: 0.8915\n", + "Epoch 320/324\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2123 - accuracy: 0.9409 - val_loss: 0.3236 - val_accuracy: 0.9471 - lr: 0.0128 - momentum: 0.8506\n", + "Epoch 321/324\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2150 - accuracy: 0.9463 - val_loss: 0.2515 - val_accuracy: 0.9407 - lr: 0.0105 - momentum: 0.8688\n", + "Epoch 322/324\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.2006 - accuracy: 0.9565 - val_loss: 0.4065 - val_accuracy: 0.9231 - lr: 0.0060 - momentum: 0.9037\n", + "Epoch 323/324\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.1589 - accuracy: 0.9658 - val_loss: 0.2775 - val_accuracy: 0.9455 - lr: 0.0017 - momentum: 0.9367\n", + "Epoch 324/324\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.1196 - accuracy: 0.9775 - val_loss: 0.3090 - val_accuracy: 0.9439 - lr: 5.1600e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9439\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3090\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m323.91 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m278.00 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m45.91 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [54] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m55\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 324)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01284\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 325/330\n", + "256/256 [==============================] - 50s 181ms/step - loss: 0.2029 - accuracy: 0.9453 - val_loss: 0.2921 - val_accuracy: 0.9423 - lr: 0.0077 - momentum: 0.8915\n", + "Epoch 326/330\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2446 - accuracy: 0.9370 - val_loss: 0.2964 - val_accuracy: 0.9471 - lr: 0.0128 - momentum: 0.8506\n", + "Epoch 327/330\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.1971 - accuracy: 0.9600 - val_loss: 0.2869 - val_accuracy: 0.9439 - lr: 0.0104 - momentum: 0.8688\n", + "Epoch 328/330\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.1913 - accuracy: 0.9512 - val_loss: 0.2939 - val_accuracy: 0.9471 - lr: 0.0059 - momentum: 0.9037\n", + "Epoch 329/330\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.1584 - accuracy: 0.9697 - val_loss: 0.2358 - val_accuracy: 0.9487 - lr: 0.0017 - momentum: 0.9367\n", + "Epoch 330/330\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.1532 - accuracy: 0.9702 - val_loss: 0.3036 - val_accuracy: 0.9391 - lr: 5.1360e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9391\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3036\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m320.31 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m277.67 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m42.64 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [55] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m56\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 330)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01278\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 331/336\n", + "256/256 [==============================] - 50s 180ms/step - loss: 0.2183 - accuracy: 0.9463 - val_loss: 0.3834 - val_accuracy: 0.9471 - lr: 0.0077 - momentum: 0.8915\n", + "Epoch 332/336\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2782 - accuracy: 0.9302 - val_loss: 0.2965 - val_accuracy: 0.9199 - lr: 0.0127 - momentum: 0.8506\n", + "Epoch 333/336\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2234 - accuracy: 0.9443 - val_loss: 0.3496 - val_accuracy: 0.9167 - lr: 0.0104 - momentum: 0.8688\n", + "Epoch 334/336\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2153 - accuracy: 0.9507 - val_loss: 0.2646 - val_accuracy: 0.9487 - lr: 0.0059 - momentum: 0.9037\n", + "Epoch 335/336\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1670 - accuracy: 0.9639 - val_loss: 0.2361 - val_accuracy: 0.9503 - lr: 0.0017 - momentum: 0.9367\n", + "Epoch 336/336\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.1308 - accuracy: 0.9727 - val_loss: 0.2959 - val_accuracy: 0.9487 - lr: 5.1120e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9487\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2959\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m321.70 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m278.40 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m43.29 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [56] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m57\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 336)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01272\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 337/342\n", + "256/256 [==============================] - 50s 179ms/step - loss: 0.1836 - accuracy: 0.9468 - val_loss: 0.3220 - val_accuracy: 0.9471 - lr: 0.0077 - momentum: 0.8915\n", + "Epoch 338/342\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1932 - accuracy: 0.9541 - val_loss: 0.4142 - val_accuracy: 0.9423 - lr: 0.0126 - momentum: 0.8506\n", + "Epoch 339/342\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2285 - accuracy: 0.9453 - val_loss: 0.3098 - val_accuracy: 0.9423 - lr: 0.0103 - momentum: 0.8688\n", + "Epoch 340/342\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2177 - accuracy: 0.9565 - val_loss: 0.2746 - val_accuracy: 0.9407 - lr: 0.0059 - momentum: 0.9037\n", + "Epoch 341/342\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1587 - accuracy: 0.9648 - val_loss: 0.2236 - val_accuracy: 0.9407 - lr: 0.0017 - momentum: 0.9367\n", + "Epoch 342/342\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.1290 - accuracy: 0.9688 - val_loss: 0.2223 - val_accuracy: 0.9471 - lr: 5.0880e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9471\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2223\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m321.11 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m277.05 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m44.06 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [57] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m58\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 342)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01266\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 343/348\n", + "256/256 [==============================] - 50s 181ms/step - loss: 0.2228 - accuracy: 0.9375 - val_loss: 0.3681 - val_accuracy: 0.9487 - lr: 0.0076 - momentum: 0.8915\n", + "Epoch 344/348\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2606 - accuracy: 0.9331 - val_loss: 0.2884 - val_accuracy: 0.9391 - lr: 0.0126 - momentum: 0.8506\n", + "Epoch 345/348\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2535 - accuracy: 0.9443 - val_loss: 0.3066 - val_accuracy: 0.9391 - lr: 0.0103 - momentum: 0.8688\n", + "Epoch 346/348\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2376 - accuracy: 0.9312 - val_loss: 0.4077 - val_accuracy: 0.9119 - lr: 0.0059 - momentum: 0.9037\n", + "Epoch 347/348\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1648 - accuracy: 0.9604 - val_loss: 0.3524 - val_accuracy: 0.9231 - lr: 0.0017 - momentum: 0.9367\n", + "Epoch 348/348\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.1279 - accuracy: 0.9692 - val_loss: 0.3673 - val_accuracy: 0.9215 - lr: 5.0640e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9215\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3673\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m321.88 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m278.26 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m43.63 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [58] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m59\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 348)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0126\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 349/354\n", + "256/256 [==============================] - 50s 181ms/step - loss: 0.2090 - accuracy: 0.9336 - val_loss: 0.2747 - val_accuracy: 0.9359 - lr: 0.0076 - momentum: 0.8915\n", + "Epoch 350/354\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2300 - accuracy: 0.9370 - val_loss: 0.3635 - val_accuracy: 0.8990 - lr: 0.0125 - momentum: 0.8506\n", + "Epoch 351/354\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.1793 - accuracy: 0.9521 - val_loss: 0.3843 - val_accuracy: 0.9311 - lr: 0.0102 - momentum: 0.8688\n", + "Epoch 352/354\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.1660 - accuracy: 0.9639 - val_loss: 0.4227 - val_accuracy: 0.9183 - lr: 0.0058 - momentum: 0.9037\n", + "Epoch 353/354\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.1396 - accuracy: 0.9717 - val_loss: 0.3311 - val_accuracy: 0.9311 - lr: 0.0017 - momentum: 0.9367\n", + "Epoch 354/354\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.0912 - accuracy: 0.9834 - val_loss: 0.4019 - val_accuracy: 0.9343 - lr: 5.0400e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9343\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.4019\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m322.12 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m278.08 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m44.05 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [59] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m60\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 354)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01254\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 355/360\n", + "256/256 [==============================] - 50s 180ms/step - loss: 0.2260 - accuracy: 0.9287 - val_loss: 0.3412 - val_accuracy: 0.9375 - lr: 0.0075 - momentum: 0.8915\n", + "Epoch 356/360\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2372 - accuracy: 0.9341 - val_loss: 0.2991 - val_accuracy: 0.9279 - lr: 0.0125 - momentum: 0.8506\n", + "Epoch 357/360\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.2529 - accuracy: 0.9365 - val_loss: 0.3348 - val_accuracy: 0.9327 - lr: 0.0102 - momentum: 0.8688\n", + "Epoch 358/360\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.2205 - accuracy: 0.9346 - val_loss: 0.6641 - val_accuracy: 0.9215 - lr: 0.0058 - momentum: 0.9037\n", + "Epoch 359/360\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1792 - accuracy: 0.9556 - val_loss: 0.3750 - val_accuracy: 0.9375 - lr: 0.0017 - momentum: 0.9367\n", + "Epoch 360/360\n", + "256/256 [==============================] - 45s 176ms/step - loss: 0.1322 - accuracy: 0.9692 - val_loss: 0.3344 - val_accuracy: 0.9359 - lr: 5.0160e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9359\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3344\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m321.70 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m277.08 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m44.62 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [60] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m61\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 360)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01248\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 361/366\n", + "256/256 [==============================] - 50s 181ms/step - loss: 0.2142 - accuracy: 0.9424 - val_loss: 0.3247 - val_accuracy: 0.9295 - lr: 0.0075 - momentum: 0.8915\n", + "Epoch 362/366\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.2195 - accuracy: 0.9370 - val_loss: 0.2452 - val_accuracy: 0.9295 - lr: 0.0124 - momentum: 0.8506\n", + "Epoch 363/366\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.2001 - accuracy: 0.9492 - val_loss: 0.2200 - val_accuracy: 0.9423 - lr: 0.0101 - momentum: 0.8688\n", + "Epoch 364/366\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.1986 - accuracy: 0.9551 - val_loss: 0.3045 - val_accuracy: 0.9407 - lr: 0.0058 - momentum: 0.9037\n", + "Epoch 365/366\n", + "256/256 [==============================] - 45s 177ms/step - loss: 0.1594 - accuracy: 0.9619 - val_loss: 0.2310 - val_accuracy: 0.9503 - lr: 0.0017 - momentum: 0.9367\n", + "Epoch 366/366\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.1611 - accuracy: 0.9683 - val_loss: 0.2259 - val_accuracy: 0.9407 - lr: 4.9920e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9407\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2259\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m323.29 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m278.45 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m44.84 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [61] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m62\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 366)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01242\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 367/372\n", + "256/256 [==============================] - 50s 181ms/step - loss: 0.1995 - accuracy: 0.9409 - val_loss: 0.2822 - val_accuracy: 0.9391 - lr: 0.0075 - momentum: 0.8915\n", + "Epoch 368/372\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2105 - accuracy: 0.9443 - val_loss: 0.3644 - val_accuracy: 0.9279 - lr: 0.0124 - momentum: 0.8506\n", + "Epoch 369/372\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2125 - accuracy: 0.9541 - val_loss: 0.4127 - val_accuracy: 0.9343 - lr: 0.0101 - momentum: 0.8688\n", + "Epoch 370/372\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.2211 - accuracy: 0.9404 - val_loss: 0.3342 - val_accuracy: 0.9295 - lr: 0.0057 - momentum: 0.9037\n", + "Epoch 371/372\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.1400 - accuracy: 0.9678 - val_loss: 0.3963 - val_accuracy: 0.9231 - lr: 0.0017 - momentum: 0.9367\n", + "Epoch 372/372\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1065 - accuracy: 0.9736 - val_loss: 0.4282 - val_accuracy: 0.9279 - lr: 4.9680e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9279\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.4283\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m323.93 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m279.19 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m44.75 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [62] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m63\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 372)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01236\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 373/378\n", + "256/256 [==============================] - 50s 180ms/step - loss: 0.1993 - accuracy: 0.9419 - val_loss: 0.6388 - val_accuracy: 0.8958 - lr: 0.0074 - momentum: 0.8915\n", + "Epoch 374/378\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1768 - accuracy: 0.9512 - val_loss: 0.3051 - val_accuracy: 0.9359 - lr: 0.0123 - momentum: 0.8506\n", + "Epoch 375/378\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1863 - accuracy: 0.9497 - val_loss: 0.5612 - val_accuracy: 0.8942 - lr: 0.0100 - momentum: 0.8688\n", + "Epoch 376/378\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1359 - accuracy: 0.9736 - val_loss: 0.4201 - val_accuracy: 0.9215 - lr: 0.0057 - momentum: 0.9037\n", + "Epoch 377/378\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.1530 - accuracy: 0.9727 - val_loss: 0.3494 - val_accuracy: 0.9359 - lr: 0.0016 - momentum: 0.9367\n", + "Epoch 378/378\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1128 - accuracy: 0.9800 - val_loss: 0.4005 - val_accuracy: 0.9279 - lr: 4.9440e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9279\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.4006\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m324.05 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m278.63 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m45.42 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [63] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m64\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 378)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;31m- Debug DP Sample dir: \u001b[0m\u001b[0;32mSamples/TSR_SUB_400_y2023_m12_d22-h03_m51_s22\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0123\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 379/384\n", + "256/256 [==============================] - 50s 181ms/step - loss: 0.2071 - accuracy: 0.9448 - val_loss: 0.2535 - val_accuracy: 0.9391 - lr: 0.0074 - momentum: 0.8915\n", + "Epoch 380/384\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2124 - accuracy: 0.9375 - val_loss: 0.2774 - val_accuracy: 0.9519 - lr: 0.0122 - momentum: 0.8506\n", + "Epoch 381/384\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1860 - accuracy: 0.9536 - val_loss: 0.3779 - val_accuracy: 0.9006 - lr: 0.0100 - momentum: 0.8688\n", + "Epoch 382/384\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2226 - accuracy: 0.9448 - val_loss: 0.3410 - val_accuracy: 0.9022 - lr: 0.0057 - momentum: 0.9037\n", + "Epoch 383/384\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.4106 - accuracy: 0.8672 - val_loss: 0.2579 - val_accuracy: 0.9119 - lr: 0.0016 - momentum: 0.9367\n", + "Epoch 384/384\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.2415 - accuracy: 0.9312 - val_loss: 0.2493 - val_accuracy: 0.9359 - lr: 4.9200e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9359\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2493\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m337.92 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m280.58 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m57.34 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [64] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m65\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 384)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01224\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 385/390\n", + "256/256 [==============================] - 50s 181ms/step - loss: 0.2728 - accuracy: 0.9106 - val_loss: 0.2036 - val_accuracy: 0.9439 - lr: 0.0074 - momentum: 0.8915\n", + "Epoch 386/390\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.2622 - accuracy: 0.9248 - val_loss: 0.3522 - val_accuracy: 0.8878 - lr: 0.0122 - momentum: 0.8506\n", + "Epoch 387/390\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2370 - accuracy: 0.9297 - val_loss: 0.2233 - val_accuracy: 0.9407 - lr: 0.0099 - momentum: 0.8688\n", + "Epoch 388/390\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2396 - accuracy: 0.9438 - val_loss: 0.2375 - val_accuracy: 0.9343 - lr: 0.0057 - momentum: 0.9037\n", + "Epoch 389/390\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1642 - accuracy: 0.9614 - val_loss: 0.2921 - val_accuracy: 0.9327 - lr: 0.0016 - momentum: 0.9367\n", + "Epoch 390/390\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.1278 - accuracy: 0.9722 - val_loss: 0.2668 - val_accuracy: 0.9375 - lr: 4.8960e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;31mERROR: Failed to load weights. Error: max() arg is an empty sequence\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9375\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2668\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m325.07 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m279.13 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m45.94 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [65] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m66\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 390)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01218\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 391/396\n", + "256/256 [==============================] - 51s 185ms/step - loss: 0.2335 - accuracy: 0.9302 - val_loss: 0.4710 - val_accuracy: 0.9022 - lr: 0.0073 - momentum: 0.8915\n", + "Epoch 392/396\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2452 - accuracy: 0.9243 - val_loss: 0.3078 - val_accuracy: 0.9327 - lr: 0.0121 - momentum: 0.8506\n", + "Epoch 393/396\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2054 - accuracy: 0.9448 - val_loss: 0.2711 - val_accuracy: 0.9263 - lr: 0.0099 - momentum: 0.8688\n", + "Epoch 394/396\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2032 - accuracy: 0.9512 - val_loss: 0.3266 - val_accuracy: 0.9279 - lr: 0.0056 - momentum: 0.9037\n", + "Epoch 395/396\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1553 - accuracy: 0.9575 - val_loss: 0.3865 - val_accuracy: 0.9311 - lr: 0.0016 - momentum: 0.9367\n", + "Epoch 396/396\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1363 - accuracy: 0.9634 - val_loss: 0.3360 - val_accuracy: 0.9327 - lr: 4.8720e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9327\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3360\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m327.52 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m281.07 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m46.44 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [66] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m67\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 396)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01212\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 397/402\n", + "256/256 [==============================] - 50s 181ms/step - loss: 0.2119 - accuracy: 0.9331 - val_loss: 0.3073 - val_accuracy: 0.9183 - lr: 0.0073 - momentum: 0.8915\n", + "Epoch 398/402\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2331 - accuracy: 0.9326 - val_loss: 0.4935 - val_accuracy: 0.8942 - lr: 0.0121 - momentum: 0.8506\n", + "Epoch 399/402\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2253 - accuracy: 0.9390 - val_loss: 0.3324 - val_accuracy: 0.9279 - lr: 0.0098 - momentum: 0.8688\n", + "Epoch 400/402\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1832 - accuracy: 0.9575 - val_loss: 0.3528 - val_accuracy: 0.9359 - lr: 0.0056 - momentum: 0.9037\n", + "Epoch 401/402\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1368 - accuracy: 0.9673 - val_loss: 0.4640 - val_accuracy: 0.9295 - lr: 0.0016 - momentum: 0.9367\n", + "Epoch 402/402\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1098 - accuracy: 0.9727 - val_loss: 0.4956 - val_accuracy: 0.9263 - lr: 4.8480e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9263\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.4957\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m327.68 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m280.88 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m46.79 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [67] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m68\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 402)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01206\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 403/408\n", + "256/256 [==============================] - 50s 182ms/step - loss: 0.1981 - accuracy: 0.9429 - val_loss: 0.4115 - val_accuracy: 0.9311 - lr: 0.0073 - momentum: 0.8915\n", + "Epoch 404/408\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.2037 - accuracy: 0.9409 - val_loss: 0.4208 - val_accuracy: 0.9327 - lr: 0.0120 - momentum: 0.8506\n", + "Epoch 405/408\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2167 - accuracy: 0.9478 - val_loss: 0.3391 - val_accuracy: 0.9263 - lr: 0.0098 - momentum: 0.8688\n", + "Epoch 406/408\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2090 - accuracy: 0.9482 - val_loss: 1.6039 - val_accuracy: 0.4167 - lr: 0.0056 - momentum: 0.9037\n", + "Epoch 407/408\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2090 - accuracy: 0.9385 - val_loss: 0.3508 - val_accuracy: 0.9279 - lr: 0.0016 - momentum: 0.9367\n", + "Epoch 408/408\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1647 - accuracy: 0.9629 - val_loss: 0.3060 - val_accuracy: 0.9279 - lr: 4.8240e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9279\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3060\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m325.31 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m279.16 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m46.15 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [68] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m69\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 408)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.012\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 409/414\n", + "256/256 [==============================] - 51s 185ms/step - loss: 0.2424 - accuracy: 0.9204 - val_loss: 0.2597 - val_accuracy: 0.9375 - lr: 0.0072 - momentum: 0.8915\n", + "Epoch 410/414\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.2372 - accuracy: 0.9287 - val_loss: 0.4092 - val_accuracy: 0.9054 - lr: 0.0119 - momentum: 0.8506\n", + "Epoch 411/414\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2232 - accuracy: 0.9429 - val_loss: 0.3071 - val_accuracy: 0.9375 - lr: 0.0097 - momentum: 0.8688\n", + "Epoch 412/414\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1619 - accuracy: 0.9561 - val_loss: 0.3597 - val_accuracy: 0.9263 - lr: 0.0056 - momentum: 0.9037\n", + "Epoch 413/414\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1335 - accuracy: 0.9707 - val_loss: 0.3930 - val_accuracy: 0.9022 - lr: 0.0016 - momentum: 0.9367\n", + "Epoch 414/414\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1109 - accuracy: 0.9751 - val_loss: 0.3899 - val_accuracy: 0.9215 - lr: 4.8000e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9215\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3899\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m326.10 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m280.09 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m46.01 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [69] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m70\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 414)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01194\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 415/420\n", + "256/256 [==============================] - 51s 185ms/step - loss: 0.1985 - accuracy: 0.9414 - val_loss: 0.3671 - val_accuracy: 0.9391 - lr: 0.0072 - momentum: 0.8915\n", + "Epoch 416/420\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.2515 - accuracy: 0.9468 - val_loss: 0.3416 - val_accuracy: 0.9439 - lr: 0.0119 - momentum: 0.8506\n", + "Epoch 417/420\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.1785 - accuracy: 0.9507 - val_loss: 0.3911 - val_accuracy: 0.9503 - lr: 0.0097 - momentum: 0.8688\n", + "Epoch 418/420\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1634 - accuracy: 0.9658 - val_loss: 0.2619 - val_accuracy: 0.9375 - lr: 0.0055 - momentum: 0.9037\n", + "Epoch 419/420\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1609 - accuracy: 0.9644 - val_loss: 0.2554 - val_accuracy: 0.9391 - lr: 0.0016 - momentum: 0.9367\n", + "Epoch 420/420\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.1210 - accuracy: 0.9775 - val_loss: 0.3009 - val_accuracy: 0.9423 - lr: 4.7760e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9423\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3009\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m328.70 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m282.09 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m46.61 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [70] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m71\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 420)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01188\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 421/426\n", + "256/256 [==============================] - 50s 183ms/step - loss: 0.1919 - accuracy: 0.9419 - val_loss: 0.2697 - val_accuracy: 0.9311 - lr: 0.0072 - momentum: 0.8915\n", + "Epoch 422/426\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2115 - accuracy: 0.9448 - val_loss: 0.5592 - val_accuracy: 0.9022 - lr: 0.0118 - momentum: 0.8506\n", + "Epoch 423/426\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2710 - accuracy: 0.9258 - val_loss: 0.3152 - val_accuracy: 0.9167 - lr: 0.0096 - momentum: 0.8688\n", + "Epoch 424/426\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1848 - accuracy: 0.9531 - val_loss: 0.2302 - val_accuracy: 0.9439 - lr: 0.0055 - momentum: 0.9037\n", + "Epoch 425/426\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1284 - accuracy: 0.9697 - val_loss: 0.2284 - val_accuracy: 0.9343 - lr: 0.0016 - momentum: 0.9367\n", + "Epoch 426/426\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1220 - accuracy: 0.9702 - val_loss: 0.2743 - val_accuracy: 0.9439 - lr: 4.7520e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9439\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2743\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m327.68 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m279.77 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m47.91 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [71] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m72\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 426)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33m└───Shuffling data...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01182\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 427/432\n", + "256/256 [==============================] - 51s 183ms/step - loss: 0.2220 - accuracy: 0.9268 - val_loss: 0.2893 - val_accuracy: 0.9311 - lr: 0.0071 - momentum: 0.8915\n", + "Epoch 428/432\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1984 - accuracy: 0.9468 - val_loss: 0.2627 - val_accuracy: 0.9391 - lr: 0.0118 - momentum: 0.8506\n", + "Epoch 429/432\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1814 - accuracy: 0.9561 - val_loss: 0.2701 - val_accuracy: 0.9247 - lr: 0.0096 - momentum: 0.8688\n", + "Epoch 430/432\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1460 - accuracy: 0.9683 - val_loss: 0.2917 - val_accuracy: 0.9215 - lr: 0.0055 - momentum: 0.9037\n", + "Epoch 431/432\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1252 - accuracy: 0.9688 - val_loss: 0.2932 - val_accuracy: 0.9279 - lr: 0.0016 - momentum: 0.9367\n", + "Epoch 432/432\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1040 - accuracy: 0.9785 - val_loss: 0.2979 - val_accuracy: 0.9343 - lr: 4.7280e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9343\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2979\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m330.52 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m280.69 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m49.83 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [72] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m73\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 432)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01176\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 433/438\n", + "256/256 [==============================] - 50s 183ms/step - loss: 0.2199 - accuracy: 0.9404 - val_loss: 0.3522 - val_accuracy: 0.9295 - lr: 0.0071 - momentum: 0.8915\n", + "Epoch 434/438\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1987 - accuracy: 0.9419 - val_loss: 0.2896 - val_accuracy: 0.9359 - lr: 0.0117 - momentum: 0.8506\n", + "Epoch 435/438\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2374 - accuracy: 0.9331 - val_loss: 0.3696 - val_accuracy: 0.8942 - lr: 0.0095 - momentum: 0.8688\n", + "Epoch 436/438\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2211 - accuracy: 0.9424 - val_loss: 0.2876 - val_accuracy: 0.9327 - lr: 0.0054 - momentum: 0.9037\n", + "Epoch 437/438\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1664 - accuracy: 0.9590 - val_loss: 0.3804 - val_accuracy: 0.9135 - lr: 0.0016 - momentum: 0.9367\n", + "Epoch 438/438\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1248 - accuracy: 0.9683 - val_loss: 0.4128 - val_accuracy: 0.9151 - lr: 4.7040e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9151\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.4128\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m327.69 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m280.16 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m47.53 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [73] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m74\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 438)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0117\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 439/444\n", + "256/256 [==============================] - 50s 183ms/step - loss: 0.1985 - accuracy: 0.9458 - val_loss: 0.2751 - val_accuracy: 0.9391 - lr: 0.0070 - momentum: 0.8915\n", + "Epoch 440/444\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2099 - accuracy: 0.9419 - val_loss: 0.3547 - val_accuracy: 0.9135 - lr: 0.0116 - momentum: 0.8506\n", + "Epoch 441/444\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1777 - accuracy: 0.9487 - val_loss: 0.4030 - val_accuracy: 0.9119 - lr: 0.0095 - momentum: 0.8688\n", + "Epoch 442/444\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2029 - accuracy: 0.9619 - val_loss: 0.3166 - val_accuracy: 0.9343 - lr: 0.0054 - momentum: 0.9037\n", + "Epoch 443/444\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1744 - accuracy: 0.9619 - val_loss: 0.3691 - val_accuracy: 0.9311 - lr: 0.0016 - momentum: 0.9367\n", + "Epoch 444/444\n", + "256/256 [==============================] - 46s 177ms/step - loss: 0.1244 - accuracy: 0.9746 - val_loss: 0.4260 - val_accuracy: 0.9279 - lr: 4.6800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9279\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.4260\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m328.49 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m280.19 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m48.30 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [74] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m75\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 444)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01164\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 445/450\n", + "256/256 [==============================] - 50s 182ms/step - loss: 0.1918 - accuracy: 0.9497 - val_loss: 0.2884 - val_accuracy: 0.9375 - lr: 0.0070 - momentum: 0.8915\n", + "Epoch 446/450\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1896 - accuracy: 0.9512 - val_loss: 0.3865 - val_accuracy: 0.9295 - lr: 0.0116 - momentum: 0.8506\n", + "Epoch 447/450\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1949 - accuracy: 0.9526 - val_loss: 0.3544 - val_accuracy: 0.9038 - lr: 0.0094 - momentum: 0.8688\n", + "Epoch 448/450\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1458 - accuracy: 0.9697 - val_loss: 0.3064 - val_accuracy: 0.9311 - lr: 0.0054 - momentum: 0.9037\n", + "Epoch 449/450\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1634 - accuracy: 0.9604 - val_loss: 0.2541 - val_accuracy: 0.9375 - lr: 0.0016 - momentum: 0.9367\n", + "Epoch 450/450\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1089 - accuracy: 0.9751 - val_loss: 0.2729 - val_accuracy: 0.9247 - lr: 4.6560e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9247\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2729\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m328.95 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m280.78 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m48.17 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [75] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m76\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 450)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01158\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 451/456\n", + "256/256 [==============================] - 50s 183ms/step - loss: 0.2100 - accuracy: 0.9326 - val_loss: 0.3282 - val_accuracy: 0.9215 - lr: 0.0070 - momentum: 0.8915\n", + "Epoch 452/456\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2066 - accuracy: 0.9404 - val_loss: 0.3057 - val_accuracy: 0.9279 - lr: 0.0115 - momentum: 0.8506\n", + "Epoch 453/456\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2336 - accuracy: 0.9370 - val_loss: 0.6003 - val_accuracy: 0.8910 - lr: 0.0094 - momentum: 0.8688\n", + "Epoch 454/456\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2006 - accuracy: 0.9536 - val_loss: 0.2152 - val_accuracy: 0.9311 - lr: 0.0054 - momentum: 0.9037\n", + "Epoch 455/456\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1549 - accuracy: 0.9683 - val_loss: 0.2208 - val_accuracy: 0.9327 - lr: 0.0015 - momentum: 0.9367\n", + "Epoch 456/456\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1337 - accuracy: 0.9639 - val_loss: 0.2067 - val_accuracy: 0.9471 - lr: 4.6320e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9471\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2066\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.20500072836875916. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m329.90 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m281.11 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m48.79 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [76] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m77\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 456)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01152\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 457/462\n", + "256/256 [==============================] - 51s 183ms/step - loss: 0.2102 - accuracy: 0.9434 - val_loss: 0.2762 - val_accuracy: 0.9263 - lr: 0.0069 - momentum: 0.8915\n", + "Epoch 458/462\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2329 - accuracy: 0.9370 - val_loss: 0.3336 - val_accuracy: 0.9247 - lr: 0.0115 - momentum: 0.8506\n", + "Epoch 459/462\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2184 - accuracy: 0.9478 - val_loss: 0.3217 - val_accuracy: 0.9391 - lr: 0.0094 - momentum: 0.8688\n", + "Epoch 460/462\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.2258 - accuracy: 0.9360 - val_loss: 0.2642 - val_accuracy: 0.9359 - lr: 0.0053 - momentum: 0.9037\n", + "Epoch 461/462\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1569 - accuracy: 0.9600 - val_loss: 0.1852 - val_accuracy: 0.9519 - lr: 0.0015 - momentum: 0.9367\n", + "Epoch 462/462\n", + "256/256 [==============================] - 46s 178ms/step - loss: 0.1207 - accuracy: 0.9722 - val_loss: 0.1820 - val_accuracy: 0.9503 - lr: 4.6080e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-461-0.9519.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9519\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1852\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model loss from \u001b[0m\u001b[0;32m0.20500072836875916 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.1852283775806427\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m332.46 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m281.08 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m51.38 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [77] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m78\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 462)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01146\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 463/468\n", + "256/256 [==============================] - 51s 186ms/step - loss: 0.2278 - accuracy: 0.9287 - val_loss: 0.1914 - val_accuracy: 0.9455 - lr: 0.0069 - momentum: 0.8915\n", + "Epoch 464/468\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2232 - accuracy: 0.9263 - val_loss: 0.2165 - val_accuracy: 0.9247 - lr: 0.0114 - momentum: 0.8506\n", + "Epoch 465/468\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1997 - accuracy: 0.9438 - val_loss: 0.2093 - val_accuracy: 0.9343 - lr: 0.0093 - momentum: 0.8688\n", + "Epoch 466/468\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1686 - accuracy: 0.9585 - val_loss: 0.2183 - val_accuracy: 0.9471 - lr: 0.0053 - momentum: 0.9037\n", + "Epoch 467/468\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.1469 - accuracy: 0.9624 - val_loss: 0.1976 - val_accuracy: 0.9423 - lr: 0.0015 - momentum: 0.9367\n", + "Epoch 468/468\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1212 - accuracy: 0.9663 - val_loss: 0.1866 - val_accuracy: 0.9455 - lr: 4.5840e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9455\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1866\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1852283775806427. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m334.31 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m283.91 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m50.40 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [78] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m79\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 468)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0114\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 469/474\n", + "256/256 [==============================] - 51s 183ms/step - loss: 0.1757 - accuracy: 0.9502 - val_loss: 0.2176 - val_accuracy: 0.9455 - lr: 0.0069 - momentum: 0.8915\n", + "Epoch 470/474\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1973 - accuracy: 0.9526 - val_loss: 0.3784 - val_accuracy: 0.9391 - lr: 0.0113 - momentum: 0.8506\n", + "Epoch 471/474\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1922 - accuracy: 0.9507 - val_loss: 0.1918 - val_accuracy: 0.9503 - lr: 0.0093 - momentum: 0.8688\n", + "Epoch 472/474\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2003 - accuracy: 0.9595 - val_loss: 0.2401 - val_accuracy: 0.9503 - lr: 0.0053 - momentum: 0.9037\n", + "Epoch 473/474\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1738 - accuracy: 0.9609 - val_loss: 0.1883 - val_accuracy: 0.9471 - lr: 0.0015 - momentum: 0.9367\n", + "Epoch 474/474\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1348 - accuracy: 0.9717 - val_loss: 0.1817 - val_accuracy: 0.9471 - lr: 4.5600e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-471-0.9503.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9503\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1918\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9567307829856873. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1852283775806427. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m333.90 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m282.81 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m51.10 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [79] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m80\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 474)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01134\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 475/480\n", + "256/256 [==============================] - 51s 186ms/step - loss: 0.2342 - accuracy: 0.9380 - val_loss: 0.2145 - val_accuracy: 0.9503 - lr: 0.0068 - momentum: 0.8915\n", + "Epoch 476/480\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2019 - accuracy: 0.9473 - val_loss: 0.2357 - val_accuracy: 0.9295 - lr: 0.0113 - momentum: 0.8506\n", + "Epoch 477/480\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1574 - accuracy: 0.9604 - val_loss: 0.2965 - val_accuracy: 0.9423 - lr: 0.0092 - momentum: 0.8688\n", + "Epoch 478/480\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1408 - accuracy: 0.9653 - val_loss: 0.1871 - val_accuracy: 0.9599 - lr: 0.0052 - momentum: 0.9037\n", + "Epoch 479/480\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1129 - accuracy: 0.9771 - val_loss: 0.2315 - val_accuracy: 0.9439 - lr: 0.0015 - momentum: 0.9367\n", + "Epoch 480/480\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1149 - accuracy: 0.9790 - val_loss: 0.1957 - val_accuracy: 0.9455 - lr: 4.5360e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-478-0.9599.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9599\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1871\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model accuracy from \u001b[0m\u001b[0;32m0.9567307829856873 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.9599359035491943\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1852283775806427. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m336.16 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m283.56 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m52.61 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [80] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m81\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 480)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01128\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 481/486\n", + "256/256 [==============================] - 52s 186ms/step - loss: 0.1831 - accuracy: 0.9502 - val_loss: 0.1883 - val_accuracy: 0.9487 - lr: 0.0068 - momentum: 0.8915\n", + "Epoch 482/486\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.2238 - accuracy: 0.9370 - val_loss: 0.1762 - val_accuracy: 0.9567 - lr: 0.0112 - momentum: 0.8506\n", + "Epoch 483/486\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1968 - accuracy: 0.9546 - val_loss: 0.2420 - val_accuracy: 0.9375 - lr: 0.0092 - momentum: 0.8688\n", + "Epoch 484/486\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1566 - accuracy: 0.9639 - val_loss: 0.2655 - val_accuracy: 0.9263 - lr: 0.0052 - momentum: 0.9037\n", + "Epoch 485/486\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1233 - accuracy: 0.9722 - val_loss: 0.3470 - val_accuracy: 0.9071 - lr: 0.0015 - momentum: 0.9367\n", + "Epoch 486/486\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.0971 - accuracy: 0.9810 - val_loss: 0.3051 - val_accuracy: 0.9295 - lr: 4.5120e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-482-0.9567.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9567\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1762\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model loss from \u001b[0m\u001b[0;32m0.1852283775806427 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.17617985606193542\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m338.26 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m283.66 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m54.60 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [81] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m82\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 486)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01122\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 487/492\n", + "256/256 [==============================] - 51s 186ms/step - loss: 0.2008 - accuracy: 0.9443 - val_loss: 0.1760 - val_accuracy: 0.9519 - lr: 0.0068 - momentum: 0.8915\n", + "Epoch 488/492\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1629 - accuracy: 0.9619 - val_loss: 0.2700 - val_accuracy: 0.9327 - lr: 0.0112 - momentum: 0.8506\n", + "Epoch 489/492\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1441 - accuracy: 0.9648 - val_loss: 0.2800 - val_accuracy: 0.9327 - lr: 0.0091 - momentum: 0.8688\n", + "Epoch 490/492\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1798 - accuracy: 0.9600 - val_loss: 0.5160 - val_accuracy: 0.9135 - lr: 0.0052 - momentum: 0.9037\n", + "Epoch 491/492\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1522 - accuracy: 0.9668 - val_loss: 0.3251 - val_accuracy: 0.9295 - lr: 0.0015 - momentum: 0.9367\n", + "Epoch 492/492\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1104 - accuracy: 0.9761 - val_loss: 0.3280 - val_accuracy: 0.9279 - lr: 4.4880e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-487-0.9519.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9519\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1760\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model loss from \u001b[0m\u001b[0;32m0.17617985606193542 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.17603926360607147\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m337.12 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m282.51 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m54.60 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [82] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m83\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 492)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01116\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 493/498\n", + "256/256 [==============================] - 51s 187ms/step - loss: 0.2007 - accuracy: 0.9419 - val_loss: 0.2081 - val_accuracy: 0.9327 - lr: 0.0067 - momentum: 0.8915\n", + "Epoch 494/498\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2041 - accuracy: 0.9438 - val_loss: 0.2344 - val_accuracy: 0.9295 - lr: 0.0111 - momentum: 0.8506\n", + "Epoch 495/498\n", + "256/256 [==============================] - 47s 183ms/step - loss: 0.1820 - accuracy: 0.9580 - val_loss: 0.1937 - val_accuracy: 0.9503 - lr: 0.0091 - momentum: 0.8688\n", + "Epoch 496/498\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1421 - accuracy: 0.9673 - val_loss: 0.2166 - val_accuracy: 0.9439 - lr: 0.0052 - momentum: 0.9037\n", + "Epoch 497/498\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1428 - accuracy: 0.9653 - val_loss: 0.1948 - val_accuracy: 0.9455 - lr: 0.0015 - momentum: 0.9367\n", + "Epoch 498/498\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1332 - accuracy: 0.9702 - val_loss: 0.2214 - val_accuracy: 0.9439 - lr: 4.4640e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9439\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2214\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m336.41 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m283.57 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m52.84 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [83] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m84\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 498)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0111\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 499/504\n", + "256/256 [==============================] - 51s 184ms/step - loss: 0.2168 - accuracy: 0.9409 - val_loss: 0.2371 - val_accuracy: 0.9423 - lr: 0.0067 - momentum: 0.8915\n", + "Epoch 500/504\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2798 - accuracy: 0.9038 - val_loss: 1.5811 - val_accuracy: 0.3862 - lr: 0.0110 - momentum: 0.8506\n", + "Epoch 501/504\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.4885 - accuracy: 0.8281 - val_loss: 0.3287 - val_accuracy: 0.9071 - lr: 0.0090 - momentum: 0.8688\n", + "Epoch 502/504\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.3234 - accuracy: 0.9023 - val_loss: 0.4095 - val_accuracy: 0.8622 - lr: 0.0051 - momentum: 0.9037\n", + "Epoch 503/504\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2540 - accuracy: 0.9277 - val_loss: 0.2722 - val_accuracy: 0.9343 - lr: 0.0015 - momentum: 0.9367\n", + "Epoch 504/504\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2126 - accuracy: 0.9404 - val_loss: 0.2674 - val_accuracy: 0.9279 - lr: 4.4400e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9279\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2674\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m333.93 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m281.70 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m52.23 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [84] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m85\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 504)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01104\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 505/510\n", + "256/256 [==============================] - 51s 184ms/step - loss: 0.2841 - accuracy: 0.9102 - val_loss: 0.2492 - val_accuracy: 0.9295 - lr: 0.0066 - momentum: 0.8915\n", + "Epoch 506/510\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2734 - accuracy: 0.9165 - val_loss: 0.2569 - val_accuracy: 0.9231 - lr: 0.0110 - momentum: 0.8506\n", + "Epoch 507/510\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2439 - accuracy: 0.9253 - val_loss: 0.3360 - val_accuracy: 0.8894 - lr: 0.0090 - momentum: 0.8688\n", + "Epoch 508/510\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2390 - accuracy: 0.9292 - val_loss: 0.2527 - val_accuracy: 0.9151 - lr: 0.0051 - momentum: 0.9037\n", + "Epoch 509/510\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1864 - accuracy: 0.9482 - val_loss: 0.3553 - val_accuracy: 0.8878 - lr: 0.0015 - momentum: 0.9367\n", + "Epoch 510/510\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1744 - accuracy: 0.9438 - val_loss: 0.3095 - val_accuracy: 0.8958 - lr: 4.4160e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.8958\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3095\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m334.09 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m282.77 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m51.33 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [85] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m86\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 510)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01098\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 511/516\n", + "256/256 [==============================] - 51s 184ms/step - loss: 0.2299 - accuracy: 0.9194 - val_loss: 0.3775 - val_accuracy: 0.8942 - lr: 0.0066 - momentum: 0.8915\n", + "Epoch 512/516\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2336 - accuracy: 0.9224 - val_loss: 0.4209 - val_accuracy: 0.8766 - lr: 0.0109 - momentum: 0.8506\n", + "Epoch 513/516\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2213 - accuracy: 0.9326 - val_loss: 0.3701 - val_accuracy: 0.8910 - lr: 0.0089 - momentum: 0.8688\n", + "Epoch 514/516\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2196 - accuracy: 0.9316 - val_loss: 0.3015 - val_accuracy: 0.9119 - lr: 0.0051 - momentum: 0.9037\n", + "Epoch 515/516\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1547 - accuracy: 0.9551 - val_loss: 0.4705 - val_accuracy: 0.8958 - lr: 0.0015 - momentum: 0.9367\n", + "Epoch 516/516\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1172 - accuracy: 0.9673 - val_loss: 0.5644 - val_accuracy: 0.8862 - lr: 4.3920e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.8862\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.5644\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m333.32 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m282.27 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m51.05 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [86] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m87\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 516)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01092\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 517/522\n", + "256/256 [==============================] - 51s 184ms/step - loss: 0.2431 - accuracy: 0.9302 - val_loss: 0.4163 - val_accuracy: 0.8974 - lr: 0.0066 - momentum: 0.8915\n", + "Epoch 518/522\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2538 - accuracy: 0.9268 - val_loss: 0.3237 - val_accuracy: 0.9231 - lr: 0.0109 - momentum: 0.8506\n", + "Epoch 519/522\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2172 - accuracy: 0.9468 - val_loss: 0.5438 - val_accuracy: 0.8590 - lr: 0.0089 - momentum: 0.8688\n", + "Epoch 520/522\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2061 - accuracy: 0.9468 - val_loss: 0.2792 - val_accuracy: 0.9103 - lr: 0.0051 - momentum: 0.9037\n", + "Epoch 521/522\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1580 - accuracy: 0.9570 - val_loss: 0.2834 - val_accuracy: 0.9119 - lr: 0.0015 - momentum: 0.9367\n", + "Epoch 522/522\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1259 - accuracy: 0.9702 - val_loss: 0.3188 - val_accuracy: 0.9119 - lr: 4.3680e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9119\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3188\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m334.25 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m282.96 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m51.29 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [87] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m88\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 522)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01086\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 523/528\n", + "256/256 [==============================] - 51s 184ms/step - loss: 0.1763 - accuracy: 0.9492 - val_loss: 0.2541 - val_accuracy: 0.9295 - lr: 0.0065 - momentum: 0.8915\n", + "Epoch 524/528\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2082 - accuracy: 0.9346 - val_loss: 0.2674 - val_accuracy: 0.9151 - lr: 0.0108 - momentum: 0.8506\n", + "Epoch 525/528\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1956 - accuracy: 0.9502 - val_loss: 0.2429 - val_accuracy: 0.9375 - lr: 0.0088 - momentum: 0.8688\n", + "Epoch 526/528\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1450 - accuracy: 0.9629 - val_loss: 0.3454 - val_accuracy: 0.9231 - lr: 0.0050 - momentum: 0.9037\n", + "Epoch 527/528\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1203 - accuracy: 0.9775 - val_loss: 0.3013 - val_accuracy: 0.9167 - lr: 0.0014 - momentum: 0.9367\n", + "Epoch 528/528\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.0980 - accuracy: 0.9795 - val_loss: 0.2988 - val_accuracy: 0.9199 - lr: 4.3440e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9199\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2988\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m334.91 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m282.87 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m52.04 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [88] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m89\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 528)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0108\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 529/534\n", + "256/256 [==============================] - 51s 184ms/step - loss: 0.2229 - accuracy: 0.9336 - val_loss: 0.2800 - val_accuracy: 0.9375 - lr: 0.0065 - momentum: 0.8915\n", + "Epoch 530/534\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2037 - accuracy: 0.9399 - val_loss: 0.2988 - val_accuracy: 0.9343 - lr: 0.0107 - momentum: 0.8506\n", + "Epoch 531/534\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1945 - accuracy: 0.9443 - val_loss: 0.3587 - val_accuracy: 0.9038 - lr: 0.0088 - momentum: 0.8688\n", + "Epoch 532/534\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1608 - accuracy: 0.9604 - val_loss: 0.2243 - val_accuracy: 0.9407 - lr: 0.0050 - momentum: 0.9037\n", + "Epoch 533/534\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1717 - accuracy: 0.9551 - val_loss: 0.3388 - val_accuracy: 0.9167 - lr: 0.0014 - momentum: 0.9367\n", + "Epoch 534/534\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1270 - accuracy: 0.9658 - val_loss: 0.2928 - val_accuracy: 0.9279 - lr: 4.3200e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9279\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2928\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m335.29 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m283.12 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m52.17 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [89] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m90\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 534)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33m└───Shuffling data...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01074\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 535/540\n", + "256/256 [==============================] - 51s 184ms/step - loss: 0.2553 - accuracy: 0.9204 - val_loss: 0.2754 - val_accuracy: 0.9038 - lr: 0.0065 - momentum: 0.8915\n", + "Epoch 536/540\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2352 - accuracy: 0.9268 - val_loss: 0.2348 - val_accuracy: 0.9311 - lr: 0.0107 - momentum: 0.8506\n", + "Epoch 537/540\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2315 - accuracy: 0.9336 - val_loss: 0.2594 - val_accuracy: 0.9167 - lr: 0.0087 - momentum: 0.8688\n", + "Epoch 538/540\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1771 - accuracy: 0.9561 - val_loss: 0.2493 - val_accuracy: 0.9311 - lr: 0.0050 - momentum: 0.9037\n", + "Epoch 539/540\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1767 - accuracy: 0.9531 - val_loss: 0.3012 - val_accuracy: 0.9103 - lr: 0.0014 - momentum: 0.9367\n", + "Epoch 540/540\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1363 - accuracy: 0.9658 - val_loss: 0.3091 - val_accuracy: 0.9231 - lr: 4.2960e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9231\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3091\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m338.08 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m282.91 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m55.17 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [90] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m91\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 540)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01068\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 541/546\n", + "256/256 [==============================] - 51s 184ms/step - loss: 0.2324 - accuracy: 0.9355 - val_loss: 0.2321 - val_accuracy: 0.9311 - lr: 0.0064 - momentum: 0.8915\n", + "Epoch 542/546\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2370 - accuracy: 0.9365 - val_loss: 0.2630 - val_accuracy: 0.9247 - lr: 0.0106 - momentum: 0.8506\n", + "Epoch 543/546\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2311 - accuracy: 0.9336 - val_loss: 0.3506 - val_accuracy: 0.9167 - lr: 0.0087 - momentum: 0.8688\n", + "Epoch 544/546\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2093 - accuracy: 0.9463 - val_loss: 0.2880 - val_accuracy: 0.9311 - lr: 0.0049 - momentum: 0.9037\n", + "Epoch 545/546\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1566 - accuracy: 0.9629 - val_loss: 0.2458 - val_accuracy: 0.9263 - lr: 0.0014 - momentum: 0.9367\n", + "Epoch 546/546\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1261 - accuracy: 0.9678 - val_loss: 0.2360 - val_accuracy: 0.9263 - lr: 4.2720e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9263\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2360\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m334.53 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m282.56 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m51.97 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [91] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m92\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 546)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01062\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 547/552\n", + "256/256 [==============================] - 51s 183ms/step - loss: 0.1908 - accuracy: 0.9429 - val_loss: 0.2946 - val_accuracy: 0.9311 - lr: 0.0064 - momentum: 0.8915\n", + "Epoch 548/552\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2102 - accuracy: 0.9370 - val_loss: 0.2310 - val_accuracy: 0.9199 - lr: 0.0106 - momentum: 0.8506\n", + "Epoch 549/552\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1688 - accuracy: 0.9536 - val_loss: 0.2412 - val_accuracy: 0.9247 - lr: 0.0086 - momentum: 0.8688\n", + "Epoch 550/552\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1506 - accuracy: 0.9614 - val_loss: 0.2306 - val_accuracy: 0.9279 - lr: 0.0049 - momentum: 0.9037\n", + "Epoch 551/552\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2527 - accuracy: 0.9248 - val_loss: 0.2805 - val_accuracy: 0.9135 - lr: 0.0014 - momentum: 0.9367\n", + "Epoch 552/552\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2524 - accuracy: 0.9219 - val_loss: 0.2080 - val_accuracy: 0.9439 - lr: 4.2480e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9439\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2080\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m335.50 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m281.77 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m53.73 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [92] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m93\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 552)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01056\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 553/558\n", + "256/256 [==============================] - 51s 183ms/step - loss: 0.2802 - accuracy: 0.9033 - val_loss: 0.3070 - val_accuracy: 0.8702 - lr: 0.0064 - momentum: 0.8915\n", + "Epoch 554/558\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2819 - accuracy: 0.9077 - val_loss: 0.3020 - val_accuracy: 0.9022 - lr: 0.0105 - momentum: 0.8506\n", + "Epoch 555/558\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.2129 - accuracy: 0.9341 - val_loss: 0.2712 - val_accuracy: 0.9087 - lr: 0.0086 - momentum: 0.8688\n", + "Epoch 556/558\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1881 - accuracy: 0.9492 - val_loss: 0.2646 - val_accuracy: 0.9231 - lr: 0.0049 - momentum: 0.9037\n", + "Epoch 557/558\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1644 - accuracy: 0.9575 - val_loss: 0.2279 - val_accuracy: 0.9247 - lr: 0.0014 - momentum: 0.9367\n", + "Epoch 558/558\n", + "256/256 [==============================] - 46s 179ms/step - loss: 0.1322 - accuracy: 0.9648 - val_loss: 0.2427 - val_accuracy: 0.9247 - lr: 4.2240e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9247\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2427\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m335.69 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m282.18 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m53.52 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [93] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m94\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 558)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0105\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 559/564\n", + "256/256 [==============================] - 51s 184ms/step - loss: 0.2274 - accuracy: 0.9258 - val_loss: 0.2463 - val_accuracy: 0.9247 - lr: 0.0063 - momentum: 0.8915\n", + "Epoch 560/564\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2199 - accuracy: 0.9316 - val_loss: 0.3196 - val_accuracy: 0.9231 - lr: 0.0104 - momentum: 0.8506\n", + "Epoch 561/564\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1923 - accuracy: 0.9512 - val_loss: 0.3223 - val_accuracy: 0.9263 - lr: 0.0085 - momentum: 0.8688\n", + "Epoch 562/564\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1664 - accuracy: 0.9526 - val_loss: 0.3450 - val_accuracy: 0.9295 - lr: 0.0049 - momentum: 0.9037\n", + "Epoch 563/564\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1356 - accuracy: 0.9688 - val_loss: 0.3262 - val_accuracy: 0.9215 - lr: 0.0014 - momentum: 0.9367\n", + "Epoch 564/564\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1128 - accuracy: 0.9707 - val_loss: 0.4118 - val_accuracy: 0.9167 - lr: 4.2000e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9167\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.4117\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m336.89 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m282.83 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m54.06 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [94] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m95\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 564)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01044\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 565/570\n", + "256/256 [==============================] - 51s 184ms/step - loss: 0.1628 - accuracy: 0.9497 - val_loss: 0.3142 - val_accuracy: 0.9215 - lr: 0.0063 - momentum: 0.8915\n", + "Epoch 566/570\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1794 - accuracy: 0.9512 - val_loss: 0.2638 - val_accuracy: 0.9199 - lr: 0.0104 - momentum: 0.8506\n", + "Epoch 567/570\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1562 - accuracy: 0.9600 - val_loss: 0.2997 - val_accuracy: 0.9183 - lr: 0.0085 - momentum: 0.8688\n", + "Epoch 568/570\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1372 - accuracy: 0.9653 - val_loss: 0.3107 - val_accuracy: 0.9103 - lr: 0.0048 - momentum: 0.9037\n", + "Epoch 569/570\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1054 - accuracy: 0.9741 - val_loss: 0.3363 - val_accuracy: 0.9247 - lr: 0.0014 - momentum: 0.9367\n", + "Epoch 570/570\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.0834 - accuracy: 0.9814 - val_loss: 0.3877 - val_accuracy: 0.9215 - lr: 4.1760e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9215\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3878\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m335.70 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m282.71 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m53.00 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [95] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m96\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 570)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01038\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 571/576\n", + "256/256 [==============================] - 51s 185ms/step - loss: 0.1892 - accuracy: 0.9360 - val_loss: 0.3353 - val_accuracy: 0.9263 - lr: 0.0062 - momentum: 0.8915\n", + "Epoch 572/576\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2201 - accuracy: 0.9375 - val_loss: 0.4019 - val_accuracy: 0.9263 - lr: 0.0103 - momentum: 0.8506\n", + "Epoch 573/576\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.2044 - accuracy: 0.9517 - val_loss: 0.4528 - val_accuracy: 0.9103 - lr: 0.0084 - momentum: 0.8688\n", + "Epoch 574/576\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1589 - accuracy: 0.9634 - val_loss: 0.3456 - val_accuracy: 0.9167 - lr: 0.0048 - momentum: 0.9037\n", + "Epoch 575/576\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1979 - accuracy: 0.9438 - val_loss: 0.2332 - val_accuracy: 0.9247 - lr: 0.0014 - momentum: 0.9367\n", + "Epoch 576/576\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.2101 - accuracy: 0.9341 - val_loss: 0.2202 - val_accuracy: 0.9327 - lr: 4.1520e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9327\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2202\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m337.49 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m283.96 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m53.53 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [96] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m97\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 576)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01032\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 577/582\n", + "256/256 [==============================] - 51s 185ms/step - loss: 0.2547 - accuracy: 0.9233 - val_loss: 0.2767 - val_accuracy: 0.9295 - lr: 0.0062 - momentum: 0.8915\n", + "Epoch 578/582\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.2499 - accuracy: 0.9272 - val_loss: 0.2187 - val_accuracy: 0.9279 - lr: 0.0103 - momentum: 0.8506\n", + "Epoch 579/582\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2153 - accuracy: 0.9429 - val_loss: 0.2452 - val_accuracy: 0.9359 - lr: 0.0084 - momentum: 0.8688\n", + "Epoch 580/582\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1696 - accuracy: 0.9551 - val_loss: 0.4149 - val_accuracy: 0.9167 - lr: 0.0048 - momentum: 0.9037\n", + "Epoch 581/582\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1423 - accuracy: 0.9639 - val_loss: 0.3355 - val_accuracy: 0.9279 - lr: 0.0014 - momentum: 0.9367\n", + "Epoch 582/582\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.0996 - accuracy: 0.9785 - val_loss: 0.3712 - val_accuracy: 0.9215 - lr: 4.1280e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9215\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3712\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m337.11 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m283.64 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m53.47 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [97] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m98\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 582)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01026\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 583/588\n", + "256/256 [==============================] - 51s 185ms/step - loss: 0.2201 - accuracy: 0.9385 - val_loss: 0.2735 - val_accuracy: 0.9151 - lr: 0.0062 - momentum: 0.8915\n", + "Epoch 584/588\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2809 - accuracy: 0.9204 - val_loss: 0.2670 - val_accuracy: 0.9199 - lr: 0.0102 - momentum: 0.8506\n", + "Epoch 585/588\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1999 - accuracy: 0.9458 - val_loss: 0.2282 - val_accuracy: 0.9279 - lr: 0.0083 - momentum: 0.8688\n", + "Epoch 586/588\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.1734 - accuracy: 0.9565 - val_loss: 0.2593 - val_accuracy: 0.9311 - lr: 0.0047 - momentum: 0.9037\n", + "Epoch 587/588\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1565 - accuracy: 0.9629 - val_loss: 0.3165 - val_accuracy: 0.9343 - lr: 0.0014 - momentum: 0.9367\n", + "Epoch 588/588\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1175 - accuracy: 0.9751 - val_loss: 0.3327 - val_accuracy: 0.9295 - lr: 4.1040e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9295\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3327\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m336.90 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m283.22 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m53.68 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [98] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m99\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 588)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0102\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 589/594\n", + "256/256 [==============================] - 51s 185ms/step - loss: 0.2144 - accuracy: 0.9351 - val_loss: 0.2413 - val_accuracy: 0.9263 - lr: 0.0061 - momentum: 0.8915\n", + "Epoch 590/594\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.2054 - accuracy: 0.9375 - val_loss: 0.6813 - val_accuracy: 0.8446 - lr: 0.0101 - momentum: 0.8506\n", + "Epoch 591/594\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1723 - accuracy: 0.9551 - val_loss: 0.4419 - val_accuracy: 0.8958 - lr: 0.0083 - momentum: 0.8688\n", + "Epoch 592/594\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1436 - accuracy: 0.9658 - val_loss: 0.4394 - val_accuracy: 0.9087 - lr: 0.0047 - momentum: 0.9037\n", + "Epoch 593/594\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.1841 - accuracy: 0.9478 - val_loss: 0.2847 - val_accuracy: 0.9071 - lr: 0.0014 - momentum: 0.9367\n", + "Epoch 594/594\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.1491 - accuracy: 0.9604 - val_loss: 0.2885 - val_accuracy: 0.9183 - lr: 4.0800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9183\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2885\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m338.08 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m284.22 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m53.86 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [99] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m100\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 594)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01014\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 595/600\n", + "256/256 [==============================] - 51s 185ms/step - loss: 0.2367 - accuracy: 0.9297 - val_loss: 0.3142 - val_accuracy: 0.9167 - lr: 0.0061 - momentum: 0.8915\n", + "Epoch 596/600\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.2108 - accuracy: 0.9380 - val_loss: 0.3346 - val_accuracy: 0.9279 - lr: 0.0101 - momentum: 0.8506\n", + "Epoch 597/600\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.1840 - accuracy: 0.9536 - val_loss: 0.2564 - val_accuracy: 0.9199 - lr: 0.0082 - momentum: 0.8688\n", + "Epoch 598/600\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1475 - accuracy: 0.9644 - val_loss: 0.2859 - val_accuracy: 0.9343 - lr: 0.0047 - momentum: 0.9037\n", + "Epoch 599/600\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1354 - accuracy: 0.9688 - val_loss: 0.2552 - val_accuracy: 0.9327 - lr: 0.0014 - momentum: 0.9367\n", + "Epoch 600/600\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.1124 - accuracy: 0.9771 - val_loss: 0.2574 - val_accuracy: 0.9279 - lr: 4.0560e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9279\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2574\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m338.89 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m284.02 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m54.87 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [100] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m101\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 600)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01008\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 601/606\n", + "256/256 [==============================] - 51s 185ms/step - loss: 0.1830 - accuracy: 0.9463 - val_loss: 0.3112 - val_accuracy: 0.9359 - lr: 0.0061 - momentum: 0.8915\n", + "Epoch 602/606\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.3670 - accuracy: 0.8677 - val_loss: 0.4187 - val_accuracy: 0.8702 - lr: 0.0100 - momentum: 0.8506\n", + "Epoch 603/606\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.2610 - accuracy: 0.9189 - val_loss: 0.2511 - val_accuracy: 0.9183 - lr: 0.0082 - momentum: 0.8688\n", + "Epoch 604/606\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.2041 - accuracy: 0.9419 - val_loss: 0.2197 - val_accuracy: 0.9279 - lr: 0.0047 - momentum: 0.9037\n", + "Epoch 605/606\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1508 - accuracy: 0.9624 - val_loss: 0.3026 - val_accuracy: 0.9135 - lr: 0.0013 - momentum: 0.9367\n", + "Epoch 606/606\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.1358 - accuracy: 0.9619 - val_loss: 0.2853 - val_accuracy: 0.9135 - lr: 4.0320e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9135\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2853\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m339.30 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m284.24 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m55.06 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [101] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m102\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 606)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.01002\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 607/612\n", + "256/256 [==============================] - 51s 185ms/step - loss: 0.2251 - accuracy: 0.9243 - val_loss: 0.3275 - val_accuracy: 0.9183 - lr: 0.0060 - momentum: 0.8915\n", + "Epoch 608/612\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.1922 - accuracy: 0.9419 - val_loss: 0.2651 - val_accuracy: 0.9199 - lr: 0.0100 - momentum: 0.8506\n", + "Epoch 609/612\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.2009 - accuracy: 0.9463 - val_loss: 0.3290 - val_accuracy: 0.9167 - lr: 0.0081 - momentum: 0.8688\n", + "Epoch 610/612\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.1772 - accuracy: 0.9561 - val_loss: 0.2485 - val_accuracy: 0.9215 - lr: 0.0046 - momentum: 0.9037\n", + "Epoch 611/612\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.1503 - accuracy: 0.9648 - val_loss: 0.2716 - val_accuracy: 0.9295 - lr: 0.0013 - momentum: 0.9367\n", + "Epoch 612/612\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.1122 - accuracy: 0.9736 - val_loss: 0.2813 - val_accuracy: 0.9231 - lr: 4.0080e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9215\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2813\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m340.10 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m283.82 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m56.28 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [102] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m103\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 612)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00996\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 613/618\n", + "256/256 [==============================] - 51s 184ms/step - loss: 0.1833 - accuracy: 0.9429 - val_loss: 0.2361 - val_accuracy: 0.9199 - lr: 0.0060 - momentum: 0.8915\n", + "Epoch 614/618\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.2008 - accuracy: 0.9365 - val_loss: 0.2739 - val_accuracy: 0.9215 - lr: 0.0099 - momentum: 0.8506\n", + "Epoch 615/618\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.1891 - accuracy: 0.9556 - val_loss: 0.2914 - val_accuracy: 0.9215 - lr: 0.0081 - momentum: 0.8688\n", + "Epoch 616/618\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.2023 - accuracy: 0.9424 - val_loss: 0.4254 - val_accuracy: 0.8990 - lr: 0.0046 - momentum: 0.9037\n", + "Epoch 617/618\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1599 - accuracy: 0.9634 - val_loss: 0.3748 - val_accuracy: 0.9135 - lr: 0.0013 - momentum: 0.9367\n", + "Epoch 618/618\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.1402 - accuracy: 0.9629 - val_loss: 0.2984 - val_accuracy: 0.9247 - lr: 3.9840e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9247\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2983\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m340.30 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m284.35 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m55.96 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [103] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m104\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 618)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0099\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 619/624\n", + "256/256 [==============================] - 51s 184ms/step - loss: 0.2028 - accuracy: 0.9375 - val_loss: 0.3109 - val_accuracy: 0.9311 - lr: 0.0060 - momentum: 0.8915\n", + "Epoch 620/624\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.1986 - accuracy: 0.9390 - val_loss: 0.3335 - val_accuracy: 0.9311 - lr: 0.0098 - momentum: 0.8506\n", + "Epoch 621/624\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1618 - accuracy: 0.9521 - val_loss: 0.2746 - val_accuracy: 0.9343 - lr: 0.0080 - momentum: 0.8688\n", + "Epoch 622/624\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.1339 - accuracy: 0.9639 - val_loss: 0.2910 - val_accuracy: 0.9423 - lr: 0.0046 - momentum: 0.9037\n", + "Epoch 623/624\n", + "256/256 [==============================] - 46s 180ms/step - loss: 0.1103 - accuracy: 0.9736 - val_loss: 0.3011 - val_accuracy: 0.9327 - lr: 0.0013 - momentum: 0.9367\n", + "Epoch 624/624\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.1004 - accuracy: 0.9727 - val_loss: 0.2698 - val_accuracy: 0.9423 - lr: 3.9600e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9423\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2698\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m340.71 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m284.80 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m55.91 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [104] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m105\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 624)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00984\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 625/630\n", + "256/256 [==============================] - 51s 185ms/step - loss: 0.1806 - accuracy: 0.9463 - val_loss: 0.3013 - val_accuracy: 0.9327 - lr: 0.0059 - momentum: 0.8915\n", + "Epoch 626/630\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.1939 - accuracy: 0.9399 - val_loss: 0.3433 - val_accuracy: 0.9359 - lr: 0.0098 - momentum: 0.8506\n", + "Epoch 627/630\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1779 - accuracy: 0.9580 - val_loss: 0.2583 - val_accuracy: 0.9375 - lr: 0.0080 - momentum: 0.8688\n", + "Epoch 628/630\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.1552 - accuracy: 0.9614 - val_loss: 0.2010 - val_accuracy: 0.9423 - lr: 0.0046 - momentum: 0.9037\n", + "Epoch 629/630\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.1256 - accuracy: 0.9731 - val_loss: 0.2472 - val_accuracy: 0.9407 - lr: 0.0013 - momentum: 0.9367\n", + "Epoch 630/630\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1038 - accuracy: 0.9736 - val_loss: 0.2743 - val_accuracy: 0.9343 - lr: 3.9360e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9343\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2743\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m341.65 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m285.25 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m56.39 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [105] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m106\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 630)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00978\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 631/636\n", + "256/256 [==============================] - 51s 185ms/step - loss: 0.1986 - accuracy: 0.9355 - val_loss: 0.3703 - val_accuracy: 0.9279 - lr: 0.0059 - momentum: 0.8915\n", + "Epoch 632/636\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.2135 - accuracy: 0.9341 - val_loss: 0.3344 - val_accuracy: 0.9263 - lr: 0.0097 - momentum: 0.8506\n", + "Epoch 633/636\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.1732 - accuracy: 0.9536 - val_loss: 0.3282 - val_accuracy: 0.9295 - lr: 0.0079 - momentum: 0.8688\n", + "Epoch 634/636\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.1595 - accuracy: 0.9546 - val_loss: 0.2195 - val_accuracy: 0.9423 - lr: 0.0045 - momentum: 0.9037\n", + "Epoch 635/636\n", + "256/256 [==============================] - 46s 181ms/step - loss: 0.1178 - accuracy: 0.9712 - val_loss: 0.3095 - val_accuracy: 0.9279 - lr: 0.0013 - momentum: 0.9367\n", + "Epoch 636/636\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.0843 - accuracy: 0.9805 - val_loss: 0.2921 - val_accuracy: 0.9343 - lr: 3.9120e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9343\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2922\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m341.52 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m284.79 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m56.73 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [106] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m107\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 636)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00972\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 637/642\n", + "256/256 [==============================] - 51s 185ms/step - loss: 0.2098 - accuracy: 0.9419 - val_loss: 0.2439 - val_accuracy: 0.9359 - lr: 0.0059 - momentum: 0.8915\n", + "Epoch 638/642\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.1870 - accuracy: 0.9443 - val_loss: 0.1919 - val_accuracy: 0.9455 - lr: 0.0097 - momentum: 0.8506\n", + "Epoch 639/642\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.2163 - accuracy: 0.9434 - val_loss: 0.3369 - val_accuracy: 0.9343 - lr: 0.0079 - momentum: 0.8688\n", + "Epoch 640/642\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1658 - accuracy: 0.9595 - val_loss: 0.3353 - val_accuracy: 0.9199 - lr: 0.0045 - momentum: 0.9037\n", + "Epoch 641/642\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.1276 - accuracy: 0.9683 - val_loss: 0.3116 - val_accuracy: 0.9375 - lr: 0.0013 - momentum: 0.9367\n", + "Epoch 642/642\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.0968 - accuracy: 0.9810 - val_loss: 0.2878 - val_accuracy: 0.9359 - lr: 3.8880e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9359\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2878\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m342.12 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m285.32 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m56.80 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [107] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m108\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 642)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33m└───Shuffling data...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00966\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 643/648\n", + "256/256 [==============================] - 51s 185ms/step - loss: 0.1801 - accuracy: 0.9463 - val_loss: 0.1890 - val_accuracy: 0.9359 - lr: 0.0058 - momentum: 0.8915\n", + "Epoch 644/648\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1856 - accuracy: 0.9507 - val_loss: 0.2684 - val_accuracy: 0.9359 - lr: 0.0096 - momentum: 0.8506\n", + "Epoch 645/648\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.1530 - accuracy: 0.9663 - val_loss: 0.2430 - val_accuracy: 0.9311 - lr: 0.0078 - momentum: 0.8688\n", + "Epoch 646/648\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1236 - accuracy: 0.9707 - val_loss: 0.3698 - val_accuracy: 0.9311 - lr: 0.0045 - momentum: 0.9037\n", + "Epoch 647/648\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.0929 - accuracy: 0.9790 - val_loss: 0.3974 - val_accuracy: 0.9167 - lr: 0.0013 - momentum: 0.9367\n", + "Epoch 648/648\n", + "256/256 [==============================] - 47s 181ms/step - loss: 0.0803 - accuracy: 0.9814 - val_loss: 0.3989 - val_accuracy: 0.9343 - lr: 3.8640e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9343\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3989\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17603926360607147. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m343.31 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m285.08 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m58.22 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [108] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m109\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 648)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0096\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 649/654\n", + "256/256 [==============================] - 51s 186ms/step - loss: 0.1925 - accuracy: 0.9478 - val_loss: 0.2649 - val_accuracy: 0.9375 - lr: 0.0058 - momentum: 0.8915\n", + "Epoch 650/654\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1847 - accuracy: 0.9517 - val_loss: 0.2350 - val_accuracy: 0.9295 - lr: 0.0095 - momentum: 0.8506\n", + "Epoch 651/654\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1808 - accuracy: 0.9531 - val_loss: 0.1877 - val_accuracy: 0.9471 - lr: 0.0078 - momentum: 0.8688\n", + "Epoch 652/654\n", + "256/256 [==============================] - 48s 185ms/step - loss: 0.1492 - accuracy: 0.9639 - val_loss: 0.1736 - val_accuracy: 0.9519 - lr: 0.0044 - momentum: 0.9037\n", + "Epoch 653/654\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1449 - accuracy: 0.9619 - val_loss: 0.2009 - val_accuracy: 0.9295 - lr: 0.0013 - momentum: 0.9367\n", + "Epoch 654/654\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1233 - accuracy: 0.9648 - val_loss: 0.2104 - val_accuracy: 0.9183 - lr: 3.8400e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-652-0.9519.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9519\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1736\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mImproved model loss from \u001b[0m\u001b[0;32m0.17603926360607147 \u001b[0m\u001b[0;33mto \u001b[0m\u001b[0;32m0.17357933521270752\u001b[0m\u001b[0;33m. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m346.39 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m286.87 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.52 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [109] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m110\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 654)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00954\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 655/660\n", + "256/256 [==============================] - 52s 189ms/step - loss: 0.2077 - accuracy: 0.9434 - val_loss: 0.2075 - val_accuracy: 0.9455 - lr: 0.0057 - momentum: 0.8915\n", + "Epoch 656/660\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.2784 - accuracy: 0.9106 - val_loss: 0.1934 - val_accuracy: 0.9375 - lr: 0.0095 - momentum: 0.8506\n", + "Epoch 657/660\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1754 - accuracy: 0.9463 - val_loss: 0.1810 - val_accuracy: 0.9439 - lr: 0.0077 - momentum: 0.8688\n", + "Epoch 658/660\n", + "256/256 [==============================] - 47s 184ms/step - loss: 0.1393 - accuracy: 0.9590 - val_loss: 0.2485 - val_accuracy: 0.9359 - lr: 0.0044 - momentum: 0.9037\n", + "Epoch 659/660\n", + "256/256 [==============================] - 48s 186ms/step - loss: 0.1253 - accuracy: 0.9663 - val_loss: 0.2008 - val_accuracy: 0.9503 - lr: 0.0013 - momentum: 0.9367\n", + "Epoch 660/660\n", + "256/256 [==============================] - 47s 183ms/step - loss: 0.1082 - accuracy: 0.9707 - val_loss: 0.2412 - val_accuracy: 0.9343 - lr: 3.8160e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9343\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2412\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17357933521270752. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m346.15 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m288.55 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m57.61 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [110] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m111\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 660)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00948\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 661/666\n", + "256/256 [==============================] - 52s 186ms/step - loss: 0.1970 - accuracy: 0.9434 - val_loss: 0.2500 - val_accuracy: 0.9135 - lr: 0.0057 - momentum: 0.8915\n", + "Epoch 662/666\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1853 - accuracy: 0.9438 - val_loss: 0.3575 - val_accuracy: 0.9054 - lr: 0.0094 - momentum: 0.8506\n", + "Epoch 663/666\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.2017 - accuracy: 0.9419 - val_loss: 0.2965 - val_accuracy: 0.9311 - lr: 0.0077 - momentum: 0.8688\n", + "Epoch 664/666\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1682 - accuracy: 0.9619 - val_loss: 0.3504 - val_accuracy: 0.9071 - lr: 0.0044 - momentum: 0.9037\n", + "Epoch 665/666\n", + "256/256 [==============================] - 47s 183ms/step - loss: 0.1360 - accuracy: 0.9634 - val_loss: 0.3270 - val_accuracy: 0.9215 - lr: 0.0013 - momentum: 0.9367\n", + "Epoch 666/666\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1031 - accuracy: 0.9771 - val_loss: 0.3206 - val_accuracy: 0.9279 - lr: 3.7920e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9279\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3206\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17357933521270752. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m346.50 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m286.51 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.99 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [111] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m112\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 666)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00942\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 667/672\n", + "256/256 [==============================] - 52s 187ms/step - loss: 0.2001 - accuracy: 0.9443 - val_loss: 0.2240 - val_accuracy: 0.9407 - lr: 0.0057 - momentum: 0.8915\n", + "Epoch 668/672\n", + "256/256 [==============================] - 48s 185ms/step - loss: 0.1905 - accuracy: 0.9531 - val_loss: 0.1934 - val_accuracy: 0.9535 - lr: 0.0094 - momentum: 0.8506\n", + "Epoch 669/672\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1603 - accuracy: 0.9541 - val_loss: 0.2721 - val_accuracy: 0.9375 - lr: 0.0076 - momentum: 0.8688\n", + "Epoch 670/672\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1281 - accuracy: 0.9673 - val_loss: 0.2933 - val_accuracy: 0.9295 - lr: 0.0044 - momentum: 0.9037\n", + "Epoch 671/672\n", + "256/256 [==============================] - 47s 183ms/step - loss: 0.1257 - accuracy: 0.9678 - val_loss: 0.2563 - val_accuracy: 0.9327 - lr: 0.0013 - momentum: 0.9367\n", + "Epoch 672/672\n", + "256/256 [==============================] - 47s 183ms/step - loss: 0.0931 - accuracy: 0.9790 - val_loss: 0.2896 - val_accuracy: 0.9327 - lr: 3.7680e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9327\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2896\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17357933521270752. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m345.11 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m287.78 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m57.33 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [112] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m113\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 672)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00936\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 673/678\n", + "256/256 [==============================] - 52s 186ms/step - loss: 0.1816 - accuracy: 0.9497 - val_loss: 0.2687 - val_accuracy: 0.9375 - lr: 0.0056 - momentum: 0.8915\n", + "Epoch 674/678\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1864 - accuracy: 0.9448 - val_loss: 0.2001 - val_accuracy: 0.9359 - lr: 0.0093 - momentum: 0.8506\n", + "Epoch 675/678\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1929 - accuracy: 0.9448 - val_loss: 0.2183 - val_accuracy: 0.9503 - lr: 0.0076 - momentum: 0.8688\n", + "Epoch 676/678\n", + "256/256 [==============================] - 47s 183ms/step - loss: 0.1564 - accuracy: 0.9639 - val_loss: 0.2089 - val_accuracy: 0.9487 - lr: 0.0043 - momentum: 0.9037\n", + "Epoch 677/678\n", + "256/256 [==============================] - 47s 183ms/step - loss: 0.1309 - accuracy: 0.9658 - val_loss: 0.1883 - val_accuracy: 0.9455 - lr: 0.0012 - momentum: 0.9367\n", + "Epoch 678/678\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1132 - accuracy: 0.9746 - val_loss: 0.2309 - val_accuracy: 0.9343 - lr: 3.7440e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9343\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2309\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17357933521270752. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m345.87 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m287.05 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m58.82 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [113] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m114\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 678)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0093\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 679/684\n", + "256/256 [==============================] - 52s 186ms/step - loss: 0.2029 - accuracy: 0.9429 - val_loss: 1004.5974 - val_accuracy: 0.6250 - lr: 0.0056 - momentum: 0.8915\n", + "Epoch 680/684\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.2141 - accuracy: 0.9326 - val_loss: 0.3285 - val_accuracy: 0.9343 - lr: 0.0092 - momentum: 0.8506\n", + "Epoch 681/684\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1780 - accuracy: 0.9585 - val_loss: 0.4111 - val_accuracy: 0.8942 - lr: 0.0075 - momentum: 0.8688\n", + "Epoch 682/684\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1443 - accuracy: 0.9648 - val_loss: 0.2738 - val_accuracy: 0.9343 - lr: 0.0043 - momentum: 0.9037\n", + "Epoch 683/684\n", + "256/256 [==============================] - 47s 183ms/step - loss: 0.1348 - accuracy: 0.9658 - val_loss: 0.2881 - val_accuracy: 0.9199 - lr: 0.0012 - momentum: 0.9367\n", + "Epoch 684/684\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1383 - accuracy: 0.9629 - val_loss: 0.3661 - val_accuracy: 0.9071 - lr: 3.7200e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9071\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3661\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17357933521270752. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m345.52 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m286.82 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m58.70 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [114] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m115\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 684)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00924\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 685/690\n", + "256/256 [==============================] - 52s 187ms/step - loss: 0.1926 - accuracy: 0.9409 - val_loss: 0.4415 - val_accuracy: 0.9038 - lr: 0.0056 - momentum: 0.8915\n", + "Epoch 686/690\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1677 - accuracy: 0.9531 - val_loss: 0.2239 - val_accuracy: 0.9343 - lr: 0.0092 - momentum: 0.8506\n", + "Epoch 687/690\n", + "256/256 [==============================] - 47s 183ms/step - loss: 0.1899 - accuracy: 0.9478 - val_loss: 0.2276 - val_accuracy: 0.9311 - lr: 0.0075 - momentum: 0.8688\n", + "Epoch 688/690\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1573 - accuracy: 0.9590 - val_loss: 0.2471 - val_accuracy: 0.9215 - lr: 0.0043 - momentum: 0.9037\n", + "Epoch 689/690\n", + "256/256 [==============================] - 47s 182ms/step - loss: 0.1212 - accuracy: 0.9702 - val_loss: 0.2242 - val_accuracy: 0.9311 - lr: 0.0012 - momentum: 0.9367\n", + "Epoch 690/690\n", + "256/256 [==============================] - 47s 183ms/step - loss: 0.1065 - accuracy: 0.9731 - val_loss: 0.2341 - val_accuracy: 0.9311 - lr: 3.6960e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9311\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2341\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17357933521270752. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m345.10 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m287.11 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m57.98 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [115] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m116\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 690)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00918\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 691/696\n", + "256/256 [==============================] - 60s 212ms/step - loss: 0.1553 - accuracy: 0.9570 - val_loss: 0.3812 - val_accuracy: 0.9167 - lr: 0.0055 - momentum: 0.8915\n", + "Epoch 692/696\n", + "256/256 [==============================] - 53s 206ms/step - loss: 0.1734 - accuracy: 0.9541 - val_loss: 0.1841 - val_accuracy: 0.9455 - lr: 0.0091 - momentum: 0.8506\n", + "Epoch 693/696\n", + "256/256 [==============================] - 53s 206ms/step - loss: 0.1747 - accuracy: 0.9482 - val_loss: 0.1765 - val_accuracy: 0.9503 - lr: 0.0075 - momentum: 0.8688\n", + "Epoch 694/696\n", + "256/256 [==============================] - 53s 207ms/step - loss: 0.1481 - accuracy: 0.9590 - val_loss: 0.2868 - val_accuracy: 0.9311 - lr: 0.0042 - momentum: 0.9037\n", + "Epoch 695/696\n", + "256/256 [==============================] - 53s 205ms/step - loss: 0.1279 - accuracy: 0.9707 - val_loss: 0.2949 - val_accuracy: 0.9327 - lr: 0.0012 - momentum: 0.9367\n", + "Epoch 696/696\n", + "256/256 [==============================] - 53s 205ms/step - loss: 0.1000 - accuracy: 0.9746 - val_loss: 0.2569 - val_accuracy: 0.9487 - lr: 3.6720e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9487\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2569\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17357933521270752. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m395.56 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m325.05 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m70.51 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [116] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m117\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 696)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00912\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 697/702\n", + "256/256 [==============================] - 59s 210ms/step - loss: 0.1846 - accuracy: 0.9502 - val_loss: 0.1894 - val_accuracy: 0.9519 - lr: 0.0055 - momentum: 0.8915\n", + "Epoch 698/702\n", + "256/256 [==============================] - 53s 205ms/step - loss: 0.1809 - accuracy: 0.9497 - val_loss: 0.2153 - val_accuracy: 0.9423 - lr: 0.0091 - momentum: 0.8506\n", + "Epoch 699/702\n", + "256/256 [==============================] - 53s 208ms/step - loss: 0.1674 - accuracy: 0.9624 - val_loss: 0.2563 - val_accuracy: 0.9471 - lr: 0.0074 - momentum: 0.8688\n", + "Epoch 700/702\n", + "256/256 [==============================] - 52s 204ms/step - loss: 0.1207 - accuracy: 0.9731 - val_loss: 0.2883 - val_accuracy: 0.9359 - lr: 0.0042 - momentum: 0.9037\n", + "Epoch 701/702\n", + "256/256 [==============================] - 52s 202ms/step - loss: 0.1315 - accuracy: 0.9707 - val_loss: 0.2695 - val_accuracy: 0.9343 - lr: 0.0012 - momentum: 0.9367\n", + "Epoch 702/702\n", + "256/256 [==============================] - 53s 206ms/step - loss: 0.0929 - accuracy: 0.9819 - val_loss: 0.2316 - val_accuracy: 0.9487 - lr: 3.6480e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9487\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2316\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17357933521270752. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m397.21 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m323.14 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m74.07 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [117] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m118\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 702)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00906\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 703/708\n", + "256/256 [==============================] - 59s 212ms/step - loss: 0.1751 - accuracy: 0.9478 - val_loss: 0.4427 - val_accuracy: 0.8494 - lr: 0.0055 - momentum: 0.8915\n", + "Epoch 704/708\n", + "256/256 [==============================] - 53s 206ms/step - loss: 0.2010 - accuracy: 0.9326 - val_loss: 0.2286 - val_accuracy: 0.9439 - lr: 0.0090 - momentum: 0.8506\n", + "Epoch 705/708\n", + "256/256 [==============================] - 53s 206ms/step - loss: 0.1860 - accuracy: 0.9443 - val_loss: 0.2116 - val_accuracy: 0.9423 - lr: 0.0074 - momentum: 0.8688\n", + "Epoch 706/708\n", + "256/256 [==============================] - 53s 207ms/step - loss: 0.1386 - accuracy: 0.9614 - val_loss: 0.2094 - val_accuracy: 0.9407 - lr: 0.0042 - momentum: 0.9037\n", + "Epoch 707/708\n", + "256/256 [==============================] - 53s 206ms/step - loss: 0.1317 - accuracy: 0.9600 - val_loss: 0.2283 - val_accuracy: 0.9455 - lr: 0.0012 - momentum: 0.9367\n", + "Epoch 708/708\n", + "256/256 [==============================] - 53s 205ms/step - loss: 0.1034 - accuracy: 0.9746 - val_loss: 0.2400 - val_accuracy: 0.9455 - lr: 3.6240e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9455\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2400\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17357933521270752. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m401.07 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m325.12 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m75.95 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [118] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m119\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 708)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.009\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 709/714\n", + "256/256 [==============================] - 59s 212ms/step - loss: 0.1538 - accuracy: 0.9619 - val_loss: 0.2188 - val_accuracy: 0.9359 - lr: 0.0054 - momentum: 0.8915\n", + "Epoch 710/714\n", + "256/256 [==============================] - 53s 207ms/step - loss: 0.2128 - accuracy: 0.9355 - val_loss: 0.2977 - val_accuracy: 0.9135 - lr: 0.0089 - momentum: 0.8506\n", + "Epoch 711/714\n", + "256/256 [==============================] - 53s 207ms/step - loss: 0.2005 - accuracy: 0.9409 - val_loss: 0.1948 - val_accuracy: 0.9407 - lr: 0.0073 - momentum: 0.8688\n", + "Epoch 712/714\n", + "256/256 [==============================] - 53s 204ms/step - loss: 0.1602 - accuracy: 0.9561 - val_loss: 0.2424 - val_accuracy: 0.9407 - lr: 0.0042 - momentum: 0.9037\n", + "Epoch 713/714\n", + "256/256 [==============================] - 53s 208ms/step - loss: 0.1633 - accuracy: 0.9546 - val_loss: 0.3531 - val_accuracy: 0.9119 - lr: 0.0012 - momentum: 0.9367\n", + "Epoch 714/714\n", + "256/256 [==============================] - 53s 207ms/step - loss: 0.1262 - accuracy: 0.9648 - val_loss: 0.3895 - val_accuracy: 0.9087 - lr: 3.6000e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9087\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3879\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17357933521270752. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m399.51 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m325.86 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m73.65 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [119] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m120\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 714)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00894\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 715/720\n", + "256/256 [==============================] - 60s 213ms/step - loss: 0.2853 - accuracy: 0.8921 - val_loss: 44.1765 - val_accuracy: 0.4968 - lr: 0.0054 - momentum: 0.8915\n", + "Epoch 716/720\n", + "256/256 [==============================] - 53s 207ms/step - loss: 0.4900 - accuracy: 0.7964 - val_loss: 0.3934 - val_accuracy: 0.8510 - lr: 0.0089 - momentum: 0.8506\n", + "Epoch 717/720\n", + "256/256 [==============================] - 53s 207ms/step - loss: 0.3208 - accuracy: 0.8809 - val_loss: 0.4923 - val_accuracy: 0.8077 - lr: 0.0073 - momentum: 0.8688\n", + "Epoch 718/720\n", + "256/256 [==============================] - 53s 204ms/step - loss: 0.2574 - accuracy: 0.9146 - val_loss: 0.2903 - val_accuracy: 0.9087 - lr: 0.0041 - momentum: 0.9037\n", + "Epoch 719/720\n", + "256/256 [==============================] - 53s 206ms/step - loss: 0.2284 - accuracy: 0.9263 - val_loss: 0.2491 - val_accuracy: 0.9247 - lr: 0.0012 - momentum: 0.9367\n", + "Epoch 720/720\n", + "256/256 [==============================] - 53s 206ms/step - loss: 0.1839 - accuracy: 0.9434 - val_loss: 0.2490 - val_accuracy: 0.9215 - lr: 3.5760e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9215\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2490\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17357933521270752. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m401.69 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m325.27 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m76.42 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [120] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m121\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 720)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00888\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 721/726\n", + "256/256 [==============================] - 60s 213ms/step - loss: 0.2467 - accuracy: 0.9219 - val_loss: 0.2657 - val_accuracy: 0.9167 - lr: 0.0053 - momentum: 0.8915\n", + "Epoch 722/726\n", + "256/256 [==============================] - 53s 206ms/step - loss: 0.2420 - accuracy: 0.9209 - val_loss: 0.5724 - val_accuracy: 0.7051 - lr: 0.0088 - momentum: 0.8506\n", + "Epoch 723/726\n", + "256/256 [==============================] - 52s 204ms/step - loss: 0.2383 - accuracy: 0.9258 - val_loss: 0.2247 - val_accuracy: 0.9215 - lr: 0.0072 - momentum: 0.8688\n", + "Epoch 724/726\n", + "256/256 [==============================] - 53s 207ms/step - loss: 0.1874 - accuracy: 0.9458 - val_loss: 0.2328 - val_accuracy: 0.9295 - lr: 0.0041 - momentum: 0.9037\n", + "Epoch 725/726\n", + "256/256 [==============================] - 53s 207ms/step - loss: 0.1532 - accuracy: 0.9512 - val_loss: 0.2350 - val_accuracy: 0.9247 - lr: 0.0012 - momentum: 0.9367\n", + "Epoch 726/726\n", + "256/256 [==============================] - 53s 206ms/step - loss: 0.1403 - accuracy: 0.9590 - val_loss: 0.2735 - val_accuracy: 0.9215 - lr: 3.5520e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9215\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2735\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17357933521270752. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m400.42 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m325.23 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m75.19 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [121] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m122\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 726)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00882\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 727/732\n", + "256/256 [==============================] - 60s 215ms/step - loss: 0.2011 - accuracy: 0.9351 - val_loss: 0.2269 - val_accuracy: 0.9167 - lr: 0.0053 - momentum: 0.8915\n", + "Epoch 728/732\n", + "256/256 [==============================] - 53s 206ms/step - loss: 0.2072 - accuracy: 0.9307 - val_loss: 0.2523 - val_accuracy: 0.9343 - lr: 0.0088 - momentum: 0.8506\n", + "Epoch 729/732\n", + "256/256 [==============================] - 53s 206ms/step - loss: 0.4140 - accuracy: 0.8306 - val_loss: 0.2999 - val_accuracy: 0.8910 - lr: 0.0072 - momentum: 0.8688\n", + "Epoch 730/732\n", + "256/256 [==============================] - 53s 204ms/step - loss: 0.2811 - accuracy: 0.9058 - val_loss: 0.2931 - val_accuracy: 0.9231 - lr: 0.0041 - momentum: 0.9037\n", + "Epoch 731/732\n", + "256/256 [==============================] - 53s 204ms/step - loss: 0.1958 - accuracy: 0.9380 - val_loss: 0.2793 - val_accuracy: 0.9135 - lr: 0.0012 - momentum: 0.9367\n", + "Epoch 732/732\n", + "256/256 [==============================] - 53s 204ms/step - loss: 0.1798 - accuracy: 0.9507 - val_loss: 0.2849 - val_accuracy: 0.9247 - lr: 3.5280e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9247\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2849\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17357933521270752. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m400.06 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m324.16 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m75.90 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [122] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m123\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 732)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00876\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 733/738\n", + "256/256 [==============================] - 60s 214ms/step - loss: 0.2367 - accuracy: 0.9277 - val_loss: 0.3106 - val_accuracy: 0.9215 - lr: 0.0053 - momentum: 0.8915\n", + "Epoch 734/738\n", + "256/256 [==============================] - 52s 204ms/step - loss: 0.2412 - accuracy: 0.9282 - val_loss: 0.2260 - val_accuracy: 0.9327 - lr: 0.0087 - momentum: 0.8506\n", + "Epoch 735/738\n", + "256/256 [==============================] - 52s 204ms/step - loss: 0.2291 - accuracy: 0.9307 - val_loss: 0.2937 - val_accuracy: 0.9022 - lr: 0.0071 - momentum: 0.8688\n", + "Epoch 736/738\n", + "256/256 [==============================] - 52s 204ms/step - loss: 0.2225 - accuracy: 0.9292 - val_loss: 0.2502 - val_accuracy: 0.9231 - lr: 0.0041 - momentum: 0.9037\n", + "Epoch 737/738\n", + "256/256 [==============================] - 52s 203ms/step - loss: 0.2293 - accuracy: 0.9238 - val_loss: 0.1966 - val_accuracy: 0.9247 - lr: 0.0012 - momentum: 0.9367\n", + "Epoch 738/738\n", + "256/256 [==============================] - 48s 188ms/step - loss: 0.2015 - accuracy: 0.9307 - val_loss: 0.2094 - val_accuracy: 0.9247 - lr: 3.5040e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9247\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2095\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17357933521270752. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m393.92 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m317.95 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m75.97 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [123] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m124\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 738)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0087\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 739/744\n", + "256/256 [==============================] - 55s 195ms/step - loss: 0.2290 - accuracy: 0.9224 - val_loss: 0.2004 - val_accuracy: 0.9295 - lr: 0.0052 - momentum: 0.8915\n", + "Epoch 740/744\n", + "256/256 [==============================] - 48s 188ms/step - loss: 0.2007 - accuracy: 0.9385 - val_loss: 0.2318 - val_accuracy: 0.9295 - lr: 0.0087 - momentum: 0.8506\n", + "Epoch 741/744\n", + "256/256 [==============================] - 49s 189ms/step - loss: 0.1886 - accuracy: 0.9404 - val_loss: 0.2851 - val_accuracy: 0.9247 - lr: 0.0071 - momentum: 0.8688\n", + "Epoch 742/744\n", + "256/256 [==============================] - 48s 189ms/step - loss: 0.1909 - accuracy: 0.9404 - val_loss: 0.2319 - val_accuracy: 0.9231 - lr: 0.0040 - momentum: 0.9037\n", + "Epoch 743/744\n", + "256/256 [==============================] - 49s 190ms/step - loss: 0.1498 - accuracy: 0.9536 - val_loss: 0.2112 - val_accuracy: 0.9247 - lr: 0.0012 - momentum: 0.9367\n", + "Epoch 744/744\n", + "256/256 [==============================] - 53s 205ms/step - loss: 0.1260 - accuracy: 0.9624 - val_loss: 0.2279 - val_accuracy: 0.9247 - lr: 3.4800e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9247\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2279\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17357933521270752. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m373.91 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m302.21 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m71.70 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [124] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m125\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 744)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00864\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 745/750\n", + "256/256 [==============================] - 59s 212ms/step - loss: 0.2364 - accuracy: 0.9268 - val_loss: 0.1946 - val_accuracy: 0.9295 - lr: 0.0052 - momentum: 0.8915\n", + "Epoch 746/750\n", + "256/256 [==============================] - 52s 203ms/step - loss: 0.2108 - accuracy: 0.9292 - val_loss: 0.1880 - val_accuracy: 0.9423 - lr: 0.0086 - momentum: 0.8506\n", + "Epoch 747/750\n", + "256/256 [==============================] - 52s 204ms/step - loss: 0.1923 - accuracy: 0.9429 - val_loss: 0.2489 - val_accuracy: 0.9311 - lr: 0.0070 - momentum: 0.8688\n", + "Epoch 748/750\n", + "256/256 [==============================] - 52s 204ms/step - loss: 0.1763 - accuracy: 0.9443 - val_loss: 0.2070 - val_accuracy: 0.9359 - lr: 0.0040 - momentum: 0.9037\n", + "Epoch 749/750\n", + "256/256 [==============================] - 52s 201ms/step - loss: 0.1504 - accuracy: 0.9595 - val_loss: 0.2190 - val_accuracy: 0.9295 - lr: 0.0012 - momentum: 0.9367\n", + "Epoch 750/750\n", + "256/256 [==============================] - 51s 200ms/step - loss: 0.1180 - accuracy: 0.9673 - val_loss: 0.1815 - val_accuracy: 0.9375 - lr: 3.4560e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9375\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1815\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17357933521270752. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m396.27 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m320.17 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m76.10 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [125] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m126\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 750)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33m└───Shuffling data...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00858\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 751/756\n", + "256/256 [==============================] - 59s 211ms/step - loss: 0.1875 - accuracy: 0.9395 - val_loss: 0.1891 - val_accuracy: 0.9391 - lr: 0.0052 - momentum: 0.8915\n", + "Epoch 752/756\n", + "256/256 [==============================] - 54s 209ms/step - loss: 0.1770 - accuracy: 0.9521 - val_loss: 0.2006 - val_accuracy: 0.9391 - lr: 0.0085 - momentum: 0.8506\n", + "Epoch 753/756\n", + "256/256 [==============================] - 54s 209ms/step - loss: 0.1858 - accuracy: 0.9434 - val_loss: 0.3627 - val_accuracy: 0.9119 - lr: 0.0070 - momentum: 0.8688\n", + "Epoch 754/756\n", + "256/256 [==============================] - 54s 209ms/step - loss: 0.1482 - accuracy: 0.9648 - val_loss: 0.2054 - val_accuracy: 0.9279 - lr: 0.0040 - momentum: 0.9037\n", + "Epoch 755/756\n", + "256/256 [==============================] - 54s 210ms/step - loss: 0.1425 - accuracy: 0.9614 - val_loss: 0.2056 - val_accuracy: 0.9343 - lr: 0.0011 - momentum: 0.9367\n", + "Epoch 756/756\n", + "256/256 [==============================] - 54s 208ms/step - loss: 0.1070 - accuracy: 0.9736 - val_loss: 0.2019 - val_accuracy: 0.9407 - lr: 3.4320e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9407\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2019\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17357933521270752. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m408.38 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m328.23 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m80.15 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [126] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m127\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 756)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00852\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 757/762\n", + "256/256 [==============================] - 60s 215ms/step - loss: 0.1892 - accuracy: 0.9414 - val_loss: 0.1697 - val_accuracy: 0.9439 - lr: 0.0051 - momentum: 0.8915\n", + "Epoch 758/762\n", + "256/256 [==============================] - 53s 205ms/step - loss: 0.1981 - accuracy: 0.9399 - val_loss: 0.2210 - val_accuracy: 0.9359 - lr: 0.0085 - momentum: 0.8506\n", + "Epoch 759/762\n", + "256/256 [==============================] - 52s 204ms/step - loss: 0.1458 - accuracy: 0.9624 - val_loss: 0.2387 - val_accuracy: 0.9327 - lr: 0.0069 - momentum: 0.8688\n", + "Epoch 760/762\n", + "256/256 [==============================] - 53s 208ms/step - loss: 0.1451 - accuracy: 0.9604 - val_loss: 0.4134 - val_accuracy: 0.7933 - lr: 0.0039 - momentum: 0.9037\n", + "Epoch 761/762\n", + "256/256 [==============================] - 52s 200ms/step - loss: 0.1456 - accuracy: 0.9644 - val_loss: 0.1825 - val_accuracy: 0.9455 - lr: 0.0011 - momentum: 0.9367\n", + "Epoch 762/762\n", + "256/256 [==============================] - 49s 190ms/step - loss: 0.1163 - accuracy: 0.9712 - val_loss: 0.2020 - val_accuracy: 0.9423 - lr: 3.4080e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;31mERROR: Failed to load weights. Error: max() arg is an empty sequence\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9423\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2020\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17357933521270752. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m397.31 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m319.57 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m77.74 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [127] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m128\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 762)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;31m- Debug DP Sample dir: \u001b[0m\u001b[0;32mSamples/TSR_SUB_400_y2023_m12_d22-h10_m02_s37\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.00846\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 763/768\n", + "256/256 [==============================] - 59s 216ms/step - loss: 0.2035 - accuracy: 0.9404 - val_loss: 0.1979 - val_accuracy: 0.9343 - lr: 0.0051 - momentum: 0.8915\n", + "Epoch 764/768\n", + "256/256 [==============================] - 53s 205ms/step - loss: 0.1976 - accuracy: 0.9448 - val_loss: 0.1832 - val_accuracy: 0.9343 - lr: 0.0084 - momentum: 0.8506\n", + "Epoch 765/768\n", + "256/256 [==============================] - 53s 207ms/step - loss: 0.1549 - accuracy: 0.9556 - val_loss: 0.2422 - val_accuracy: 0.9327 - lr: 0.0069 - momentum: 0.8688\n", + "Epoch 766/768\n", + "256/256 [==============================] - 53s 205ms/step - loss: 0.1255 - accuracy: 0.9629 - val_loss: 0.2552 - val_accuracy: 0.9343 - lr: 0.0039 - momentum: 0.9037\n", + "Epoch 767/768\n", + "256/256 [==============================] - 52s 204ms/step - loss: 0.1143 - accuracy: 0.9697 - val_loss: 0.2569 - val_accuracy: 0.9327 - lr: 0.0011 - momentum: 0.9367\n", + "Epoch 768/768\n", + "256/256 [==============================] - 53s 205ms/step - loss: 0.0877 - accuracy: 0.9805 - val_loss: 0.2370 - val_accuracy: 0.9295 - lr: 3.3840e-08 - momentum: 0.9500\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{95.6756}, \u001b[0m\u001b[0;33mloss{0.0111}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{97.5646}, loss{0.0020}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9295\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2370\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9599359035491943. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.17357933521270752. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m412.87 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m323.62 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m89.24 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [128] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m129\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m384 (TSEC: 768)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Fine tuning]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|2048|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training OneCycleLr::maxlr to \u001b[0m\u001b[0;32m[0.0084\u001b[0m\u001b[0;31m\u001b[0m\u001b[0;32m]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[6]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 769/774\n", + "256/256 [==============================] - 60s 216ms/step - loss: 0.1933 - accuracy: 0.9355 - val_loss: 0.2169 - val_accuracy: 0.9359 - lr: 0.0051 - momentum: 0.8915\n", + "Epoch 770/774\n", + "256/256 [==============================] - 50s 196ms/step - loss: 0.2025 - accuracy: 0.9351 - val_loss: 0.2068 - val_accuracy: 0.9423 - lr: 0.0084 - momentum: 0.8506\n", + "Epoch 771/774\n", + "256/256 [==============================] - 48s 187ms/step - loss: 0.1770 - accuracy: 0.9531 - val_loss: 0.2168 - val_accuracy: 0.9247 - lr: 0.0068 - momentum: 0.8688\n", + "Epoch 772/774\n", + "256/256 [==============================] - 49s 190ms/step - loss: 0.3436 - accuracy: 0.8853 - val_loss: 0.3420 - val_accuracy: 0.8878 - lr: 0.0039 - momentum: 0.9037\n", + "Epoch 773/774\n", + "121/256 [=============>................] - ETA: 20s - loss: 0.2507 - accuracy: 0.9163" ] } ], diff --git a/README.md b/README.md index 0e51500..a9a811d 100644 --- a/README.md +++ b/README.md @@ -67,13 +67,12 @@ The model provided in this project should not be used for medical diagnosis with ## Results > [!WARNING] -> Results were achived using Rev2 training method and Rev1.2 model. -#### N/A +> Results were achived using Rev2 training method and Rev1.2 model and +> with `backup/V4/Model_T&T.ipynb` code. + +![img1](doc/ACC_P.png) +![img2](doc/LOSS__P.png) diff --git a/doc/ACC_P.png b/doc/ACC_P.png new file mode 100644 index 0000000..7a7e36f Binary files /dev/null and b/doc/ACC_P.png differ diff --git a/doc/LOSS__P.png b/doc/LOSS__P.png new file mode 100644 index 0000000..0280f0e Binary files /dev/null and b/doc/LOSS__P.png differ