diff --git a/BETA_E_Model_T&T.ipynb b/BETA_E_Model_T&T.ipynb index 058612b..1b8eea4 100644 --- a/BETA_E_Model_T&T.ipynb +++ b/BETA_E_Model_T&T.ipynb @@ -4852,110 +4852,82 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 8, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\aydin\\Desktop\\Pneumonia AI Dev\\venv\\lib\\site-packages\\vit_keras\\utils.py:81: UserWarning: Resizing position embeddings from 12, 12 to 7, 7\n", + " warnings.warn(\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "Model: \"vit-l16\"\n", + "Model: \"vit-b32\"\n", "_________________________________________________________________\n", " Layer (type) Output Shape Param # \n", "=================================================================\n", " input_1 (InputLayer) [(None, 224, 224, 3)] 0 \n", " \n", - " embedding (Conv2D) (None, 14, 14, 1024) 787456 \n", + " embedding (Conv2D) (None, 7, 7, 768) 2360064 \n", " \n", - " reshape (Reshape) (None, 196, 1024) 0 \n", + " reshape (Reshape) (None, 49, 768) 0 \n", " \n", - " class_token (ClassToken) (None, 197, 1024) 1024 \n", + " class_token (ClassToken) (None, 50, 768) 768 \n", " \n", - " Transformer/posembed_input (None, 197, 1024) 201728 \n", + " Transformer/posembed_input (None, 50, 768) 38400 \n", " (AddPositionEmbs) \n", " \n", - " Transformer/encoderblock_0 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", - " \n", - " Transformer/encoderblock_1 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", - " \n", - " Transformer/encoderblock_2 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", - " \n", - " Transformer/encoderblock_3 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", - " \n", - " Transformer/encoderblock_4 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", - " \n", - " Transformer/encoderblock_5 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", + " Transformer/encoderblock_0 ((None, 50, 768), 7087872 \n", + " (TransformerBlock) (None, 12, None, None)) \n", " \n", - " Transformer/encoderblock_6 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", + " Transformer/encoderblock_1 ((None, 50, 768), 7087872 \n", + " (TransformerBlock) (None, 12, None, None)) \n", " \n", - " Transformer/encoderblock_7 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", + " Transformer/encoderblock_2 ((None, 50, 768), 7087872 \n", + " (TransformerBlock) (None, 12, None, None)) \n", " \n", - " Transformer/encoderblock_8 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", + " Transformer/encoderblock_3 ((None, 50, 768), 7087872 \n", + " (TransformerBlock) (None, 12, None, None)) \n", " \n", - " Transformer/encoderblock_9 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", + " Transformer/encoderblock_4 ((None, 50, 768), 7087872 \n", + " (TransformerBlock) (None, 12, None, None)) \n", " \n", - " Transformer/encoderblock_10 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", + " Transformer/encoderblock_5 ((None, 50, 768), 7087872 \n", + " (TransformerBlock) (None, 12, None, None)) \n", " \n", - " Transformer/encoderblock_11 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", + " Transformer/encoderblock_6 ((None, 50, 768), 7087872 \n", + " (TransformerBlock) (None, 12, None, None)) \n", " \n", - " Transformer/encoderblock_12 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", + " Transformer/encoderblock_7 ((None, 50, 768), 7087872 \n", + " (TransformerBlock) (None, 12, None, None)) \n", " \n", - " Transformer/encoderblock_13 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", + " Transformer/encoderblock_8 ((None, 50, 768), 7087872 \n", + " (TransformerBlock) (None, 12, None, None)) \n", " \n", - " Transformer/encoderblock_14 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", + " Transformer/encoderblock_9 ((None, 50, 768), 7087872 \n", + " (TransformerBlock) (None, 12, None, None)) \n", " \n", - " Transformer/encoderblock_15 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", + " Transformer/encoderblock_10 ((None, 50, 768), 7087872 \n", + " (TransformerBlock) (None, 12, None, None)) \n", " \n", - " Transformer/encoderblock_16 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", + " Transformer/encoderblock_11 ((None, 50, 768), 7087872 \n", + " (TransformerBlock) (None, 12, None, None)) \n", " \n", - " Transformer/encoderblock_17 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", - " \n", - " Transformer/encoderblock_18 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", - " \n", - " Transformer/encoderblock_19 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", - " \n", - " Transformer/encoderblock_20 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", - " \n", - " Transformer/encoderblock_21 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", - " \n", - " Transformer/encoderblock_22 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", - " \n", - " Transformer/encoderblock_23 ((None, 197, 1024), 12596224 \n", - " (TransformerBlock) (None, 16, None, None)) \n", - " \n", - " Transformer/encoder_norm (L (None, 197, 1024) 2048 \n", + " Transformer/encoder_norm (L (None, 50, 768) 1536 \n", " ayerNormalization) \n", " \n", - " ExtractToken (Lambda) (None, 1024) 0 \n", + " ExtractToken (Lambda) (None, 768) 0 \n", " \n", - " head (Dense) (None, 2) 2050 \n", + " head (Dense) (None, 2) 1538 \n", " \n", "=================================================================\n", - "Total params: 303,303,682\n", - "Trainable params: 303,303,682\n", + "Total params: 87,456,770\n", + "Trainable params: 87,456,770\n", "Non-trainable params: 0\n", "_________________________________________________________________\n" ] @@ -4963,7 +4935,7 @@ ], "source": [ "# Create a simple transformer model\n", - "model = vit.vit_l16(\n", + "model = vit.vit_b32(\n", " image_size=224,\n", " activation='sigmoid',\n", " pretrained=True,\n", @@ -4972,7 +4944,7 @@ " classes=2\n", ")\n", "# model compile + summary\n", - "opt = SGD(learning_rate=0.01, momentum=0.9)\n", + "opt = SGD(learning_rate=0.01, momentum=0.9) # noqa: F405\n", "model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])\n", "model.summary()" ] @@ -7379,7 +7351,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 9, "metadata": { "ExecuteTime": { "end_time": "2023-12-28T07:04:23.573633300Z", @@ -7394,8 +7366,8 @@ "Training the model...\n", "\u001b[0;33m\n", "Setup Verbose:\u001b[0m\n", - "\u001b[0m\u001b[0m\u001b[0;36mExperiment name: \u001b[0m\u001b[0;32m[ViT_Bigboy_y2024_m04_d05-h22_m43_s56]\u001b[0m\u001b[0;36m...\u001b[0m\n", - "\u001b[0m\u001b[0m\u001b[0;36mSetting TensorBoard Log dir to \u001b[0m\u001b[0;32m[logs/fit/ViT_Bigboy_y2024_m04_d05-h22_m43_s56]\u001b[0m\u001b[0;36m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mExperiment name: \u001b[0m\u001b[0;32m[ViT32_y2024_m04_d06-h16_m06_s06]\u001b[0m\u001b[0;36m...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSetting TensorBoard Log dir to \u001b[0m\u001b[0;32m[logs/fit/ViT32_y2024_m04_d06-h16_m06_s06]\u001b[0m\u001b[0;36m...\u001b[0m\n", "\u001b[0m\u001b[0m\u001b[0;36mUse_extended_tensorboard \u001b[0m\u001b[0;32m[False]\u001b[0m\u001b[0;36m.\u001b[0m\n", "\u001b[0m\u001b[0m\u001b[0;36mDebug_OUTPUT_DPS \u001b[0m\u001b[0;32m[True]\u001b[0m\u001b[0;36m.\u001b[0m\n", "\u001b[0m\u001b[0m\u001b[0;36mUse_OneCycleLr \u001b[0m\u001b[0;32m[False]\u001b[0m\u001b[0;36m.\u001b[0m\n", @@ -7415,7 +7387,1619 @@ "\u001b[0;33mPreparing train data...\u001b[0m\n", "\u001b[0;33m- Loading fitted ImageDataGenerator...\u001b[0m\n", "\u001b[0;33m- ImageDataGenerator fit done.\u001b[0m\n", - "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n" + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;31m- Debug DP Sample dir: \u001b[0m\u001b[0;32mSamples/TSR_SUB_400_y2024_m04_d06-h16_m07_s00\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 1/2\n", + "256/256 [==============================] - 25s 71ms/step - loss: 0.8367 - accuracy: 0.5007 - val_loss: 0.8709 - val_accuracy: 0.3750 - lr: 0.0100\n", + "Epoch 2/2\n", + "256/256 [==============================] - 17s 67ms/step - loss: 0.7089 - accuracy: 0.5125 - val_loss: 0.6688 - val_accuracy: 0.5978 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;31mERROR: Failed to load weights. Error: max() arg is an empty sequence\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.5978\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.6688\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model accuracy from 0.000000 to 0.597756. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model loss from inf to 0.66878748. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m111.81 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m43.50 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m68.32 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [1] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m2\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 2)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 3/4\n", + "256/256 [==============================] - 19s 69ms/step - loss: 0.6869 - accuracy: 0.5625 - val_loss: 0.7185 - val_accuracy: 0.4167 - lr: 0.0100\n", + "Epoch 4/4\n", + "256/256 [==============================] - 18s 69ms/step - loss: 0.6137 - accuracy: 0.6550 - val_loss: 0.5447 - val_accuracy: 0.7099 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-004-0.7099.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.7099\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.5447\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model accuracy from 0.597756 to 0.709936. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model loss from 0.66878748 to 0.54466116. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m96.99 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.32 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.67 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [2] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m3\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 4)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 5/6\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.5255 - accuracy: 0.7437 - val_loss: 0.4661 - val_accuracy: 0.7821 - lr: 0.0100\n", + "Epoch 6/6\n", + "256/256 [==============================] - 17s 67ms/step - loss: 0.4751 - accuracy: 0.7771 - val_loss: 0.6533 - val_accuracy: 0.6458 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-005-0.7821.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.7821\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.4661\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model accuracy from 0.709936 to 0.782051. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model loss from 0.54466116 to 0.46606123. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.52 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.19 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m60.34 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [3] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m4\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 6)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 7/8\n", + "256/256 [==============================] - 19s 70ms/step - loss: 0.4827 - accuracy: 0.7671 - val_loss: 0.5618 - val_accuracy: 0.6811 - lr: 0.0100\n", + "Epoch 8/8\n", + "256/256 [==============================] - 18s 68ms/step - loss: 0.4276 - accuracy: 0.8057 - val_loss: 0.4083 - val_accuracy: 0.7949 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-008-0.7949.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.7949\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.4083\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model accuracy from 0.782051 to 0.794872. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model loss from 0.46606123 to 0.40827900. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.23 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.31 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.92 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [4] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m5\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 8)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 9/10\n", + "256/256 [==============================] - 19s 70ms/step - loss: 0.4178 - accuracy: 0.8120 - val_loss: 0.4692 - val_accuracy: 0.7404 - lr: 0.0100\n", + "Epoch 10/10\n", + "256/256 [==============================] - 18s 69ms/step - loss: 0.3705 - accuracy: 0.8445 - val_loss: 0.3654 - val_accuracy: 0.8141 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-010-0.8141.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.8141\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3654\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model accuracy from 0.794872 to 0.814103. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model loss from 0.40827900 to 0.36542034. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.36 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.43 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.93 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [5] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m6\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 10)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 11/12\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.3568 - accuracy: 0.8538 - val_loss: 0.5302 - val_accuracy: 0.7131 - lr: 0.0100\n", + "Epoch 12/12\n", + "256/256 [==============================] - 18s 69ms/step - loss: 0.3142 - accuracy: 0.8704 - val_loss: 0.2370 - val_accuracy: 0.9103 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-012-0.9103.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9103\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2370\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model accuracy from 0.814103 to 0.910256. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model loss from 0.36542034 to 0.23698555. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.93 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.87 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m60.06 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [6] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m7\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 12)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 13/14\n", + "256/256 [==============================] - 19s 69ms/step - loss: 0.3267 - accuracy: 0.8662 - val_loss: 0.2273 - val_accuracy: 0.9183 - lr: 0.0100\n", + "Epoch 14/14\n", + "256/256 [==============================] - 17s 67ms/step - loss: 0.2725 - accuracy: 0.8914 - val_loss: 0.2209 - val_accuracy: 0.8974 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-013-0.9183.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9183\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2273\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model accuracy from 0.910256 to 0.918269. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model loss from 0.23698555 to 0.22731614. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.04 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m36.94 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m60.10 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [7] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m8\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 14)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 15/16\n", + "256/256 [==============================] - 19s 70ms/step - loss: 0.3145 - accuracy: 0.8711 - val_loss: 0.2619 - val_accuracy: 0.8750 - lr: 0.0100\n", + "Epoch 16/16\n", + "256/256 [==============================] - 17s 67ms/step - loss: 0.2718 - accuracy: 0.8887 - val_loss: 0.3634 - val_accuracy: 0.8253 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-015-0.8750.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.8750\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2619\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9182692170. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.2273161411. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m95.40 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.23 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m58.17 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [8] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m9\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 16)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 17/18\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.3088 - accuracy: 0.8728 - val_loss: 0.2533 - val_accuracy: 0.8830 - lr: 0.0100\n", + "Epoch 18/18\n", + "256/256 [==============================] - 18s 69ms/step - loss: 0.2585 - accuracy: 0.8989 - val_loss: 0.2679 - val_accuracy: 0.8910 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-018-0.8910.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.8910\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2679\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9182692170. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.2273161411. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m95.61 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.73 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m57.87 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [9] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m10\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 18)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 19/20\n", + "256/256 [==============================] - 19s 70ms/step - loss: 0.2949 - accuracy: 0.8811 - val_loss: 0.2529 - val_accuracy: 0.8766 - lr: 0.0100\n", + "Epoch 20/20\n", + "256/256 [==============================] - 18s 68ms/step - loss: 0.2337 - accuracy: 0.9082 - val_loss: 0.2200 - val_accuracy: 0.9135 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-020-0.9135.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9135\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2200\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9182692170. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model loss from 0.22731614 to 0.22000232. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m96.10 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.35 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m58.75 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [10] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m11\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 20)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 21/22\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.3011 - accuracy: 0.8740 - val_loss: 0.3893 - val_accuracy: 0.8285 - lr: 0.0100\n", + "Epoch 22/22\n", + "256/256 [==============================] - 17s 67ms/step - loss: 0.3807 - accuracy: 0.8325 - val_loss: 0.3839 - val_accuracy: 0.8061 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-021-0.8285.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.8285\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3893\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9182692170. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.2200023234. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m95.61 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.51 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m58.11 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [11] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m12\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 22)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 23/24\n", + "256/256 [==============================] - 19s 70ms/step - loss: 0.3116 - accuracy: 0.8704 - val_loss: 0.2682 - val_accuracy: 0.8894 - lr: 0.0100\n", + "Epoch 24/24\n", + "256/256 [==============================] - 17s 67ms/step - loss: 0.2301 - accuracy: 0.9099 - val_loss: 0.3072 - val_accuracy: 0.8670 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-023-0.8894.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.8894\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2682\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9182692170. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.2200023234. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m94.92 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.11 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m57.81 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [12] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m13\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 24)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 25/26\n", + "256/256 [==============================] - 19s 70ms/step - loss: 0.2987 - accuracy: 0.8799 - val_loss: 0.2121 - val_accuracy: 0.9071 - lr: 0.0100\n", + "Epoch 26/26\n", + "256/256 [==============================] - 18s 68ms/step - loss: 0.2363 - accuracy: 0.9075 - val_loss: 0.2319 - val_accuracy: 0.9151 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-026-0.9151.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9151\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2319\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9182692170. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.2200023234. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m95.58 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.39 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m58.20 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [13] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m14\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 26)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 27/28\n", + "256/256 [==============================] - 20s 73ms/step - loss: 0.2616 - accuracy: 0.8994 - val_loss: 0.2634 - val_accuracy: 0.8894 - lr: 0.0100\n", + "Epoch 28/28\n", + "256/256 [==============================] - 19s 72ms/step - loss: 0.2123 - accuracy: 0.9189 - val_loss: 0.2989 - val_accuracy: 0.9151 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-028-0.9151.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9151\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2989\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9182692170. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.2200023234. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.49 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m39.20 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m58.29 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [14] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m15\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 28)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 29/30\n", + "256/256 [==============================] - 20s 74ms/step - loss: 0.2677 - accuracy: 0.8933 - val_loss: 0.2094 - val_accuracy: 0.9263 - lr: 0.0100\n", + "Epoch 30/30\n", + "256/256 [==============================] - 18s 71ms/step - loss: 0.2183 - accuracy: 0.9150 - val_loss: 0.1922 - val_accuracy: 0.9247 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-029-0.9263.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9263\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2094\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model accuracy from 0.918269 to 0.926282. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model loss from 0.22000232 to 0.20936431. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m114.34 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m39.30 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m75.04 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [15] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m16\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 30)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 31/32\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.2729 - accuracy: 0.8867 - val_loss: 0.2265 - val_accuracy: 0.9215 - lr: 0.0100\n", + "Epoch 32/32\n", + "256/256 [==============================] - 18s 68ms/step - loss: 0.2206 - accuracy: 0.9126 - val_loss: 0.2086 - val_accuracy: 0.9022 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-031-0.9215.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9215\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2265\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9262820482. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.2093643099. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m105.10 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.48 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m67.62 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [16] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m17\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 32)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 33/34\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.2454 - accuracy: 0.9016 - val_loss: 0.2215 - val_accuracy: 0.8910 - lr: 0.0100\n", + "Epoch 34/34\n", + "256/256 [==============================] - 18s 70ms/step - loss: 0.1990 - accuracy: 0.9233 - val_loss: 0.2374 - val_accuracy: 0.9199 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-034-0.9199.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9199\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2374\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9262820482. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.2093643099. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m98.76 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m38.13 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m60.64 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [17] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m18\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 34)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 35/36\n", + "256/256 [==============================] - 21s 74ms/step - loss: 0.2599 - accuracy: 0.8997 - val_loss: 0.3463 - val_accuracy: 0.8830 - lr: 0.0100\n", + "Epoch 36/36\n", + "256/256 [==============================] - 19s 72ms/step - loss: 0.2175 - accuracy: 0.9163 - val_loss: 0.2044 - val_accuracy: 0.9311 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-036-0.9311.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9311\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2044\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model accuracy from 0.926282 to 0.931090. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model loss from 0.20936431 to 0.20440124. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m116.78 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m40.12 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m76.66 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [18] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m19\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 36)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 37/38\n", + "256/256 [==============================] - 20s 73ms/step - loss: 0.2629 - accuracy: 0.8977 - val_loss: 0.1886 - val_accuracy: 0.9263 - lr: 0.0100\n", + "Epoch 38/38\n", + "256/256 [==============================] - 18s 70ms/step - loss: 0.2133 - accuracy: 0.9141 - val_loss: 0.3803 - val_accuracy: 0.8269 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-037-0.9263.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9263\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1886\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9310897589. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model loss from 0.20440124 to 0.18864366. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m112.25 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m38.82 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m73.44 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [19] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m20\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 38)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 39/40\n", + "256/256 [==============================] - 20s 73ms/step - loss: 0.2349 - accuracy: 0.9072 - val_loss: 0.1834 - val_accuracy: 0.9231 - lr: 0.0100\n", + "Epoch 40/40\n", + "256/256 [==============================] - 18s 70ms/step - loss: 0.2016 - accuracy: 0.9211 - val_loss: 0.2597 - val_accuracy: 0.9151 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-039-0.9231.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9231\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1834\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9310897589. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model loss from 0.18864366 to 0.18342663. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m112.19 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m38.89 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m73.30 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [20] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m21\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 40)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 41/42\n", + "256/256 [==============================] - 21s 75ms/step - loss: 0.2398 - accuracy: 0.9055 - val_loss: 0.1960 - val_accuracy: 0.9215 - lr: 0.0100\n", + "Epoch 42/42\n", + "256/256 [==============================] - 18s 71ms/step - loss: 0.1825 - accuracy: 0.9324 - val_loss: 0.2144 - val_accuracy: 0.9199 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-041-0.9215.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9215\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1961\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9310897589. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1834266335. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m113.68 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m39.69 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m74.00 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [21] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m22\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 42)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 43/44\n", + "256/256 [==============================] - 19s 70ms/step - loss: 0.2349 - accuracy: 0.9067 - val_loss: 0.2273 - val_accuracy: 0.9038 - lr: 0.0100\n", + "Epoch 44/44\n", + "256/256 [==============================] - 17s 67ms/step - loss: 0.2072 - accuracy: 0.9182 - val_loss: 0.2831 - val_accuracy: 0.8798 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-043-0.9038.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9038\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2272\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9310897589. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1834266335. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m105.33 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.10 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m68.22 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [22] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m23\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 44)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 45/46\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.2415 - accuracy: 0.9097 - val_loss: 0.1941 - val_accuracy: 0.9247 - lr: 0.0100\n", + "Epoch 46/46\n", + "256/256 [==============================] - 18s 70ms/step - loss: 0.1958 - accuracy: 0.9250 - val_loss: 0.2088 - val_accuracy: 0.9311 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-046-0.9311.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9311\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2088\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9310897589. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1834266335. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m95.74 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.94 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m57.80 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [23] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m24\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 46)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 47/48\n", + "256/256 [==============================] - 19s 72ms/step - loss: 0.2723 - accuracy: 0.8916 - val_loss: 0.3538 - val_accuracy: 0.8798 - lr: 0.0100\n", + "Epoch 48/48\n", + "256/256 [==============================] - 18s 68ms/step - loss: 0.2987 - accuracy: 0.8660 - val_loss: 0.6846 - val_accuracy: 0.6571 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-047-0.8798.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.8814\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3538\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9310897589. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1834266335. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.79 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.91 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.89 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [24] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m25\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 48)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 1]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 49/50\n", + "256/256 [==============================] - 19s 72ms/step - loss: 0.2676 - accuracy: 0.8975 - val_loss: 0.1891 - val_accuracy: 0.9359 - lr: 0.0100\n", + "Epoch 50/50\n", + "256/256 [==============================] - 17s 68ms/step - loss: 0.2387 - accuracy: 0.9062 - val_loss: 0.2811 - val_accuracy: 0.8718 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-049-0.9359.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9359\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1891\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model accuracy from 0.931090 to 0.935897. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1834266335. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m98.46 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.78 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m60.69 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [25] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m26\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 50)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 51/52\n", + "256/256 [==============================] - 19s 72ms/step - loss: 0.2453 - accuracy: 0.9045 - val_loss: 0.2678 - val_accuracy: 0.9151 - lr: 0.0100\n", + "Epoch 52/52\n", + "256/256 [==============================] - 18s 69ms/step - loss: 0.1953 - accuracy: 0.9216 - val_loss: 0.2645 - val_accuracy: 0.9231 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-052-0.9231.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9231\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2645\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9358974099. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1834266335. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.10 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m38.01 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.09 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [26] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m27\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 52)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 53/54\n", + "256/256 [==============================] - 19s 72ms/step - loss: 0.2499 - accuracy: 0.9019 - val_loss: 0.2369 - val_accuracy: 0.9231 - lr: 0.0100\n", + "Epoch 54/54\n", + "256/256 [==============================] - 17s 68ms/step - loss: 0.1871 - accuracy: 0.9260 - val_loss: 0.2163 - val_accuracy: 0.9183 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9231}, \u001b[0m\u001b[0;33mloss{0.2163}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9359}, loss{0.1834}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9183\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2163\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9358974099. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1834266335. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m96.89 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.70 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.19 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [27] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m28\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 54)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 55/56\n", + "256/256 [==============================] - 20s 72ms/step - loss: 0.2469 - accuracy: 0.8970 - val_loss: 0.1939 - val_accuracy: 0.9231 - lr: 0.0100\n", + "Epoch 56/56\n", + "256/256 [==============================] - 18s 68ms/step - loss: 0.1926 - accuracy: 0.9285 - val_loss: 0.2572 - val_accuracy: 0.8878 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9231}, \u001b[0m\u001b[0;33mloss{0.1939}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9359}, loss{0.1834}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.8878\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2572\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9358974099. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1834266335. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.86 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m38.16 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.70 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [28] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m29\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 56)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 57/58\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.2306 - accuracy: 0.9097 - val_loss: 0.2215 - val_accuracy: 0.9359 - lr: 0.0100\n", + "Epoch 58/58\n", + "256/256 [==============================] - 18s 68ms/step - loss: 0.1796 - accuracy: 0.9287 - val_loss: 0.2737 - val_accuracy: 0.9215 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9359}, \u001b[0m\u001b[0;33mloss{0.2215}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9359}, loss{0.1834}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9215\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2737\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9358974099. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1834266335. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m96.97 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.71 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.26 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [29] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m30\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 58)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 59/60\n", + "256/256 [==============================] - 20s 72ms/step - loss: 0.2441 - accuracy: 0.9021 - val_loss: 0.1955 - val_accuracy: 0.9231 - lr: 0.0100\n", + "Epoch 60/60\n", + "256/256 [==============================] - 18s 68ms/step - loss: 0.1946 - accuracy: 0.9246 - val_loss: 0.3419 - val_accuracy: 0.8974 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9231}, \u001b[0m\u001b[0;33mloss{0.1955}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9359}, loss{0.1834}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.8974\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3419\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9358974099. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1834266335. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.54 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m38.05 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.49 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [30] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m31\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 60)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 61/62\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.2511 - accuracy: 0.9014 - val_loss: 0.2011 - val_accuracy: 0.9054 - lr: 0.0100\n", + "Epoch 62/62\n", + "256/256 [==============================] - 18s 68ms/step - loss: 0.1863 - accuracy: 0.9243 - val_loss: 0.2396 - val_accuracy: 0.8990 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9054}, \u001b[0m\u001b[0;33mloss{0.2011}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9359}, loss{0.1834}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.8990\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2396\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9358974099. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1834266335. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.18 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.66 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.52 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [31] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m32\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 62)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 63/64\n", + "256/256 [==============================] - 19s 72ms/step - loss: 0.2098 - accuracy: 0.9180 - val_loss: 0.1952 - val_accuracy: 0.9343 - lr: 0.0100\n", + "Epoch 64/64\n", + "256/256 [==============================] - 17s 67ms/step - loss: 0.1582 - accuracy: 0.9402 - val_loss: 0.2406 - val_accuracy: 0.9151 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9343}, \u001b[0m\u001b[0;33mloss{0.1952}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9359}, loss{0.1834}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9151\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2406\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9358974099. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1834266335. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m96.16 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.61 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m58.55 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [32] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m33\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 64)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 65/66\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.2209 - accuracy: 0.9126 - val_loss: 0.3849 - val_accuracy: 0.8638 - lr: 0.0100\n", + "Epoch 66/66\n", + "256/256 [==============================] - 18s 70ms/step - loss: 0.1848 - accuracy: 0.9277 - val_loss: 0.1687 - val_accuracy: 0.9375 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-066-0.9375.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9375\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1687\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model accuracy from 0.935897 to 0.937500. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model loss from 0.18342663 to 0.16869387. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m98.57 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m38.00 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m60.57 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [33] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m34\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 66)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 67/68\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.2264 - accuracy: 0.9099 - val_loss: 0.1725 - val_accuracy: 0.9247 - lr: 0.0100\n", + "Epoch 68/68\n", + "256/256 [==============================] - 18s 70ms/step - loss: 0.1751 - accuracy: 0.9358 - val_loss: 0.1851 - val_accuracy: 0.9263 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9263}, \u001b[0m\u001b[0;33mloss{0.1725}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9375}, loss{0.1687}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9263\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1851\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9375000000. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.02 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m38.12 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m58.90 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [34] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m35\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 68)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 69/70\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.2261 - accuracy: 0.9111 - val_loss: 0.2452 - val_accuracy: 0.8958 - lr: 0.0100\n", + "Epoch 70/70\n", + "256/256 [==============================] - 18s 70ms/step - loss: 0.1780 - accuracy: 0.9319 - val_loss: 0.2020 - val_accuracy: 0.9263 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9263}, \u001b[0m\u001b[0;33mloss{0.2020}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9375}, loss{0.1687}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9263\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2020\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9375000000. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.51 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m38.28 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.24 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [35] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m36\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 70)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 71/72\n", + "256/256 [==============================] - 20s 72ms/step - loss: 0.2328 - accuracy: 0.9097 - val_loss: 0.2041 - val_accuracy: 0.9215 - lr: 0.0100\n", + "Epoch 72/72\n", + "256/256 [==============================] - 18s 68ms/step - loss: 0.1806 - accuracy: 0.9326 - val_loss: 0.2556 - val_accuracy: 0.8910 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9215}, \u001b[0m\u001b[0;33mloss{0.2041}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9375}, loss{0.1687}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.8910\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2556\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9375000000. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m96.75 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m38.04 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m58.71 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [36] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m37\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 72)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 73/74\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.2290 - accuracy: 0.9094 - val_loss: 0.2032 - val_accuracy: 0.9151 - lr: 0.0100\n", + "Epoch 74/74\n", + "256/256 [==============================] - 18s 69ms/step - loss: 0.1739 - accuracy: 0.9329 - val_loss: 0.1716 - val_accuracy: 0.9263 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9263}, \u001b[0m\u001b[0;33mloss{0.1716}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9375}, loss{0.1687}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9263\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1716\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9375000000. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m96.40 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.78 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m58.62 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [37] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m38\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 74)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 75/76\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.2069 - accuracy: 0.9199 - val_loss: 0.2228 - val_accuracy: 0.9279 - lr: 0.0100\n", + "Epoch 76/76\n", + "256/256 [==============================] - 18s 71ms/step - loss: 0.1692 - accuracy: 0.9319 - val_loss: 0.2496 - val_accuracy: 0.9151 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9279}, \u001b[0m\u001b[0;33mloss{0.2228}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9375}, loss{0.1687}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9151\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2496\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9375000000. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.44 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m38.26 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.18 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [38] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m39\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 76)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 77/78\n", + "256/256 [==============================] - 20s 75ms/step - loss: 0.2157 - accuracy: 0.9155 - val_loss: 0.3724 - val_accuracy: 0.8526 - lr: 0.0100\n", + "Epoch 78/78\n", + "256/256 [==============================] - 19s 73ms/step - loss: 0.1656 - accuracy: 0.9370 - val_loss: 0.1861 - val_accuracy: 0.9343 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9343}, \u001b[0m\u001b[0;33mloss{0.1861}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9375}, loss{0.1687}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9343\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1861\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9375000000. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m113.36 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m40.16 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m73.20 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [39] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m40\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 78)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 79/80\n", + "256/256 [==============================] - 20s 74ms/step - loss: 0.2160 - accuracy: 0.9158 - val_loss: 0.2195 - val_accuracy: 0.9006 - lr: 0.0100\n", + "Epoch 80/80\n", + "256/256 [==============================] - 18s 70ms/step - loss: 0.1609 - accuracy: 0.9397 - val_loss: 0.3028 - val_accuracy: 0.8974 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9006}, \u001b[0m\u001b[0;33mloss{0.2195}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9375}, loss{0.1687}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.8974\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.3028\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9375000000. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m112.30 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m38.98 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m73.32 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [40] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m41\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 80)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 81/82\n", + "256/256 [==============================] - 20s 74ms/step - loss: 0.2166 - accuracy: 0.9170 - val_loss: 0.2219 - val_accuracy: 0.9167 - lr: 0.0100\n", + "Epoch 82/82\n", + "256/256 [==============================] - 18s 70ms/step - loss: 0.1595 - accuracy: 0.9397 - val_loss: 0.6414 - val_accuracy: 0.7885 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9167}, \u001b[0m\u001b[0;33mloss{0.2219}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9375}, loss{0.1687}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.7885\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.6414\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9375000000. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m112.83 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m39.32 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m73.50 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [41] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m42\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 82)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33m└───Shuffling data...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;31m- Debug DP Sample dir: \u001b[0m\u001b[0;32mSamples/TSR_SUB_400_y2024_m04_d06-h17_m16_s19\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 83/84\n", + "256/256 [==============================] - 20s 74ms/step - loss: 0.2083 - accuracy: 0.9172 - val_loss: 0.2138 - val_accuracy: 0.9311 - lr: 0.0100\n", + "Epoch 84/84\n", + "256/256 [==============================] - 18s 71ms/step - loss: 0.1555 - accuracy: 0.9387 - val_loss: 0.2416 - val_accuracy: 0.9263 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9311}, \u001b[0m\u001b[0;33mloss{0.2138}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9375}, loss{0.1687}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9263\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2417\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9375000000. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m127.33 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m39.40 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m87.93 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [42] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m43\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 84)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 85/86\n", + "256/256 [==============================] - 21s 75ms/step - loss: 0.2064 - accuracy: 0.9197 - val_loss: 0.3617 - val_accuracy: 0.8782 - lr: 0.0100\n", + "Epoch 86/86\n", + "256/256 [==============================] - 19s 72ms/step - loss: 0.1588 - accuracy: 0.9363 - val_loss: 0.2461 - val_accuracy: 0.9119 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9119}, \u001b[0m\u001b[0;33mloss{0.2461}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9375}, loss{0.1687}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9119\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2461\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9375000000. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m113.20 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m40.04 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m73.16 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [43] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m44\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 86)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 87/88\n", + "256/256 [==============================] - 21s 75ms/step - loss: 0.2267 - accuracy: 0.9109 - val_loss: 0.2034 - val_accuracy: 0.9359 - lr: 0.0100\n", + "Epoch 88/88\n", + "256/256 [==============================] - 18s 71ms/step - loss: 0.1806 - accuracy: 0.9314 - val_loss: 0.2390 - val_accuracy: 0.9006 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9359}, \u001b[0m\u001b[0;33mloss{0.2034}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9375}, loss{0.1687}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9006\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2390\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9375000000. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m113.51 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m39.89 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m73.62 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [44] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m45\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 88)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 89/90\n", + "256/256 [==============================] - 20s 74ms/step - loss: 0.2210 - accuracy: 0.9124 - val_loss: 0.1992 - val_accuracy: 0.9151 - lr: 0.0100\n", + "Epoch 90/90\n", + "256/256 [==============================] - 19s 73ms/step - loss: 0.1686 - accuracy: 0.9368 - val_loss: 0.1782 - val_accuracy: 0.9359 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9359}, \u001b[0m\u001b[0;33mloss{0.1782}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9375}, loss{0.1687}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9359\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1782\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9375000000. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m115.22 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m40.09 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m75.13 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [45] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m46\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 90)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 91/92\n", + "256/256 [==============================] - 20s 74ms/step - loss: 0.2121 - accuracy: 0.9187 - val_loss: 0.1879 - val_accuracy: 0.9391 - lr: 0.0100\n", + "Epoch 92/92\n", + "256/256 [==============================] - 18s 71ms/step - loss: 0.1643 - accuracy: 0.9370 - val_loss: 0.1969 - val_accuracy: 0.9279 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-091-0.9391.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9391\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1879\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model accuracy from 0.937500 to 0.939103. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m116.99 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m39.59 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m77.39 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [46] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m47\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 92)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 93/94\n", + "256/256 [==============================] - 20s 75ms/step - loss: 0.1960 - accuracy: 0.9255 - val_loss: 0.1750 - val_accuracy: 0.9407 - lr: 0.0100\n", + "Epoch 94/94\n", + "256/256 [==============================] - 18s 72ms/step - loss: 0.1516 - accuracy: 0.9487 - val_loss: 0.1977 - val_accuracy: 0.9343 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-093-0.9407.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9407\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1750\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model accuracy from 0.939103 to 0.940705. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m114.54 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m39.79 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m74.75 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [47] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m48\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 94)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 95/96\n", + "256/256 [==============================] - 20s 75ms/step - loss: 0.2014 - accuracy: 0.9263 - val_loss: 0.2266 - val_accuracy: 0.9054 - lr: 0.0100\n", + "Epoch 96/96\n", + "256/256 [==============================] - 19s 72ms/step - loss: 0.1636 - accuracy: 0.9377 - val_loss: 0.2173 - val_accuracy: 0.9183 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9183}, \u001b[0m\u001b[0;33mloss{0.2173}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9407}, loss{0.1687}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9183\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2173\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9407051206. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m114.56 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m39.93 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m74.63 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [48] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m49\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 96)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 97/98\n", + "256/256 [==============================] - 21s 75ms/step - loss: 0.2336 - accuracy: 0.9080 - val_loss: 0.2019 - val_accuracy: 0.9087 - lr: 0.0100\n", + "Epoch 98/98\n", + "256/256 [==============================] - 18s 72ms/step - loss: 0.1643 - accuracy: 0.9368 - val_loss: 0.2377 - val_accuracy: 0.9006 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9087}, \u001b[0m\u001b[0;33mloss{0.2019}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9407}, loss{0.1687}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9022\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2377\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9407051206. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m115.94 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m39.97 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m75.97 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [49] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m50\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 98)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 99/100\n", + "256/256 [==============================] - 20s 74ms/step - loss: 0.2086 - accuracy: 0.9204 - val_loss: 0.1710 - val_accuracy: 0.9343 - lr: 0.0100\n", + "Epoch 100/100\n", + "256/256 [==============================] - 18s 72ms/step - loss: 0.1562 - accuracy: 0.9431 - val_loss: 0.2099 - val_accuracy: 0.9279 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9343}, \u001b[0m\u001b[0;33mloss{0.1710}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9407}, loss{0.1687}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9279\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2099\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9407051206. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m116.69 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m39.53 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m77.16 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [50] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m51\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 100)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 101/102\n", + "256/256 [==============================] - 19s 70ms/step - loss: 0.2198 - accuracy: 0.9097 - val_loss: 0.1792 - val_accuracy: 0.9423 - lr: 0.0100\n", + "Epoch 102/102\n", + "256/256 [==============================] - 17s 68ms/step - loss: 0.1692 - accuracy: 0.9368 - val_loss: 0.1912 - val_accuracy: 0.9359 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-101-0.9423.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9423\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1792\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model accuracy from 0.940705 to 0.942308. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.58 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.34 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m60.24 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [51] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m52\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 102)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 103/104\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.2161 - accuracy: 0.9177 - val_loss: 0.2163 - val_accuracy: 0.9263 - lr: 0.0100\n", + "Epoch 104/104\n", + "256/256 [==============================] - 18s 70ms/step - loss: 0.1614 - accuracy: 0.9385 - val_loss: 0.2004 - val_accuracy: 0.9295 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9295}, \u001b[0m\u001b[0;33mloss{0.2004}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9423}, loss{0.1687}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9295\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2004\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9423077106. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.62 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m38.22 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.40 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [52] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m53\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 104)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 105/106\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.1911 - accuracy: 0.9229 - val_loss: 0.2297 - val_accuracy: 0.9119 - lr: 0.0100\n", + "Epoch 106/106\n", + "256/256 [==============================] - 18s 70ms/step - loss: 0.1442 - accuracy: 0.9419 - val_loss: 0.1748 - val_accuracy: 0.9327 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9327}, \u001b[0m\u001b[0;33mloss{0.1748}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9423}, loss{0.1687}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9327\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1748\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9423077106. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1686938703. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.73 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m38.23 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.50 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [53] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m54\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 106)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 107/108\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.1992 - accuracy: 0.9219 - val_loss: 0.2235 - val_accuracy: 0.9279 - lr: 0.0100\n", + "Epoch 108/108\n", + "256/256 [==============================] - 18s 70ms/step - loss: 0.1593 - accuracy: 0.9368 - val_loss: 0.1652 - val_accuracy: 0.9391 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0;33mLoading the best weights...\u001b[0m\n", + "\u001b[0;33mLoading weights from file cache\\model_SUB_checkpoint-108-0.9391.h5...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9391\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1652\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9423077106. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32mImproved model loss from 0.16869387 to 0.16520111. \u001b[0m\u001b[0;96mSaving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;36mSaving full model H5 format...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m98.95 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m38.19 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m60.76 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [54] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m55\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 108)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 109/110\n", + "256/256 [==============================] - 19s 72ms/step - loss: 0.2143 - accuracy: 0.9177 - val_loss: 0.2027 - val_accuracy: 0.9263 - lr: 0.0100\n", + "Epoch 110/110\n", + "256/256 [==============================] - 18s 70ms/step - loss: 0.1634 - accuracy: 0.9404 - val_loss: 0.1778 - val_accuracy: 0.9279 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9279}, \u001b[0m\u001b[0;33mloss{0.1778}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9423}, loss{0.1652}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9279\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1778\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9423077106. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1652011126. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.63 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m38.28 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.35 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [55] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m56\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 110)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 111/112\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.2108 - accuracy: 0.9258 - val_loss: 0.2547 - val_accuracy: 0.9087 - lr: 0.0100\n", + "Epoch 112/112\n", + "256/256 [==============================] - 18s 70ms/step - loss: 0.1574 - accuracy: 0.9424 - val_loss: 0.1902 - val_accuracy: 0.9359 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9359}, \u001b[0m\u001b[0;33mloss{0.1902}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9423}, loss{0.1652}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9359\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1902\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9423077106. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1652011126. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.37 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.95 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.42 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [56] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m57\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 112)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 113/114\n", + "256/256 [==============================] - 19s 70ms/step - loss: 0.2180 - accuracy: 0.9155 - val_loss: 0.1820 - val_accuracy: 0.9311 - lr: 0.0100\n", + "Epoch 114/114\n", + "256/256 [==============================] - 18s 68ms/step - loss: 0.1632 - accuracy: 0.9392 - val_loss: 0.2201 - val_accuracy: 0.9375 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9375}, \u001b[0m\u001b[0;33mloss{0.1820}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9423}, loss{0.1652}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9375\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2201\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9423077106. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1652011126. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.31 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.53 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.78 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [57] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m58\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 114)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 115/116\n", + "256/256 [==============================] - 19s 70ms/step - loss: 0.2035 - accuracy: 0.9199 - val_loss: 0.1707 - val_accuracy: 0.9423 - lr: 0.0100\n", + "Epoch 116/116\n", + "256/256 [==============================] - 17s 67ms/step - loss: 0.1477 - accuracy: 0.9448 - val_loss: 0.1817 - val_accuracy: 0.9407 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9423}, \u001b[0m\u001b[0;33mloss{0.1707}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9423}, loss{0.1652}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9407\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1817\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9423077106. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1652011126. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.22 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.17 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m60.05 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [58] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m59\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 116)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 117/118\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.2029 - accuracy: 0.9182 - val_loss: 0.1871 - val_accuracy: 0.9423 - lr: 0.0100\n", + "Epoch 118/118\n", + "256/256 [==============================] - 18s 68ms/step - loss: 0.1554 - accuracy: 0.9385 - val_loss: 0.2191 - val_accuracy: 0.9423 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9423}, \u001b[0m\u001b[0;33mloss{0.1871}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9423}, loss{0.1652}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9423\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2191\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9423077106. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1652011126. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.40 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.52 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.87 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [59] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m60\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 118)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 119/120\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.2009 - accuracy: 0.9226 - val_loss: 0.2220 - val_accuracy: 0.9311 - lr: 0.0100\n", + "Epoch 120/120\n", + "256/256 [==============================] - 18s 69ms/step - loss: 0.1538 - accuracy: 0.9412 - val_loss: 0.2040 - val_accuracy: 0.9391 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9391}, \u001b[0m\u001b[0;33mloss{0.2040}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9423}, loss{0.1652}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9391\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2040\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9423077106. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1652011126. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.87 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.98 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.90 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [60] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m61\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 120)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 121/122\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.2148 - accuracy: 0.9141 - val_loss: 0.2356 - val_accuracy: 0.9103 - lr: 0.0100\n", + "Epoch 122/122\n", + "256/256 [==============================] - 18s 69ms/step - loss: 0.1603 - accuracy: 0.9419 - val_loss: 0.2000 - val_accuracy: 0.9279 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9279}, \u001b[0m\u001b[0;33mloss{0.2000}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9423}, loss{0.1652}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9279\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2000\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9423077106. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1652011126. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.83 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.92 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.91 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [61] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m62\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 122)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 123/124\n", + "256/256 [==============================] - 19s 70ms/step - loss: 0.2033 - accuracy: 0.9253 - val_loss: 0.2193 - val_accuracy: 0.9215 - lr: 0.0100\n", + "Epoch 124/124\n", + "256/256 [==============================] - 18s 69ms/step - loss: 0.1487 - accuracy: 0.9451 - val_loss: 0.1861 - val_accuracy: 0.9359 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9359}, \u001b[0m\u001b[0;33mloss{0.1861}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9423}, loss{0.1652}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9359\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.1861\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9423077106. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1652011126. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.79 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.82 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.97 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [62] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m63\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 124)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 125/126\n", + "256/256 [==============================] - 19s 70ms/step - loss: 0.1956 - accuracy: 0.9214 - val_loss: 0.1782 - val_accuracy: 0.9391 - lr: 0.0100\n", + "Epoch 126/126\n", + "256/256 [==============================] - 18s 68ms/step - loss: 0.1385 - accuracy: 0.9497 - val_loss: 0.2231 - val_accuracy: 0.9327 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9391}, \u001b[0m\u001b[0;33mloss{0.1782}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9423}, loss{0.1652}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9327\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2231\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9423077106. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1652011126. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.34 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.35 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.99 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [63] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m64\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 126)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 127/128\n", + "256/256 [==============================] - 19s 71ms/step - loss: 0.1836 - accuracy: 0.9285 - val_loss: 0.2193 - val_accuracy: 0.9343 - lr: 0.0100\n", + "Epoch 128/128\n", + "256/256 [==============================] - 18s 68ms/step - loss: 0.1325 - accuracy: 0.9482 - val_loss: 0.2644 - val_accuracy: 0.9022 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9343}, \u001b[0m\u001b[0;33mloss{0.2193}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9423}, loss{0.1652}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9038\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2644\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9423077106. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1652011126. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m97.23 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.55 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.68 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [64] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m65\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 128)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 129/130\n", + "256/256 [==============================] - 19s 70ms/step - loss: 0.2039 - accuracy: 0.9202 - val_loss: 0.2487 - val_accuracy: 0.9295 - lr: 0.0100\n", + "Epoch 130/130\n", + "256/256 [==============================] - 18s 68ms/step - loss: 0.1518 - accuracy: 0.9419 - val_loss: 0.2461 - val_accuracy: 0.9167 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9295}, \u001b[0m\u001b[0;33mloss{0.2461}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9423}, loss{0.1652}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9167\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2461\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9423077106. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1652011126. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m96.95 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.40 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.55 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [65] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m66\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 130)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 131/132\n", + "256/256 [==============================] - 19s 70ms/step - loss: 0.2005 - accuracy: 0.9185 - val_loss: 0.2076 - val_accuracy: 0.9423 - lr: 0.0100\n", + "Epoch 132/132\n", + "256/256 [==============================] - 17s 67ms/step - loss: 0.1488 - accuracy: 0.9436 - val_loss: 0.2322 - val_accuracy: 0.9407 - lr: 0.0100\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9423}, \u001b[0m\u001b[0;33mloss{0.2076}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9423}, loss{0.1652}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9407\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2322\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9423077106. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1652011126. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m96.76 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m37.10 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.66 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [66] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m67\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 132)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 133/134\n", + "256/256 [==============================] - ETA: 0s - loss: 0.2081 - accuracy: 0.9175\n", + "Epoch 133: ReduceLROnPlateau reducing learning rate to 0.009819999780505895.\n", + "256/256 [==============================] - 20s 74ms/step - loss: 0.2081 - accuracy: 0.9175 - val_loss: 0.1831 - val_accuracy: 0.9359 - lr: 0.0100\n", + "Epoch 134/134\n", + "256/256 [==============================] - 19s 73ms/step - loss: 0.1577 - accuracy: 0.9419 - val_loss: 0.2779 - val_accuracy: 0.9215 - lr: 0.0098\n", + "\u001b[0;32mSubset training done.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mNot loading weights\u001b[0m\u001b[0;32m[\u001b[0m\u001b[0;94mBSR:\u001b[0m\u001b[0;33macc{0.9359}, \u001b[0m\u001b[0;33mloss{0.1831}\u001b[0m\u001b[0;95m|\u001b[0m\u001b[0;94mBTR:\u001b[0m\u001b[0;32macc{0.9423}, loss{0.1652}]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test acc: \u001b[0m\u001b[0;32m0.9215\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mModel Test loss: \u001b[0m\u001b[0;32m0.2779\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel accuracy did not improve from 0.9423077106. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;91mModel loss did not improve from 0.1652011126. Not saving model.\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;32m(GPU-MEM)\u001b[0m\u001b[0;36m----[free: 17.27GB, used: 6.73GB, total, 24.00GB]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(FULL): \u001b[0m\u001b[0;32m99.62 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(SUBo): \u001b[0m\u001b[0;32m39.73 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTime taken for epoch(OTHERo): \u001b[0m\u001b[0;32m59.89 \u001b[0m\u001b[0;36msec\u001b[0m\n", + "\u001b[0;36m<---------------------------------------|Epoch [67] END|--------------------------------------->\u001b[0m\n", + "\u001b[0m\n", + "\u001b[0m\u001b[0mEpoch: \u001b[0m\u001b[0;36m68\u001b[0m\u001b[0m/\u001b[0m\u001b[0;32m489 (TSEC: 134)\u001b[0m\u001b[0;34m | \u001b[0m\u001b[0;32m[Stage 2]\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mTaking a subset of \u001b[0m\u001b[0;32m[|4096|AdvSubset:True]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;33mPreparing train data...\u001b[0m\n", + "\u001b[0;33m- Augmenting Image Data...\u001b[0m\n", + "\u001b[0;33m- Normalizing Image Data...\u001b[0m\n", + "\u001b[0m\u001b[0m\u001b[0;33mSetting training subset epoch.c to \u001b[0m\u001b[0;32m[2]\u001b[0m\u001b[0;33m...\u001b[0m\n", + "\u001b[0;32mTraining on subset...\u001b[0m\n", + "Epoch 135/136\n", + "256/256 [==============================] - 21s 77ms/step - loss: 0.2021 - accuracy: 0.9231 - val_loss: 0.1942 - val_accuracy: 0.9375 - lr: 0.0098\n", + "Epoch 136/136\n", + "256/256 [==============================] - ETA: 0s - loss: 0.1453 - accuracy: 0.9470" ] } ], @@ -7429,8 +9013,8 @@ "# Hyperparameters for training the model:\n", "max_epoch = 489 # max_epoch: Maximum number of epochs to train for. Use >=256 for full fine-tuning of large models.\n", "max_EST_epoch = 164 # max_EST_epoch: Maximum number epochs to trian the model estimation for the One Cycle UFTS.\n", - "subset_epoch = 6 # subset_epoch: Number of epochs to train each subset.\n", - "subset_epoch_FT = 6 # subset_epoch_FT: subset_epoch after pre-training epochs.\n", + "subset_epoch = 2 # subset_epoch: Number of epochs to train each subset.\n", + "subset_epoch_FT = 2 # subset_epoch_FT: subset_epoch after pre-training epochs.\n", "Stage1_epoch = 26 # Stage1_epoch: Number of pre-training epochs. Use >=24 for large models or 0/1 for fine-tuning only.\n", "subset_size = 4096 # subset_size: Size of each training subset. Common values: 512, 1024, 2048, 3200, 4096, 5846, 8192.\n", "Conf_batch_size_REV2 = 16 # Conf_batch_size_REV2: Batch size.\n", @@ -7442,7 +9026,7 @@ "MIN_LR = 0.0005 # MIN_LR: Minimum learning rate.\n", "RES_LR = 0.006 # RES_LR: Resuming learning rate.\n", "ReduceLROnPlateau_factor = 0.982 # ReduceLROnPlateau_factor: ReduceLROnPlateau factor. (Lr = Factor * Lr_prev)\n", - "ReduceLROnPlateau_patience = 10 # ReduceLROnPlateau_patience: ReduceLROnPlateau patience. (pt = ReduceLROnPlateau_patience * subset_epochs)\n", + "ReduceLROnPlateau_patience = 16 # ReduceLROnPlateau_patience: ReduceLROnPlateau patience. (pt = ReduceLROnPlateau_patience * subset_epochs)\n", "Use_OneCycleLr = False # Use_OneCycleLr: Use OneCycleLr if True. if false, use ReduceLROnPlateau.\n", "OneCycleLr_UFTS = False # OneCycleLr_UFTS: Set the OneCycleLr max epochs to the estimated full training SUB epochs. (DEC_LR and MIN_LR dont have any effect if True)\n", "Debug_OUTPUT_DPS = True # Debug_OUTPUT_DPS: Output debug image samples if True.\n", @@ -7892,7 +9476,8 @@ "\n", " print_Color(\"- Augmenting Image Data...\", [\"yellow\"])\n", " train_SUB_augmented_images = train_SUB_datagen.flow(\n", - " x_SUB_train * 255, y_SUB_train, shuffle=False, batch_size=len(x_SUB_train)\n", + " x_SUB_train * 255, y_SUB_train, shuffle=False, batch_size=len(x_SUB_train),\n", + " num_workers=16\n", " ).next()\n", " print_Color(\"- Normalizing Image Data...\", [\"yellow\"])\n", " x_SUB_train = normalize_TO_RANGE(train_SUB_augmented_images[0], 0, 255)\n",