TrainonFlickr8K-VGG16.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Copy of Alex Copy of smallmodels CS539","provenance":[{"file_id":"1tGQrlzokPe4PdgCrIR2V-lUq7iYZjOPc","timestamp":1607110433730},{"file_id":"1ED_medQZ1MXRQTO499NhkvPThTBXD17K","timestamp":1605566132298},{"file_id":"1WhSxWhfnz6mFdVmoXAqV4uSWyGK1Y0xX","timestamp":1605401537608}],"collapsed_sections":[],"toc_visible":true,"machine_shape":"hm"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU","widgets":{"application/vnd.jupyter.widget-state+json":{"c925918c93024132a6c11ab94ede28e3":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_6bc53e017bc64c89968b3008ad67f938","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_5087b99f94c54d2aace04be1e4eeecea","IPY_MODEL_6678eab665c04c50af9819725af99a9a"]}},"6bc53e017bc64c89968b3008ad67f938":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"5087b99f94c54d2aace04be1e4eeecea":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_f9525ca7d4da4336820ba9637765d9b3","_dom_classes":[],"description":"100%","_model_name":"FloatProgressModel","bar_style":"success","max":400000,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":400000,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_79930a2ebfa9467a92c1026e404f0d01"}},"6678eab665c04c50af9819725af99a9a":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_43a97e34f5bd4de8a0c0f5ce160ea700","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 400000/400000 [00:18&lt;00:00, 21405.83it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_4359d75f5e4444419c5b8ee5dabea4fa"}},"f9525ca7d4da4336820ba9637765d9b3":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"79930a2ebfa9467a92c1026e404f0d01":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"43a97e34f5bd4de8a0c0f5ce160ea700":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"4359d75f5e4444419c5b8ee5dabea4fa":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}}},"cells":[{"cell_type":"code","metadata":{"id":"yPk54N648OuP","executionInfo":{"status":"ok","timestamp":1607120483736,"user_tz":300,"elapsed":1834,"user":{"displayName":"zixuan zhao","photoUrl":"","userId":"16714798905292107392"}}},"source":["import tensorflow as tf\n","import string,os,glob\n","import tensorflow.keras.callbacks\n","from tensorflow.keras.applications import MobileNet\n","\n","from tensorflow.keras.applications.vgg16 import VGG16\n","from tensorflow.keras.applications.inception_v3 import InceptionV3\n","from tensorflow.keras.applications.inception_v3 import preprocess_input\n","from tensorflow.keras.preprocessing.image import img_to_array\n","from tqdm import tqdm_notebook\n","from tensorflow.keras.preprocessing.image import load_img\n","import pickle\n","from time import time\n","import numpy as np\n","from PIL import Image\n","from tensorflow.keras.models import Sequential\n","from tensorflow.keras.layers import LSTM, Bidirectional,Concatenate,Embedding, TimeDistributed, Dense, RepeatVector,Activation, Flatten, Reshape, concatenate, Dropout\n","from tensorflow.keras.optimizers import Adam, RMSprop\n","from tensorflow.keras import Input\n","from tensorflow.keras import optimizers\n","from tensorflow.keras.models import Model\n","from tensorflow.keras import layers\n","\n","from tensorflow.keras.preprocessing.sequence import pad_sequences\n","from tensorflow.keras.utils import to_categorical\n","import matplotlib.pyplot as plt\n","import random,copy\n","from tensorflow.keras.callbacks import ModelCheckpoint\n","from tensorflow.keras.models import load_model"],"execution_count":1,"outputs":[]},{"cell_type":"code","metadata":{"id":"JAg62p1s9s60","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1607120503522,"user_tz":300,"elapsed":21611,"user":{"displayName":"zixuan zhao","photoUrl":"","userId":"16714798905292107392"}},"outputId":"d9231104-ac6a-4d73-f9cf-5525bf40645c"},"source":["from google.colab import drive\n","drive.mount('/content/drive/')"],"execution_count":2,"outputs":[{"output_type":"stream","text":["Mounted at /content/drive/\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"L27BsnJwA5hV"},"source":["# New Section"]},{"cell_type":"markdown","metadata":{"id":"ml1eSXeoA_E3"},"source":["# New Section"]},{"cell_type":"code","metadata":{"id":"syWN9bwl-gHu","executionInfo":{"status":"ok","timestamp":1607120503523,"user_tz":300,"elapsed":21610,"user":{"displayName":"zixuan zhao","photoUrl":"","userId":"16714798905292107392"}}},"source":["base_folder = '/content/drive/Shareddrives/zhaozixuan67/20FallMLfinal/Project'\n","data_folder = '/content/drive/Shareddrives/zhaozixuan67/20FallMLfinal/Project/Data'"],"execution_count":3,"outputs":[]},{"cell_type":"code","metadata":{"id":"PUI2SG9mUMwH","executionInfo":{"status":"ok","timestamp":1607120503523,"user_tz":300,"elapsed":21609,"user":{"displayName":"zixuan zhao","photoUrl":"","userId":"16714798905292107392"}}},"source":[""],"execution_count":3,"outputs":[]},{"cell_type":"code","metadata":{"id":"jAprg3F9BQls","executionInfo":{"status":"ok","timestamp":1607120503524,"user_tz":300,"elapsed":21608,"user":{"displayName":"zixuan zhao","photoUrl":"","userId":"16714798905292107392"}}},"source":["import pickle\n","def loadmodel(mname):\n","    f=open(mname,\"rb\")\n","    lookup=pickle.load(f)\n","    maxlength=0\n","    for v in lookup.values():\n","        for i in v:\n","            maxlength=max(maxlength,len(i.split(' ')))\n","    \n","    # create the unique word set\n","    lex = set()\n","    for k in lookup.keys():\n","        [lex.update(d.split()) for d in lookup[k]]\n","    return lookup,lex,maxlength"],"execution_count":4,"outputs":[]},{"cell_type":"code","metadata":{"id":"l3swupuNBUfT","executionInfo":{"status":"ok","timestamp":1607120503524,"user_tz":300,"elapsed":21607,"user":{"displayName":"zixuan zhao","photoUrl":"","userId":"16714798905292107392"}}},"source":["import tensorflow.keras.backend as K\n","def data_generator(descriptions, photos, wtoi, max_length, num_photos_batch_size):\n","    # training data for images\n","    # caption\n","    # rest of the caption\n","    # global yield_values\n","    x1,x2,y = [],[],[]\n","    print(\"in gener,lendic \",len(descriptions))\n","\n","    for key,desc_list in descriptions.items():\n","        photo = photos[key]\n","        for desc in desc_list:\n","            # seq is sequence id\n","            seq = [wtoi[word] for word in desc.split() if word in wtoi]\n","            # add the start and end indicator\n","            seq.insert(0,wtoi[START])\n","            seq.append(wtoi[END])\n","            for i in range(1,len(seq)):\n","                in_seq, out_seq = seq[:i],seq[i]\n","                in_seq = pad_sequences([in_seq],maxlen=max_length)[0]\n","                # the out_seq is one-hot encoding\n","                out_seq = to_categorical([out_seq],num_classes=vocab_size)[0]\n","                x1.append(photo)\n","                x2.append(in_seq)\n","                y.append(out_seq)\n","\n","            \n","    print(\"out gener,lenx:\",len(x1))\n","    return [[np.array(x1),np.array(x2)], np.array(y)]\n","\n","\n","            # print([[np.array(x1).shape,np.array(x2).shape], np.array(y).shape])\n","            # yield_values = [[np.array(x1),np.array(x2)], np.array(y)]\n","            # yield [[np.array(x1),np.array(x2)], np.array(y)]"],"execution_count":5,"outputs":[]},{"cell_type":"code","metadata":{"id":"Kh7huG2hBUhz","executionInfo":{"status":"ok","timestamp":1607120503524,"user_tz":300,"elapsed":21605,"user":{"displayName":"zixuan zhao","photoUrl":"","userId":"16714798905292107392"}}},"source":["def buildmodel(structid):\n","    if structid==1:\n","        WIDTH,HEIGHT,OUTPUT_DIM=299,299,41472\n","        inputs1 = Input(shape=(OUTPUT_DIM,),name=\"image_features\") # image features\n","        fe1 = Dropout(0.5,name=\"img_dropout\")(inputs1)\n","        fe2 = Dense(512,activation='relu',name=\"img_dense\")(fe1)\n","        inputs2 = Input(shape=(max_length,),name=\"captions\") # gradually accending caption\n","        se1 = Embedding(vocab_size,embedding_dim,mask_zero=True,name=\"glove_embedding\")(inputs2)\n","        se2 = Dropout(0.5,name=\"text_dropout\")(se1)\n","        se3 = Bidirectional(LSTM(256,name=\"text_LSTM\"))(se2)\n","        decorder1 = layers.add([fe2,se3],name=\"merge\") # 256\n","        decorder2 = Dense(256,activation='relu',name=\"merge_dense\")(decorder1)\n","        outputs = Dense(vocab_size,activation='softmax',name=\"prob_distribution\")(decorder2)\n","        caption_model = Model(inputs=[inputs1,inputs2], outputs=outputs)\n","    elif structid==2:\n","        STRUCT=2\n","        WIDTH,HEIGHT,OUTPUT_DIM=299,299,41472\n","        inputs1 = Input(shape=(OUTPUT_DIM,),name=\"image_features\") # image features\n","        fe1 = Dropout(0.5,name=\"img_dropout\")(inputs1)\n","        fe2 = Dense(128,activation='relu',name=\"img_dense\")(fe1)\n","        inputs2 = Input(shape=(max_length,),name=\"captions\") # gradually accending caption\n","        se1 = Embedding(vocab_size,embedding_dim,mask_zero=True,name=\"glove_embedding\")(inputs2)\n","        se2 = Dropout(0.5,name=\"text_dropout\")(se1)\n","        se3 = LSTM(128,name=\"text_LSTM\")(se2)\n","        decorder1 = Concatenate()([fe2,se3])\n","        decorder2 = Dense(256,activation='relu',name=\"merge_dense\")(decorder1)\n","        outputs = Dense(vocab_size,activation='softmax',name=\"prob_distribution\")(decorder2)\n","        caption_model = Model(inputs=[inputs1,inputs2], outputs=outputs)\n","    elif structid==3:\n","        STRUCT=3\n","        WIDTH,HEIGHT,OUTPUT_DIM=299,299,41472\n","        inputs1 = Input(shape=(OUTPUT_DIM,),name=\"image_features\") # image features\n","        fe1 = Dropout(0.5,name=\"img_dropout\")(inputs1)\n","        fe2 = Dense(256,activation='relu',name=\"img_dense\")(fe1)\n","        inputs2 = Input(shape=(max_length,),name=\"captions\") # gradually accending caption\n","        se1 = Embedding(vocab_size,embedding_dim,mask_zero=True,name=\"glove_embedding\")(inputs2)\n","        se2 = Dropout(0.5,name=\"text_dropout\")(se1)\n","        se3 = Bidirectional(LSTM(128,name=\"text_LSTM\"))(se2)\n","        decorder1 = Concatenate()([fe2,se3])\n","        decorder2 = Dense(256,activation='relu',name=\"merge_dense\")(decorder1)\n","        outputs = Dense(vocab_size,activation='softmax',name=\"prob_distribution\")(decorder2)\n","        caption_model = Model(inputs=[inputs1,inputs2], outputs=outputs)\n","    else:#4\n","        WIDTH,HEIGHT,OUTPUT_DIM=299,299,41472\n","        inputs1 = Input(shape=(OUTPUT_DIM,),name=\"image_features\") # image features\n","        fe1 = Dropout(0.5,name=\"img_dropout\")(inputs1)\n","        fe2 = Dense(128,activation='relu',name=\"img_dense\")(fe1)\n","        inputs2 = Input(shape=(max_length,),name=\"captions\") # gradually accending caption\n","        se1 = Embedding(vocab_size,embedding_dim,mask_zero=True,name=\"glove_embedding\")(inputs2)\n","        se2 = Dropout(0.5,name=\"text_dropout\")(se1)\n","        se3 = LSTM(128,name=\"text_LSTM\",return_sequences=True)(se2)\n","        se4 = LSTM(128,name=\"text_LSTM_2\")(se3)\n","        decorder1 = Concatenate()([fe2,se4])\n","        decorder2 = Dense(256,activation='relu',name=\"merge_dense\")(decorder1)\n","        outputs = Dense(vocab_size,activation='softmax',name=\"prob_distribution\")(decorder2)\n","        caption_model = Model(inputs=[inputs1,inputs2], outputs=outputs)\n","    return caption_model"],"execution_count":6,"outputs":[]},{"cell_type":"code","metadata":{"id":"kWye1cZaBUkr","executionInfo":{"status":"ok","timestamp":1607120503525,"user_tz":300,"elapsed":21605,"user":{"displayName":"zixuan zhao","photoUrl":"","userId":"16714798905292107392"}}},"source":["import gc\n","from tensorflow import keras\n","class LossHistory(keras.callbacks.Callback):\n","    def on_train_begin(self, logs={}):\n","        self.losses = []\n"," \n","    def on_batch_end(self, batch, logs={}):\n","        self.losses.append(logs.get('loss'))\n","\n","def train_model(caption_model,batchsize,model_path,modelid,mname,EPOCHS = 10,):\n","    history = LossHistory()\n","#     checkpoint = ModelCheckpoint(model_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n","#     callbacks_list = [checkpoint]\n","    values = data_generator(train_id_to_captions,encoding_dict,word_to_id,max_length,num_pics_per_batch)\n","    # caption_model.fit_generator(generator,epochs=1,steps_per_epoch=steps)\n","    print(\"fitting\")\n","    print(\"sizecheck: \",len(values[0][0]),len(values[0][1]),len(values[1]))\n","    caption_model.fit(values[0],values[1],batch_size=batchsize,epochs=EPOCHS,validation_split=0.1,verbose=1,callbacks=[history])#,callbacks=callbacks_list)\n","    gc.collect()\n","        #save after every epoch\n","    caption_model.save(model_path)\n","    with open(working_base+\"/alex/alex-loss\"+str(MODELID)+\"-\"+mname[:5]+\"-ep\"+str(EPOCHS)+\".pkl\",'wb') as f:\n","        pickle.dump(history.losses,f)\n","        \n","    print(\"saved\")\n","    "],"execution_count":7,"outputs":[]},{"cell_type":"code","metadata":{"id":"t9xXDbNVrgpM","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1607120503525,"user_tz":300,"elapsed":21600,"user":{"displayName":"zixuan zhao","photoUrl":"","userId":"16714798905292107392"}},"outputId":"acd37334-0794-40e0-c194-3716661c4848"},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":8,"outputs":[{"output_type":"stream","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"tOuqJITaBUnr","executionInfo":{"status":"ok","timestamp":1607120503526,"user_tz":300,"elapsed":21599,"user":{"displayName":"zixuan zhao","photoUrl":"","userId":"16714798905292107392"}}},"source":["def generateCaption(image):\n","    in_text = START\n","    for i in range(max_length):\n","        id_seq = [word_to_id[w] for w in in_text.split() if w in word_to_id.keys()]\n","        id_seq = pad_sequences([id_seq],maxlen=max_length)\n","        yhat = caption_model.predict([image,id_seq],verbose=0)\n","        yhat = np.argmax(yhat)\n","        word = id_to_word[yhat]\n","        in_text += ' ' + word\n","        if word == END:\n","            break\n","    output = in_text.split()\n","    output = output[1:-1]\n","    output = ' '.join(output)\n","    return output"],"execution_count":9,"outputs":[]},{"cell_type":"code","metadata":{"id":"H5-K1a5HnXIi","executionInfo":{"status":"ok","timestamp":1607120503526,"user_tz":300,"elapsed":21598,"user":{"displayName":"zixuan zhao","photoUrl":"","userId":"16714798905292107392"}}},"source":["# VGG16 = VGG16(include_top=False,\n","#     weights=\"imagenet\",\n","#     input_tensor=None,\n","#     input_shape=(299,299,3),\n","#     pooling=None,\n","#     classes=1000,\n","#     classifier_activation=\"softmax\")\n","# VGG16.summary()\n","# encoder = Model(VGG16.input,VGG16.layers[-1].output)\n","# WIDTH,HEIGHT,OUTPUT_DIM=299,299,41472\n","# def encodeImage(img):\n","#     from PIL import Image\n","#     img = img.resize((HEIGHT,WIDTH),Image.ANTIALIAS)\n","#     x = img_to_array(img)\n","#     x = np.expand_dims(x,axis=0)\n","#     x = preprocess_input(x)\n","#     x = encoder.predict(x)\n","#     x = np.reshape(x,OUTPUT_DIM)\n","#     return x"],"execution_count":10,"outputs":[]},{"cell_type":"code","metadata":{"id":"iCmcXoKgULw4","executionInfo":{"status":"ok","timestamp":1607120504642,"user_tz":300,"elapsed":22712,"user":{"displayName":"zixuan zhao","photoUrl":"","userId":"16714798905292107392"}}},"source":["os.chdir('/content/drive/Shareddrives/zhaozixuan67/20FallMLfinal/Project/workspace/alex')"],"execution_count":11,"outputs":[]},{"cell_type":"code","metadata":{"id":"BhuYNLQsBUqT","colab":{"base_uri":"https://localhost:8080/","height":134,"referenced_widgets":["c925918c93024132a6c11ab94ede28e3","6bc53e017bc64c89968b3008ad67f938","5087b99f94c54d2aace04be1e4eeecea","6678eab665c04c50af9819725af99a9a","f9525ca7d4da4336820ba9637765d9b3","79930a2ebfa9467a92c1026e404f0d01","43a97e34f5bd4de8a0c0f5ce160ea700","4359d75f5e4444419c5b8ee5dabea4fa"]},"executionInfo":{"status":"ok","timestamp":1607120532342,"user_tz":300,"elapsed":50407,"user":{"displayName":"zixuan zhao","photoUrl":"","userId":"16714798905292107392"}},"outputId":"122e5aa8-39b6-4f75-a431-db73ab26059d"},"source":["import gc\n","import os\n","epoches=100\n","#datasets=[\"womanwomen-1787-dataset.pkl\",\"manmen-3042-dataset.pkl\",\"girl-1736-dataset.pkl\",\"dog-2300-dataset.pkl\",\"boy-1837-dataset.pkl\"]\n","datasets=[\"full-8092-dataset.pkl\"]\n","embedding_index = {}\n","\n","with open(os.path.join(f'{data_folder}/glove/glove.6B.200d.txt'),encoding='utf-8') as f:\n","    for line in tqdm_notebook(f.readlines()):\n","        vals = line.split()\n","        word = vals[0]\n","        coefs = np.array(vals[1:],dtype='float32')\n","        embedding_index[word] = coefs\n","\n","print(f\"{len(embedding_index)} word vectors\")\n"],"execution_count":12,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:9: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n","Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n","  if __name__ == '__main__':\n"],"name":"stderr"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"c925918c93024132a6c11ab94ede28e3","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=400000.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n","400000 word vectors\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"1fDFt90YUoDH","colab":{"base_uri":"https://localhost:8080/"},"outputId":"0c4afc69-bc7c-4246-f33f-47e288f02df0"},"source":["\n","for ds in datasets:\n","    lookup,lex,max_length=loadmodel(\"/content/drive/Shared drives/zhaozixuan67/20FallMLfinal/Project/Data/smallmodels-dataset/full-8092-dataset.pkl\")\n","    \n","    for MODELID in range(2,4):\n","        print(\"---------------------------------------------train model \",MODELID,\" on \",ds)\n","        #dont read\n","        img_path_list = glob.glob(f\"{data_folder}/Flickr8k_Dataset/Flicker8k_Dataset/*.jpg\")\n","        train_img_names = set(open(f'{data_folder}/Flickr8k_text/Flickr_8k.trainImages.txt').read().strip().split('\\n'))\n","        test_img_names = set(open(f'{data_folder}/Flickr8k_text/Flickr_8k.testImages.txt').read().strip().split('\\n'))\n","\n","        #for windows\n","        # train_img_names = [img_path for img_path in img_path_list if img_path.split('\\\\')[-1] in train_img_names]\n","        # test_img_names = [img_path for img_path in img_path_list if img_path.split('\\\\')[-1] in test_img_names]\n","        \n","        #for Colab(Linux)\n","        train_img_names = [img_path for img_path in img_path_list if img_path.split('/')[-1] in train_img_names]\n","        test_img_names = [img_path for img_path in img_path_list if img_path.split('/')[-1] in test_img_names]\n","\n","        #external_base_path = \"./content/drive/Shared drives/AIGroup/BigDataProject\"\n","        #gdrive_path = \"./content/drive/My Drive/Homework_And_Projects/BigData\"\n","\n","        #working_base = external_base_path\n","        working_base= \"/content/drive/Shareddrives/zhaozixuan67/20FallMLfinal/Project/workspace/alex\"\n","\n","        if not os.path.exists(\"{}/img_encoded.pkl\".format(working_base)):\n","            encoding_dict = {}\n","            for img_path in tqdm_notebook(train_img_names):\n","                img = load_img(img_path,target_size=(HEIGHT,WIDTH))\n","                id = os.path.basename(img_path).split('.')[0]\n","                # print(\"am I read?\")\n","                # print(img)\n","                # print(id)\n","                encoding_dict[id] = encodeImage(img)\n","            with open(\"{}/img_encoded.pkl\".format(working_base),'wb') as f:\n","                pickle.dump(encoding_dict,f)\n","            print(\"image pickle is saved.\")\n","        else:\n","            with open(\"{}/img_encoded.pkl\".format(working_base),'rb') as f:\n","                encoding_dict = pickle.load(f)\n","            print(\"image pickle is loaded\")\n","        print(len(encoding_dict))\n","        oldencoding_dict=copy.copy(encoding_dict)\n","        encoding_dict={}\n","\n","        for k,v in oldencoding_dict.items():\n","            if k in lookup.keys():\n","                encoding_dict[k]=v\n","        print(\"len of encoding dict: \",len(encoding_dict))\n","\n","        if not os.path.exists(\"{}/img_encoded_test.pkl\".format(working_base)):\n","            encoding_test_dict = {}\n","            for img_path in tqdm_notebook(test_img_names):\n","                img = load_img(img_path,target_size=(HEIGHT,WIDTH))\n","                id = os.path.basename(img_path).split('.')[0]\n","                encoding_test_dict[id] = encodeImage(img)\n","            with open(\"{}/img_encoded_test.pkl\".format(working_base),'wb') as f:\n","                pickle.dump(encoding_test_dict,f)\n","            print(\"image pickle (test) is saved.\")\n","        else:\n","            with open(\"{}/img_encoded_test.pkl\".format(working_base),'rb') as f:\n","                encoding_test_dict = pickle.load(f)\n","            print(\"image pickle (test) is loaded\")\n","        oldencoding_test_dict=copy.copy(encoding_test_dict)\n","        encoding_test_dict={}\n","        for k,v in oldencoding_test_dict.items():\n","            if k in lookup.keys():\n","                encoding_test_dict[k]=v\n","        len(encoding_test_dict)\n","        training_captions = []\n","        test_captions = []\n","        train_id_to_captions = {}\n","        test_id_to_captions = {}\n","\n","        train_img_ids = [os.path.basename(img).split('.')[0] for img in train_img_names]\n","        test_img_ids = [os.path.basename(img).split('.')[0] for img in test_img_names]\n","\n","        for id, caption_list in lookup.items():\n","            if id in train_img_ids:\n","                training_captions.extend(caption_list)\n","                train_id_to_captions[id] = caption_list\n","            elif id in test_img_ids:\n","                test_captions.extend(caption_list)\n","                test_id_to_captions[id] = caption_list\n","\n","        len(training_captions),len(test_captions)\n","        randomindex=int(random.random()*(len(train_id_to_captions)-6))\n","        list(train_id_to_captions.values())[randomindex:randomindex+5]\n","        count_threshold = 0\n","        START,END = \"startseq\",\"endseq\"\n","        word_counts = {}\n","        for id,cap in enumerate(training_captions): # for each sentence\n","            for w in cap.split(): # for each word\n","                word_counts[w] = word_counts.get(w,0) + 1\n","        freq_words = [w for w in word_counts if word_counts[w] >= count_threshold]\n","        print(\"{} frequent words\".format(len(freq_words)))\n","        # add two indicators\n","        freq_words.extend([START,END])\n","        id_to_word,word_to_id = {},{}\n","\n","        for id,w in enumerate(freq_words):\n","            word_to_id[w] = id+1\n","            id_to_word[id+1] = w\n","\n","        vocab_size = len(word_to_id) + 1 # \n","        vocab_size\n","        max_length += 2 # add \"startseq\",\"endseq\"\n","        max_length\n","#         embedding_index = {}\n","\n","#         with open(os.path.join(f'{base_folder}/glove.6B.200d.txt'),encoding='utf-8') as f:\n","#             for line in tqdm_notebook(f.readlines()):\n","#                 vals = line.split()\n","#                 word = vals[0]\n","#                 coefs = np.array(vals[1:],dtype='float32')\n","#                 embedding_index[word] = coefs\n","\n","#         print(f\"{len(embedding_index)} word vectors\")\n","        embedding_dim = 200 # the vector length of the embedding\n","\n","        embedding_matrix = np.zeros((vocab_size,embedding_dim))\n","        for word, index in word_to_id.items():\n","            embedding_vector = embedding_index.get(word)\n","            if embedding_vector is not None:\n","                embedding_matrix[index] = embedding_vector\n","        embedding_matrix.shape\n","        caption_model=buildmodel(MODELID)\n","        print(\"modelid: \",MODELID)\n","        caption_model.summary()\n","        from tensorflow.keras.utils import plot_model\n","        print(\"modelid: \",MODELID)\n","        plot_model(caption_model,to_file=\"model\"+str(MODELID)+\"_struct.png\",show_shapes=True,show_layer_names=True,dpi=70)\n","        if MODELID==4:\n","            embedding_layer = caption_model.layers[1]\n","        else:\n","            embedding_layer = caption_model.layers[2]\n","        #embedding_layer = caption_model.layers[2]\n","        embedding_layer.set_weights([embedding_matrix]) # should be a list, like the get_weights()\n","        embedding_layer.trainable = False\n","\n","        opt = optimizers.Adam(learning_rate=0.00001)\n","        caption_model.compile(loss='categorical_crossentropy', optimizer= opt)\n","\n","        num_pics_per_batch = 512\n","        steps = len(training_captions) // num_pics_per_batch\n","        # model_path = \"{}/caption_model.h5\".format(working_base)\n","        model_path = working_base+\"/alex/alex-model\"+str(MODELID)+\"-\"+ds[:5]+\"-ep\"+str(epoches)+\"bs-\"+str(num_pics_per_batch) +\".h5\"\n","        print(\"steps: \",steps)\n","\n","        gc.collect()\n","\n","        train_model(caption_model,512,model_path,MODELID,ds,EPOCHS=epoches)\n","        \n","        # #test\n","        # caption_model.load_weights()\n","        # num=5\n","        # image_names = np.random.choice(list(encoding_test_dict.keys()),num,False)\n","        # for i in range(num):\n","        #     image_name = image_names[i]\n","        #     image_encoding = encoding_test_dict[image_name].reshape((1,-1))\n","        #     full_image_path = f\"{base_folder}/Flickr8k_Dataset/Flicker8k_Dataset/{image_name}.jpg\"\n","        #     img = plt.imread(full_image_path)\n","        #     plt.imshow(img)\n","        #     plt.show()\n","        #     print(\"Caption=\",generateCaption(image_encoding))\n","        #     print(\"Answer=\",lookup[image_name])\n","        #     print(\"_\"*20)\n"],"execution_count":null,"outputs":[{"output_type":"stream","text":["---------------------------------------------train model  2  on  full-8092-dataset.pkl\n","image pickle is loaded\n","6000\n","len of encoding dict:  6000\n","image pickle (test) is loaded\n","7576 frequent words\n","modelid:  2\n","Model: \"functional_1\"\n","__________________________________________________________________________________________________\n","Layer (type)                    Output Shape         Param #     Connected to                     \n","==================================================================================================\n","captions (InputLayer)           [(None, 34)]         0                                            \n","__________________________________________________________________________________________________\n","image_features (InputLayer)     [(None, 41472)]      0                                            \n","__________________________________________________________________________________________________\n","glove_embedding (Embedding)     (None, 34, 200)      1515800     captions[0][0]                   \n","__________________________________________________________________________________________________\n","img_dropout (Dropout)           (None, 41472)        0           image_features[0][0]             \n","__________________________________________________________________________________________________\n","text_dropout (Dropout)          (None, 34, 200)      0           glove_embedding[0][0]            \n","__________________________________________________________________________________________________\n","img_dense (Dense)               (None, 128)          5308544     img_dropout[0][0]                \n","__________________________________________________________________________________________________\n","text_LSTM (LSTM)                (None, 128)          168448      text_dropout[0][0]               \n","__________________________________________________________________________________________________\n","concatenate (Concatenate)       (None, 256)          0           img_dense[0][0]                  \n","                                                                 text_LSTM[0][0]                  \n","__________________________________________________________________________________________________\n","merge_dense (Dense)             (None, 256)          65792       concatenate[0][0]                \n","__________________________________________________________________________________________________\n","prob_distribution (Dense)       (None, 7579)         1947803     merge_dense[0][0]                \n","==================================================================================================\n","Total params: 9,006,387\n","Trainable params: 9,006,387\n","Non-trainable params: 0\n","__________________________________________________________________________________________________\n","modelid:  2\n","steps:  58\n","in gener,lendic  6000\n","out gener,lenx: 306404\n"],"name":"stdout"}]}]}