BERT_Evaluation.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Copy of BERT_Evaluation.ipynb","version":"0.3.2","provenance":[{"file_id":"1JloEVVxQArupg_PLhqZN45lJ4o2BlwSW","timestamp":1556873733180}],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"id":"UqJ0oldGU47P","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"1oM9OfwgVIzQ","colab_type":"code","outputId":"de881b9a-5e4c-4f80-d969-2c43498871e0","executionInfo":{"status":"ok","timestamp":1556872563515,"user_tz":420,"elapsed":572,"user":{"displayName":"ALLEN CHEN","photoUrl":"","userId":"13117472557552131577"}},"colab":{"base_uri":"https://localhost:8080/","height":35}},"source":["from google.colab import drive\n","drive.mount('/content/gdrive')"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"w4PQX1BfVdYx","colab_type":"code","colab":{}},"source":["import sys\n","sys.path.insert(0, \"/content/gdrive/My Drive/cs182_proj/\") # This enables us to import Python libraries in the folder.\n","\n","root_folder = \"/content/gdrive/My Drive/cs182_proj/\""],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"sRoSGLW3AmE4","colab_type":"code","outputId":"754d0499-ff4e-4d8d-ae6c-edbaeff3d05e","executionInfo":{"status":"ok","timestamp":1556748964287,"user_tz":420,"elapsed":9185685,"user":{"displayName":"ALLEN CHEN","photoUrl":"","userId":"13117472557552131577"}},"colab":{"base_uri":"https://localhost:8080/","height":13141}},"source":["# Train the BERT model with the jokes with both questions and answers\n","!python gdrive/My\\ Drive/cs182_proj/bert-master/run_pretraining.py \\\n","--input_file=gdrive/My\\ Drive/cs182_proj/output/processed_data.txt \\\n","--output_dir=gdrive/My\\ Drive/cs182_proj/output/output.txt \\\n","--do_train=True \\\n","--do_eval=True \\\n","--bert_config_file=gdrive/My\\ Drive/cs182_proj/BERT_BASE_DIR/bert_config.json \\\n","--init_checkpoint=gdrive/My\\ Drive/cs182_proj/BERT_BASE_DIR/bert_model.ckpt \\\n","--train_batch_size=32 \\\n","--max_seq_length=128 \\\n","--max_predictions_per_seq=20 \\\n","--num_train_steps=10000 \\\n","--num_warmup_steps=10 \\\n","--learning_rate=2e-5"],"execution_count":0,"outputs":[{"output_type":"stream","text":["INFO:tensorflow:*** Input Files ***\n","INFO:tensorflow:  gdrive/My Drive/cs182_proj/output/processed_data.txt\n","\n","WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n","For more information, please see:\n","  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n","  * https://github.com/tensorflow/addons\n","If you depend on functionality not listed there, please file an issue.\n","\n","WARNING:tensorflow:Estimator's model_fn (<function model_fn_builder.<locals>.model_fn at 0x7ff2b4e45a60>) includes params argument, but params are not passed to Estimator.\n","INFO:tensorflow:Using config: {'_model_dir': 'gdrive/My Drive/cs182_proj/output/output.txt', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n","graph_options {\n","  rewrite_options {\n","    meta_optimizer_iterations: ONE\n","  }\n","}\n",", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7ff2a8205f98>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=1000, num_shards=8, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None), '_cluster': None}\n","INFO:tensorflow:_TPUContext: eval_on_tpu True\n","WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n","INFO:tensorflow:***** Running training *****\n","INFO:tensorflow:  Batch size = 32\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Colocations handled automatically by placer.\n","WARNING:tensorflow:From gdrive/My Drive/cs182_proj/bert-master/run_pretraining.py:368: parallel_interleave (from tensorflow.contrib.data.python.ops.interleave_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use `tf.data.experimental.parallel_interleave(...)`.\n","WARNING:tensorflow:From gdrive/My Drive/cs182_proj/bert-master/run_pretraining.py:385: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use `tf.data.experimental.map_and_batch(...)`.\n","WARNING:tensorflow:From gdrive/My Drive/cs182_proj/bert-master/run_pretraining.py:400: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.cast instead.\n","INFO:tensorflow:Calling model_fn.\n","INFO:tensorflow:Running train on CPU\n","INFO:tensorflow:*** Features ***\n","INFO:tensorflow:  name = input_ids, shape = (32, 128)\n","INFO:tensorflow:  name = input_mask, shape = (32, 128)\n","INFO:tensorflow:  name = masked_lm_ids, shape = (32, 20)\n","INFO:tensorflow:  name = masked_lm_positions, shape = (32, 20)\n","INFO:tensorflow:  name = masked_lm_weights, shape = (32, 20)\n","INFO:tensorflow:  name = next_sentence_labels, shape = (32, 1)\n","INFO:tensorflow:  name = segment_ids, shape = (32, 128)\n","WARNING:tensorflow:From /content/gdrive/My Drive/cs182_proj/bert-master/modeling.py:358: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n","WARNING:tensorflow:From /content/gdrive/My Drive/cs182_proj/bert-master/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use keras.layers.dense instead.\n","INFO:tensorflow:**** Trainable Variables ****\n","INFO:tensorflow:  name = bert/embeddings/word_embeddings:0, shape = (30522, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/token_type_embeddings:0, shape = (2, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/position_embeddings:0, shape = (512, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/pooler/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/pooler/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/output_bias:0, shape = (30522,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/seq_relationship/output_weights:0, shape = (2, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/seq_relationship/output_bias:0, shape = (2,), *INIT_FROM_CKPT*\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/learning_rate_decay_v2.py:321: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Deprecated in favor of operator or tf.math.divide.\n","INFO:tensorflow:Done calling model_fn.\n","INFO:tensorflow:Create CheckpointSaverHook.\n","INFO:tensorflow:Graph was finalized.\n","2019-05-01 19:43:18.456373: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n","2019-05-01 19:43:18.456631: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x18b6ec0 executing computations on platform Host. Devices:\n","2019-05-01 19:43:18.456668: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>\n","2019-05-01 19:43:18.674654: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n","2019-05-01 19:43:18.675311: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x18b7180 executing computations on platform CUDA. Devices:\n","2019-05-01 19:43:18.675346: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): Tesla T4, Compute Capability 7.5\n","2019-05-01 19:43:18.675774: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1433] Found device 0 with properties: \n","name: Tesla T4 major: 7 minor: 5 memoryClockRate(GHz): 1.59\n","pciBusID: 0000:00:04.0\n","totalMemory: 14.73GiB freeMemory: 14.60GiB\n","2019-05-01 19:43:18.675801: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1512] Adding visible gpu devices: 0\n","2019-05-01 19:43:19.212596: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] Device interconnect StreamExecutor with strength 1 edge matrix:\n","2019-05-01 19:43:19.212679: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990]      0 \n","2019-05-01 19:43:19.212694: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1003] 0:   N \n","2019-05-01 19:43:19.213008: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:42] Overriding allow_growth setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n","2019-05-01 19:43:19.213062: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 14115 MB memory) -> physical GPU (device: 0, name: Tesla T4, pci bus id: 0000:00:04.0, compute capability: 7.5)\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use standard file APIs to check for files with this prefix.\n","INFO:tensorflow:Restoring parameters from gdrive/My Drive/cs182_proj/output/output.txt/model.ckpt-20\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1070: get_checkpoint_mtimes (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use standard file utilities to get mtimes.\n","INFO:tensorflow:Running local_init_op.\n","INFO:tensorflow:Done running local_init_op.\n","INFO:tensorflow:Saving checkpoints for 20 into gdrive/My Drive/cs182_proj/output/output.txt/model.ckpt.\n","2019-05-01 19:43:45.964429: I tensorflow/stream_executor/dso_loader.cc:152] successfully opened CUDA library libcublas.so.10.0 locally\n","INFO:tensorflow:global_step/sec: 1.05505\n","INFO:tensorflow:examples/sec: 33.7615\n","INFO:tensorflow:global_step/sec: 1.13764\n","INFO:tensorflow:examples/sec: 36.4043\n","INFO:tensorflow:global_step/sec: 1.1219\n","INFO:tensorflow:examples/sec: 35.9009\n","INFO:tensorflow:global_step/sec: 1.11516\n","INFO:tensorflow:examples/sec: 35.6852\n","INFO:tensorflow:global_step/sec: 1.11421\n","INFO:tensorflow:examples/sec: 35.6547\n","INFO:tensorflow:global_step/sec: 1.1135\n","INFO:tensorflow:examples/sec: 35.632\n","INFO:tensorflow:global_step/sec: 1.11219\n","INFO:tensorflow:examples/sec: 35.5899\n","INFO:tensorflow:global_step/sec: 1.11197\n","INFO:tensorflow:examples/sec: 35.5831\n","INFO:tensorflow:global_step/sec: 1.11208\n","INFO:tensorflow:examples/sec: 35.5866\n","INFO:tensorflow:Saving checkpoints for 1020 into gdrive/My Drive/cs182_proj/output/output.txt/model.ckpt.\n","INFO:tensorflow:global_step/sec: 0.991855\n","INFO:tensorflow:examples/sec: 31.7394\n","INFO:tensorflow:global_step/sec: 1.11949\n","INFO:tensorflow:examples/sec: 35.8237\n","INFO:tensorflow:global_step/sec: 1.11397\n","INFO:tensorflow:examples/sec: 35.6471\n","INFO:tensorflow:global_step/sec: 1.11344\n","INFO:tensorflow:examples/sec: 35.6301\n","INFO:tensorflow:global_step/sec: 1.11232\n","INFO:tensorflow:examples/sec: 35.5943\n","INFO:tensorflow:global_step/sec: 1.11318\n","INFO:tensorflow:examples/sec: 35.6217\n","INFO:tensorflow:global_step/sec: 1.11487\n","INFO:tensorflow:examples/sec: 35.6758\n","INFO:tensorflow:global_step/sec: 1.11296\n","INFO:tensorflow:examples/sec: 35.6148\n","INFO:tensorflow:global_step/sec: 1.11369\n","INFO:tensorflow:examples/sec: 35.6382\n","INFO:tensorflow:global_step/sec: 1.11484\n","INFO:tensorflow:examples/sec: 35.6748\n","INFO:tensorflow:Saving checkpoints for 2020 into gdrive/My Drive/cs182_proj/output/output.txt/model.ckpt.\n","INFO:tensorflow:global_step/sec: 0.990412\n","INFO:tensorflow:examples/sec: 31.6932\n","INFO:tensorflow:global_step/sec: 1.10371\n","INFO:tensorflow:examples/sec: 35.3188\n","INFO:tensorflow:global_step/sec: 1.10527\n","INFO:tensorflow:examples/sec: 35.3686\n","INFO:tensorflow:global_step/sec: 1.10647\n","INFO:tensorflow:examples/sec: 35.4071\n","INFO:tensorflow:global_step/sec: 1.10559\n","INFO:tensorflow:examples/sec: 35.3788\n","INFO:tensorflow:global_step/sec: 1.10492\n","INFO:tensorflow:examples/sec: 35.3575\n","INFO:tensorflow:global_step/sec: 1.10668\n","INFO:tensorflow:examples/sec: 35.4137\n","INFO:tensorflow:global_step/sec: 1.10515\n","INFO:tensorflow:examples/sec: 35.365\n","INFO:tensorflow:global_step/sec: 1.10466\n","INFO:tensorflow:examples/sec: 35.3492\n","INFO:tensorflow:global_step/sec: 1.10556\n","INFO:tensorflow:examples/sec: 35.3778\n","INFO:tensorflow:Saving checkpoints for 3020 into gdrive/My Drive/cs182_proj/output/output.txt/model.ckpt.\n","INFO:tensorflow:global_step/sec: 0.987929\n","INFO:tensorflow:examples/sec: 31.6137\n","INFO:tensorflow:global_step/sec: 1.10342\n","INFO:tensorflow:examples/sec: 35.3093\n","INFO:tensorflow:global_step/sec: 1.105\n","INFO:tensorflow:examples/sec: 35.36\n","INFO:tensorflow:global_step/sec: 1.10607\n","INFO:tensorflow:examples/sec: 35.3944\n","INFO:tensorflow:global_step/sec: 1.10503\n","INFO:tensorflow:examples/sec: 35.3608\n","INFO:tensorflow:global_step/sec: 1.10576\n","INFO:tensorflow:examples/sec: 35.3842\n","INFO:tensorflow:global_step/sec: 1.10418\n","INFO:tensorflow:examples/sec: 35.3338\n","INFO:tensorflow:global_step/sec: 1.10423\n","INFO:tensorflow:examples/sec: 35.3352\n","INFO:tensorflow:global_step/sec: 1.10416\n","INFO:tensorflow:examples/sec: 35.3332\n","INFO:tensorflow:global_step/sec: 1.10428\n","INFO:tensorflow:examples/sec: 35.3369\n","INFO:tensorflow:Saving checkpoints for 4020 into gdrive/My Drive/cs182_proj/output/output.txt/model.ckpt.\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:966: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use standard file APIs to delete files with this prefix.\n","INFO:tensorflow:global_step/sec: 0.989898\n","INFO:tensorflow:examples/sec: 31.6767\n","INFO:tensorflow:global_step/sec: 1.10288\n","INFO:tensorflow:examples/sec: 35.2922\n","INFO:tensorflow:global_step/sec: 1.10451\n","INFO:tensorflow:examples/sec: 35.3442\n","INFO:tensorflow:global_step/sec: 1.10375\n","INFO:tensorflow:examples/sec: 35.32\n","INFO:tensorflow:global_step/sec: 1.1049\n","INFO:tensorflow:examples/sec: 35.3569\n","INFO:tensorflow:global_step/sec: 1.10414\n","INFO:tensorflow:examples/sec: 35.3324\n","INFO:tensorflow:global_step/sec: 1.10452\n","INFO:tensorflow:examples/sec: 35.3445\n","INFO:tensorflow:global_step/sec: 1.10464\n","INFO:tensorflow:examples/sec: 35.3484\n","INFO:tensorflow:global_step/sec: 1.10532\n","INFO:tensorflow:examples/sec: 35.3702\n","INFO:tensorflow:global_step/sec: 1.10454\n","INFO:tensorflow:examples/sec: 35.3453\n","INFO:tensorflow:Saving checkpoints for 5020 into gdrive/My Drive/cs182_proj/output/output.txt/model.ckpt.\n","INFO:tensorflow:global_step/sec: 0.987279\n","INFO:tensorflow:examples/sec: 31.5929\n","INFO:tensorflow:global_step/sec: 1.10251\n","INFO:tensorflow:examples/sec: 35.2804\n","INFO:tensorflow:global_step/sec: 1.10414\n","INFO:tensorflow:examples/sec: 35.3326\n","INFO:tensorflow:global_step/sec: 1.1048\n","INFO:tensorflow:examples/sec: 35.3536\n","INFO:tensorflow:global_step/sec: 1.10462\n","INFO:tensorflow:examples/sec: 35.3479\n","INFO:tensorflow:global_step/sec: 1.10411\n","INFO:tensorflow:examples/sec: 35.3316\n","INFO:tensorflow:global_step/sec: 1.10486\n","INFO:tensorflow:examples/sec: 35.3555\n","INFO:tensorflow:global_step/sec: 1.10445\n","INFO:tensorflow:examples/sec: 35.3425\n","INFO:tensorflow:global_step/sec: 1.10413\n","INFO:tensorflow:examples/sec: 35.3323\n","INFO:tensorflow:global_step/sec: 1.10538\n","INFO:tensorflow:examples/sec: 35.3721\n","INFO:tensorflow:Saving checkpoints for 6020 into gdrive/My Drive/cs182_proj/output/output.txt/model.ckpt.\n","INFO:tensorflow:global_step/sec: 0.989212\n","INFO:tensorflow:examples/sec: 31.6548\n","INFO:tensorflow:global_step/sec: 1.10353\n","INFO:tensorflow:examples/sec: 35.3129\n","INFO:tensorflow:global_step/sec: 1.10455\n","INFO:tensorflow:examples/sec: 35.3457\n","INFO:tensorflow:global_step/sec: 1.10483\n","INFO:tensorflow:examples/sec: 35.3544\n","INFO:tensorflow:global_step/sec: 1.10536\n","INFO:tensorflow:examples/sec: 35.3716\n","INFO:tensorflow:global_step/sec: 1.10495\n","INFO:tensorflow:examples/sec: 35.3583\n","INFO:tensorflow:global_step/sec: 1.10535\n","INFO:tensorflow:examples/sec: 35.3713\n","INFO:tensorflow:global_step/sec: 1.10475\n","INFO:tensorflow:examples/sec: 35.3521\n","INFO:tensorflow:global_step/sec: 1.10507\n","INFO:tensorflow:examples/sec: 35.3624\n","INFO:tensorflow:global_step/sec: 1.1046\n","INFO:tensorflow:examples/sec: 35.3473\n","INFO:tensorflow:Saving checkpoints for 7020 into gdrive/My Drive/cs182_proj/output/output.txt/model.ckpt.\n","INFO:tensorflow:global_step/sec: 0.987265\n","INFO:tensorflow:examples/sec: 31.5925\n","INFO:tensorflow:global_step/sec: 1.10208\n","INFO:tensorflow:examples/sec: 35.2665\n","INFO:tensorflow:global_step/sec: 1.10497\n","INFO:tensorflow:examples/sec: 35.3592\n","INFO:tensorflow:global_step/sec: 1.10457\n","INFO:tensorflow:examples/sec: 35.3463\n","INFO:tensorflow:global_step/sec: 1.1041\n","INFO:tensorflow:examples/sec: 35.3313\n","INFO:tensorflow:global_step/sec: 1.1055\n","INFO:tensorflow:examples/sec: 35.3761\n","INFO:tensorflow:global_step/sec: 1.10359\n","INFO:tensorflow:examples/sec: 35.315\n","INFO:tensorflow:global_step/sec: 1.10459\n","INFO:tensorflow:examples/sec: 35.3468\n","INFO:tensorflow:global_step/sec: 1.10486\n","INFO:tensorflow:examples/sec: 35.3556\n","INFO:tensorflow:global_step/sec: 1.10506\n","INFO:tensorflow:examples/sec: 35.3619\n","INFO:tensorflow:Saving checkpoints for 8020 into gdrive/My Drive/cs182_proj/output/output.txt/model.ckpt.\n","INFO:tensorflow:global_step/sec: 0.99556\n","INFO:tensorflow:examples/sec: 31.8579\n","INFO:tensorflow:global_step/sec: 1.10257\n","INFO:tensorflow:examples/sec: 35.2824\n","INFO:tensorflow:global_step/sec: 1.10372\n","INFO:tensorflow:examples/sec: 35.3189\n","INFO:tensorflow:global_step/sec: 1.10408\n","INFO:tensorflow:examples/sec: 35.3306\n","INFO:tensorflow:global_step/sec: 1.10483\n","INFO:tensorflow:examples/sec: 35.3545\n","INFO:tensorflow:global_step/sec: 1.10485\n","INFO:tensorflow:examples/sec: 35.3552\n","INFO:tensorflow:global_step/sec: 1.10449\n","INFO:tensorflow:examples/sec: 35.3438\n","INFO:tensorflow:global_step/sec: 1.1044\n","INFO:tensorflow:examples/sec: 35.3407\n","INFO:tensorflow:global_step/sec: 1.10435\n","INFO:tensorflow:examples/sec: 35.3393\n","INFO:tensorflow:global_step/sec: 1.10386\n","INFO:tensorflow:examples/sec: 35.3235\n","INFO:tensorflow:Saving checkpoints for 9020 into gdrive/My Drive/cs182_proj/output/output.txt/model.ckpt.\n","INFO:tensorflow:global_step/sec: 0.990333\n","INFO:tensorflow:examples/sec: 31.6907\n","INFO:tensorflow:global_step/sec: 1.10346\n","INFO:tensorflow:examples/sec: 35.3107\n","INFO:tensorflow:global_step/sec: 1.10904\n","INFO:tensorflow:examples/sec: 35.4893\n","INFO:tensorflow:global_step/sec: 1.11392\n","INFO:tensorflow:examples/sec: 35.6453\n","INFO:tensorflow:global_step/sec: 1.11565\n","INFO:tensorflow:examples/sec: 35.7008\n","INFO:tensorflow:global_step/sec: 1.11586\n","INFO:tensorflow:examples/sec: 35.7077\n","INFO:tensorflow:global_step/sec: 1.11453\n","INFO:tensorflow:examples/sec: 35.6649\n","INFO:tensorflow:global_step/sec: 1.11552\n","INFO:tensorflow:examples/sec: 35.6967\n","INFO:tensorflow:global_step/sec: 1.10785\n","INFO:tensorflow:examples/sec: 35.4511\n","INFO:tensorflow:global_step/sec: 1.11054\n","INFO:tensorflow:examples/sec: 35.5374\n","INFO:tensorflow:Saving checkpoints for 10000 into gdrive/My Drive/cs182_proj/output/output.txt/model.ckpt.\n","INFO:tensorflow:Loss for final step: 1.9323374.\n","INFO:tensorflow:training_loop marked as finished\n","INFO:tensorflow:***** Running evaluation *****\n","INFO:tensorflow:  Batch size = 8\n","INFO:tensorflow:Calling model_fn.\n","INFO:tensorflow:Running eval on CPU\n","INFO:tensorflow:*** Features ***\n","INFO:tensorflow:  name = input_ids, shape = (8, 128)\n","INFO:tensorflow:  name = input_mask, shape = (8, 128)\n","INFO:tensorflow:  name = masked_lm_ids, shape = (8, 20)\n","INFO:tensorflow:  name = masked_lm_positions, shape = (8, 20)\n","INFO:tensorflow:  name = masked_lm_weights, shape = (8, 20)\n","INFO:tensorflow:  name = next_sentence_labels, shape = (8, 1)\n","INFO:tensorflow:  name = segment_ids, shape = (8, 128)\n","INFO:tensorflow:**** Trainable Variables ****\n","INFO:tensorflow:  name = bert/embeddings/word_embeddings:0, shape = (30522, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/token_type_embeddings:0, shape = (2, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/position_embeddings:0, shape = (512, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/pooler/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/pooler/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/output_bias:0, shape = (30522,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/seq_relationship/output_weights:0, shape = (2, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/seq_relationship/output_bias:0, shape = (2,), *INIT_FROM_CKPT*\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/metrics_impl.py:455: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.cast instead.\n","INFO:tensorflow:Done calling model_fn.\n","INFO:tensorflow:Starting evaluation at 2019-05-01T22:15:53Z\n","INFO:tensorflow:Graph was finalized.\n","2019-05-01 22:15:54.127212: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1512] Adding visible gpu devices: 0\n","2019-05-01 22:15:54.127299: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] Device interconnect StreamExecutor with strength 1 edge matrix:\n","2019-05-01 22:15:54.127316: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990]      0 \n","2019-05-01 22:15:54.127327: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1003] 0:   N \n","2019-05-01 22:15:54.127479: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 14115 MB memory) -> physical GPU (device: 0, name: Tesla T4, pci bus id: 0000:00:04.0, compute capability: 7.5)\n","INFO:tensorflow:Restoring parameters from gdrive/My Drive/cs182_proj/output/output.txt/model.ckpt-10000\n","INFO:tensorflow:Running local_init_op.\n","INFO:tensorflow:Done running local_init_op.\n","INFO:tensorflow:Evaluation [10/100]\n","INFO:tensorflow:Evaluation [20/100]\n","INFO:tensorflow:Evaluation [30/100]\n","INFO:tensorflow:Evaluation [40/100]\n","INFO:tensorflow:Evaluation [50/100]\n","INFO:tensorflow:Evaluation [60/100]\n","INFO:tensorflow:Evaluation [70/100]\n","INFO:tensorflow:Evaluation [80/100]\n","INFO:tensorflow:Evaluation [90/100]\n","INFO:tensorflow:Evaluation [100/100]\n","INFO:tensorflow:Finished evaluation at 2019-05-01-22:16:03\n","INFO:tensorflow:Saving dict for global step 10000: global_step = 10000, loss = 1.1665103, masked_lm_accuracy = 0.7674654, masked_lm_loss = 1.099467, next_sentence_accuracy = 0.9725, next_sentence_loss = 0.07087372\n","INFO:tensorflow:Saving 'checkpoint_path' summary for global step 10000: gdrive/My Drive/cs182_proj/output/output.txt/model.ckpt-10000\n","INFO:tensorflow:evaluation_loop marked as finished\n","INFO:tensorflow:***** Eval results *****\n","INFO:tensorflow:  global_step = 10000\n","INFO:tensorflow:  loss = 1.1665103\n","INFO:tensorflow:  masked_lm_accuracy = 0.7674654\n","INFO:tensorflow:  masked_lm_loss = 1.099467\n","INFO:tensorflow:  next_sentence_accuracy = 0.9725\n","INFO:tensorflow:  next_sentence_loss = 0.07087372\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"3hp9Ei0dVUIH","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"VJ61eau0UG7d","colab_type":"code","outputId":"f95bafac-17a9-44a8-ded3-2c2f1525ed46","executionInfo":{"status":"ok","timestamp":1556864264676,"user_tz":420,"elapsed":9710,"user":{"displayName":"ALLEN CHEN","photoUrl":"","userId":"13117472557552131577"}},"colab":{"base_uri":"https://localhost:8080/","height":3244}},"source":["# encoder_decoder_output_jokes.txt\n","# Create pretraining_data for each model output jokes\n","!python gdrive/My\\ Drive/cs182_proj/bert-master/create_pretraining_data.py \\\n","--input_file=gdrive/My\\ Drive/cs182_proj/encoder_decoder_output_jokes.txt \\\n","--output_file=gdrive/My\\ Drive/cs182_proj/processed_encoder_decoder_output_jokes.txt \\\n","--vocab_file=gdrive/My\\ Drive/cs182_proj/BERT_BASE_DIR/vocab.txt \\\n","--do_lower_case=True \\\n","--max_seq_length=128 \\\n","--max_predictions_per_seq=20 \\\n","--masked_lm_prob=0.15 \\\n","--random_seed=12345 \\\n","--dupe_factor=5"],"execution_count":0,"outputs":[{"output_type":"stream","text":["INFO:tensorflow:*** Reading from input files ***\n","INFO:tensorflow:  gdrive/My Drive/cs182_proj/encoder_decoder_output_jokes.txt\n","INFO:tensorflow:*** Writing to output files ***\n","INFO:tensorflow:  gdrive/My Drive/cs182_proj/processed_encoder_decoder_output_jokes.txt\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] why [MASK] the english majors switch to spanish [SEP] mo [MASK] ##ing sea ##crest arkansas av ##ian bea ##gle cat ##hum ##an shake ##rs [MASK] ##r tri ##som nothing skip gift int ##rup [MASK] ball ##oca ##ust sweet ##est scary rec ##eding dd ##dd [MASK] compatible kuwait binoculars [MASK] ##oot ##er irs ##h [MASK] ao ##oh ##hh ind ##ef [MASK] ##tl ##ly ant ##ea ##ter [MASK] shootout [SEP]\n","INFO:tensorflow:input_ids: 101 2339 103 1996 2394 15279 6942 2000 3009 102 9587 103 2075 2712 25313 6751 20704 2937 26892 9354 4937 28600 2319 6073 2869 103 2099 13012 25426 2498 13558 5592 20014 21531 103 3608 24755 19966 4086 4355 12459 28667 17819 20315 14141 103 11892 13085 29549 103 17206 2121 25760 2232 103 20118 11631 23644 27427 12879 103 19646 2135 14405 5243 3334 103 18297 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 2 11 25 29 34 45 49 54 60 66 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2106 4492 18315 2100 4570 14141 8040 6089 3981 18297 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] what is the worse thing for a guy [MASK] say [MASK] sex [SEP] para ##med ##ics hair ##piece three ##pio [MASK] ##and wildly ho ##ros ##cope them ho ##ros ##cope [MASK] thunder [MASK] stuff ##y interrupting k ##hard ##ashi ##ans saving boot ##icia ##n [MASK] [MASK] [MASK] trolley sun ##burn [MASK] fiance bottled catch ##y vic ##i poisonous hunts mad ##oka app [MASK] ##ds ##w [SEP]\n","INFO:tensorflow:input_ids: 101 2054 2003 1996 4788 2518 2005 1037 3124 103 2360 103 3348 102 11498 7583 6558 2606 11198 2093 22071 103 5685 13544 7570 7352 16186 2068 7570 7352 16186 103 8505 103 4933 2100 22602 1047 11783 12914 6962 7494 9573 24108 2078 103 103 103 20820 3103 8022 103 19154 26071 4608 2100 10967 2072 22641 28526 5506 12352 10439 103 5104 2860 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 9 11 21 31 33 45 46 47 51 63 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2000 2044 2540 7278 13724 25068 2015 5072 2015 17298 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] what ##s white and drip ##s from [MASK] cloud [SEP] [MASK] dec ##af denver beans ##talk backpack ##ing put [MASK] ##get ##her ab ##omi ##nable mouths scorpion ##s poll [MASK] denver t ##wi ##zzle [MASK] [MASK] [MASK] ##y bloom cat ##walk ##s carriages rubble screw ##s swift ##s womb ##at smug ##ly bait ##ing behave ##vo ##ye ##ural ghosts [MASK] ##mer ##camp victor ##s braves ##s victor ##s incorrect [SEP]\n","INFO:tensorflow:input_ids: 101 2054 2015 2317 1998 27304 2015 2013 103 6112 102 103 11703 10354 7573 13435 28014 13383 2075 2404 103 18150 5886 11113 20936 22966 15076 22015 2015 8554 103 7573 1056 9148 17644 103 103 103 2100 13426 4937 17122 2015 17744 17538 11224 2015 9170 2015 26578 4017 20673 2135 17395 2075 16582 6767 6672 11137 11277 103 5017 26468 5125 2015 13980 2015 5125 2015 16542 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 4 8 11 15 20 30 35 36 37 60 65 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1998 1996 9939 13435 3406 20807 2869 3707 10760 2819 5125 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] did you hear about the cold ##play concert in china last weekend [SEP] losers javier consciousness fisher ##pr [MASK] cheering [MASK] carrying expansion drying di [MASK] ##ter ##ested i ##zza ##rd neglect imp ##ale [MASK] sample ##r pcs cambodia cambodia hangs hangs om ##en sailors hendricks blast ##oise pi ##xie vs [MASK] ##s [MASK] ##fighting [MASK] ##sh ##tt ##p [MASK] ##and [SEP]\n","INFO:tensorflow:input_ids: 101 2106 2017 2963 2055 1996 3147 13068 4164 1999 2859 2197 5353 102 23160 13824 8298 8731 18098 103 24867 103 4755 4935 17462 4487 103 3334 17944 1045 20715 4103 19046 17727 9453 103 7099 2099 27019 12899 12899 17991 17991 18168 2368 11279 28895 8479 23565 14255 16898 5443 103 2015 103 22158 103 4095 4779 2361 103 5685 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 19 21 26 27 35 52 54 56 60 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 6610 4755 11493 3334 2094 16584 2033 20965 3433 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] [MASK] ##ba [MASK] to the bone zone na ##la the what [SEP] q how do you spot plumage happy motor ##cy ##cl ##ist in fair weather [SEP]\n","INFO:tensorflow:input_ids: 101 103 3676 103 2000 1996 5923 4224 6583 2721 1996 2054 102 1053 2129 2079 2017 3962 28764 3407 5013 5666 20464 2923 1999 4189 4633 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 1 3 18 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 21934 6160 1037 1999 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] para ##med ##ics apache ga ##mbo ##l ur ##sel ##f out ##weig ##h [MASK] ##sel ##f butch retire [MASK] ea ##u keeping ea ##u keeping t ##gt ea ##u shoe ##box ginger ultraviolet g ##la ##zing g ##la ##zing [MASK] ##la [MASK] foyer [MASK] float [MASK] ##ote ##nce word ##play economist ##turn ##ed ##pro ##sti [MASK] [MASK] ##pro [MASK] ##ked cyprus [SEP] why are native americans the most successful ##l strip ##pers punch [MASK] all ##eft behaved here cameo ##s makes ton ##ne hon [MASK] ##es texts texts examples carved harry switches ni ##₎ ##than ##k as ##phy [MASK] ##tion sinn ##er prize ##win ##ning jenna antony footage [MASK] ##ergy int ##eg ##rable [MASK] int ##eg ##rable brothel int ##eg ##rable [MASK] [SEP]\n","INFO:tensorflow:input_ids: 101 11498 7583 6558 15895 11721 13344 2140 24471 11246 2546 2041 27204 2232 103 11246 2546 17520 11036 103 19413 2226 4363 19413 2226 4363 1056 13512 19413 2226 10818 8758 14580 26299 1043 2721 6774 1043 2721 6774 103 2721 103 16683 103 14257 103 12184 5897 2773 13068 11708 22299 2098 21572 16643 103 103 21572 103 8126 9719 102 2339 2024 3128 4841 1996 2087 3144 2140 6167 7347 8595 103 2035 29218 26979 2182 12081 2015 3084 10228 2638 10189 103 2229 6981 6981 4973 7844 4302 15924 9152 30088 21604 2243 2004 21281 103 3508 26403 2121 3396 10105 5582 13504 16262 8333 103 24395 20014 13910 16670 103 20014 13910 16670 27308 20014 13910 16670 103 102 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 14 19 37 40 42 44 46 56 57 59 74 79 85 92 94 99 109 114 122 0\n","INFO:tensorflow:masked_lm_ids: 24471 2015 1043 1043 6774 14257 17727 24518 4895 6767 6279 15501 12399 2015 10327 14787 2035 27308 27308 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] what ##s the deal with drake first he [MASK] an actor now a [MASK] [SEP] what the hell is mine ##craft mace flap ##py flip ##fl ##ip ##s cutting ser ##iff van ##t age press ##ie [MASK] dr ##apes shah [MASK] [MASK] ##ile ##ctic sip ##s statue hudson ts ##hh ##hh fell [MASK] mono ##logue zu [MASK] ##berg tre ##ache [MASK] sales ##men [MASK] mona [MASK] ##rea ##ctive chin ##ks braun [SEP]\n","INFO:tensorflow:input_ids: 101 2054 2015 1996 3066 2007 7867 2034 2002 103 2019 3364 2085 1037 103 102 2054 1996 3109 2003 3067 10419 19382 20916 7685 11238 10258 11514 2015 6276 14262 13355 3158 2102 2287 2811 2666 103 2852 29040 7890 103 103 9463 13306 10668 2015 6231 6842 24529 23644 23644 3062 103 18847 24277 16950 103 4059 29461 15395 103 4341 3549 103 13813 103 16416 15277 5413 5705 21909 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 9 14 26 37 41 42 53 57 61 64 66 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2001 10687 10258 13468 7890 4958 3062 9102 2869 26823 2512 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] two scots schultz walk [MASK] a baker one turns to the [MASK] and says [MASK] that a cake or [MASK] [MASK] [MASK] ##ue [SEP] ran dressing loop maj [MASK] ##y j den ##ile [MASK] den ##ile sac ##ril [MASK] ##ious ##hum ##or expo ##s bar ##b ##q ##t psychology homage ##s oscar ##s transplant ##s cart ##ons z ##in ##fan ##del [MASK] asks ##you diploma oh ##so [MASK] ##asian ##s crushed ##asian ##s [MASK] radical ##izing ex ##claim bed cross ##post sperm ##ato ##zo ##ide sperm ##ato ##zo ##ide [SEP]\n","INFO:tensorflow:input_ids: 101 2048 12196 22378 3328 103 1037 6243 2028 4332 2000 1996 103 1998 2758 103 2008 1037 9850 2030 103 103 103 5657 102 2743 11225 7077 16686 103 2100 1046 7939 9463 103 7939 9463 17266 15928 103 6313 28600 2953 16258 2015 3347 2497 4160 2102 6825 14822 2015 7436 2015 22291 2015 11122 5644 1062 2378 15143 9247 103 5176 29337 9827 2821 6499 103 24145 2015 10560 24145 2015 103 7490 6026 4654 25154 2793 2892 19894 18047 10610 6844 5178 18047 10610 6844 5178 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 3 5 12 15 20 21 22 29 34 39 62 68 74 79 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 3549 2627 2060 2003 1037 21442 2075 2075 23911 13910 2015 10560 17466 2015 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] blindly trades halloween ##ies [MASK] understand ce ##r ##vi ##x pixel pixel [MASK] virgin virgin virgin me ##jo ##hn ##s c ##g trick ##ort ##reate ##rs red ##itor ##z ch ##ss ##h [MASK] das fingertips [MASK] [MASK] lamar cock ##su ##cking ##mot ##her ##fu ##cking ##cu ##nt colonial close clients pink ##ish [SEP] how does jalan ##g wentworth tell you that he ##s hungry ni [MASK] ##sing [MASK] ##ava ##st [MASK] newt ##s processor salt ##water [MASK] playoff ironically valuable slams [MASK] ##s kn ##ie ##val [MASK] weekend sides cubic district black ##ing [MASK] all ##bla ##ck poet g ##yn ##o square ##pants garden leg [MASK] ##late square ##pants eye ##glass ##es dom ##i hide ##th crush ##es [SEP]\n","INFO:tensorflow:input_ids: 101 25734 14279 14414 3111 103 3305 8292 2099 5737 2595 22138 22138 103 6261 6261 6261 2033 5558 7295 2015 1039 2290 7577 11589 29313 2869 2417 15660 2480 10381 4757 2232 103 8695 12206 103 103 19756 10338 6342 23177 18938 5886 11263 23177 10841 3372 5336 2485 7846 5061 4509 102 2129 2515 28410 2290 20572 2425 2017 2008 2002 2015 7501 9152 103 7741 103 12462 3367 103 25597 2015 13151 5474 5880 103 7808 18527 7070 25967 103 2015 14161 2666 10175 103 5353 3903 11919 2212 2304 2075 103 2035 28522 3600 4802 1043 6038 2080 2675 27578 3871 4190 103 13806 2675 27578 3239 15621 2229 14383 2072 5342 2705 10188 2229 102 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 5 13 16 33 36 37 56 66 68 71 75 77 80 82 87 90 94 106 0 0\n","INFO:tensorflow:masked_lm_ids: 11023 14629 6261 15358 9009 3801 1046 18136 7479 9006 5474 7808 2463 25462 2015 14406 21985 2483 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] did you hear about [MASK] lumber ##jack who worked overtime [SEP] mango force ##kin panel ##ing rec ##liner [MASK] ##ine dog [MASK] stigma themes ##sia ##h you ##mu ##ch vassal [MASK] vassal ##s con ##tort ##ion ##ist liberia ##n who conquer ##ing on ##dick doo ##oo bandwidth [MASK] fein ##ts fein [MASK] sight [MASK] be ##ch ##del [MASK] [MASK] ##v route [MASK] mediterranean skate ##s [SEP]\n","INFO:tensorflow:input_ids: 101 2106 2017 2963 2055 103 13891 17364 2040 2499 12253 102 24792 2486 4939 5997 2075 28667 20660 103 3170 3899 103 26453 6991 8464 2232 2017 12274 2818 24351 103 24351 2015 9530 25485 3258 2923 18039 2078 2040 16152 2075 2006 24066 20160 9541 20235 103 27132 3215 27132 103 4356 103 2022 2818 9247 103 103 2615 2799 103 7095 17260 2015 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 5 19 22 31 48 52 54 58 59 62 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1996 8855 2072 2015 4091 3215 10950 1038 2595 2099 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] [MASK] ##s the difference between a mid ##get and [MASK] ve [MASK] ##eal disease [SEP] zip ##po irene pol ##itia ##ns to ##bey ##on ##ce [MASK] h ##yst [MASK] ##ct ##omy [MASK] us cot ##ill ##ion see [MASK] ##rou ##gh hum hum nut ##ella nut ##ella bad house ##wife fa ##x ear [MASK] leo ##os ##car notebook ##s milk [MASK] every ##time ##so budapest grand ##ad jeremy def ##und [MASK] veto ##ed prosecutor [SEP]\n","INFO:tensorflow:input_ids: 101 103 2015 1996 4489 2090 1037 3054 18150 1998 103 2310 103 15879 4295 102 14101 6873 12855 14955 29050 3619 2000 19182 2239 3401 103 1044 27268 103 6593 16940 103 2149 26046 8591 3258 2156 103 22494 5603 14910 14910 17490 8411 17490 8411 2919 2160 19993 6904 2595 4540 103 6688 2891 10010 14960 2015 6501 103 2296 7292 6499 10926 2882 4215 7441 13366 8630 103 22102 2098 12478 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 1 10 12 20 26 29 32 38 53 60 70 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2054 1037 3678 29050 23996 7869 14629 2705 5685 12614 2098 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] that [MASK] be please do you take giant revolving badger ##s [SEP] prix sunset question rain ##coat usa [MASK] ta [MASK] t ##wee ##tment [MASK] ##y ae ##rated split ##pe ##a [MASK] ##s flirt ##ati ##ous ta ##chy ##on split ##pe ##a fa ##d experts ex story story story ts ##ao ##s [MASK] story serves lo [MASK] ##lo [MASK] ##l surf ##bor ##t tu ##pac ##s [MASK] [SEP]\n","INFO:tensorflow:input_ids: 101 2008 103 2022 3531 2079 2017 2202 5016 24135 24186 2015 102 5431 10434 3160 4542 16531 3915 103 11937 103 1056 28394 21181 103 2100 29347 9250 3975 5051 2050 103 2015 27978 10450 3560 11937 11714 2239 3975 5051 2050 6904 2094 8519 4654 2466 2466 2466 24529 7113 2015 103 2466 4240 8840 103 4135 103 2140 14175 12821 2102 10722 19498 2015 103 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 2 19 21 25 32 39 53 57 59 67 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 3363 2378 9905 15960 16073 2239 2466 4135 4135 26347 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] elizabeth pneumonia frogs sar ##ong sar ##ong cop ##rop ##hil ##ia aspirations black ##tha ##i blond ##ine ah ##ab ##s rubber sa [MASK] jo ##vi speed ##s bitch [MASK] bing [MASK] dale ##ks buffer buffer leftist ##s motion ##ing gi ##mme flyers photo [MASK] ##py 540 ##ce st ##y trapped tee ##pee he ##l ##vet ##ica [SEP] losers javier consciousness fisher ##pr ##ice cheering [MASK] carrying expansion [MASK] di ##sin ##ter ##ested i ##zza [MASK] neglect imp ##ale ##d sample ##r pcs cambodia cambodia hangs hangs om ##en sailors hendricks blast ##oise [MASK] ##xie vs ##hit ##s me ##fighting [MASK] [MASK] recreation [MASK] response ##and [SEP]\n","INFO:tensorflow:input_ids: 101 3870 18583 17582 18906 5063 18906 5063 8872 18981 19466 2401 22877 2304 8322 2072 8855 3170 6289 7875 2015 8903 7842 103 8183 5737 3177 2015 7743 103 17620 103 8512 5705 17698 17698 24247 2015 4367 2075 21025 20058 16217 6302 103 7685 20263 3401 2358 2100 7567 17170 28084 2002 2140 19510 5555 102 23160 13824 8298 8731 18098 6610 24867 103 4755 4935 103 4487 11493 3334 17944 1045 20715 103 19046 17727 9453 2094 7099 2099 27019 12899 12899 17991 17991 18168 2368 11279 28895 8479 23565 103 16898 5443 16584 2015 2033 22158 103 103 8640 103 3433 5685 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 18 23 26 29 31 32 44 46 65 68 75 93 100 101 102 103 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 6289 4783 15222 12762 2063 8512 3597 16522 4755 17462 4103 14255 20965 4095 4779 2361 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] [MASK] all girls make po ##op ##y [SEP] to ##to drying every ##time ##so salsa mer ##ci 176 [MASK] ##leg ##ged ac ##ne responsible stomach ##ache ##s un ##lea ##ded specific [MASK] ##bo goa ##d symbols twenty [MASK] ##cking ##one limb freshly ##gh we ##r da ##aa ##aa ##m ##n ol maya mic ##gt bottom decorate [MASK] ##o po ##mp ##ome ##gra ##nate plug plug [MASK] plug [SEP]\n","INFO:tensorflow:input_ids: 101 103 2035 3057 2191 13433 7361 2100 102 2000 3406 17462 2296 7292 6499 26509 21442 6895 18561 103 23115 5999 9353 2638 3625 4308 15395 2015 4895 19738 5732 3563 103 5092 15244 2094 9255 3174 103 23177 5643 15291 20229 5603 2057 2099 4830 11057 11057 2213 2078 19330 9815 23025 13512 3953 29460 103 2080 13433 8737 8462 17643 12556 13354 13354 103 13354 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 1 18 19 23 32 34 38 42 57 66 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2079 3238 2698 2638 5920 18552 11263 4887 20996 13354 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] how can you tell when you are [MASK] to an ex ##tro ##verted engineer [SEP] holmes hum ##p ##back pm [MASK] ##q ##z ##j ##pg fr ##illy ##bo ##cker learn ##er ow ##me ##ego ##d super ##gl ##ue far ##t mom rate [MASK] ##red ##ulously ml ##k coupe [MASK] bride [MASK] tug tug tug singh brightly her ##zquez frustrating trois troubles ##hoot ##er illegally yet [MASK] ##iii [MASK] ##de dover exception philip [SEP]\n","INFO:tensorflow:input_ids: 101 2129 2064 2017 2425 2043 2017 2024 103 2000 2019 4654 13181 26686 3992 102 9106 14910 2361 5963 7610 103 4160 2480 3501 26952 10424 20577 5092 9102 4553 2121 27593 4168 20265 2094 3565 23296 5657 2521 2102 3566 3446 103 5596 21227 19875 2243 15130 103 8959 103 12888 12888 12888 5960 14224 2014 22938 25198 25527 13460 23416 2121 17800 2664 103 28954 103 3207 13985 6453 5170 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 8 21 39 43 49 51 56 58 66 68 71 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 3331 2571 2521 4297 2015 2015 2015 13046 16021 6137 6453 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] out ##jured blond blond consonant [MASK] founder if ##e constitute fu [MASK] ##bo ##is hill ##bill [MASK] bulb ##ous inappropriate ##ly pan ##tal ##oons honda cremated five ##d mu ##cking pursuit ji ##tter ##bu ##g brave fresh ##ener attraction fan ##boys imp ##ast ##a res ##ur ##re ##ct tolerate b ##l fond [MASK] [SEP] one snow ##man say to the other snow ##man fi ##ddling serial meat ##ing yu neville zero ##ex ##er ##cise ##nut [MASK] ##en mo ##oso ##uri rev ##vin ps ##st ##w ##ho ps ##st ##w ##ho not ##z [MASK] not ##z ##s wig ##gly anyway we ##ab ##oo we ##ab ##oo [MASK] ##ab ##oo wo ##pw ##op [MASK] ##pw [MASK] lover chromosomes [MASK] ##y ##be patty ##cake bu ##rka tremendous [MASK] me [SEP]\n","INFO:tensorflow:input_ids: 101 2041 26949 8855 8855 18265 103 3910 2065 2063 12346 11865 103 5092 2483 2940 24457 103 20581 3560 15884 2135 6090 9080 27174 11990 28376 2274 2094 14163 23177 8463 10147 12079 8569 2290 9191 4840 24454 8432 5470 24916 17727 14083 2050 24501 3126 2890 6593 19242 1038 2140 13545 103 102 2028 4586 2386 2360 2000 1996 2060 4586 2386 10882 21814 7642 6240 2075 9805 14871 5717 10288 2121 18380 24072 103 2368 9587 19137 9496 7065 6371 8827 3367 2860 6806 8827 3367 2860 6806 2025 2480 103 2025 2480 2015 24405 25643 4312 2057 7875 9541 2057 7875 9541 103 7875 9541 24185 28400 7361 103 28400 103 7089 26874 103 2100 4783 17798 17955 20934 22379 14388 103 2033 102\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n","INFO:tensorflow:masked_lm_positions: 1 6 12 17 41 53 57 70 75 76 93 94 106 108 112 114 116 117 125 0\n","INFO:tensorflow:masked_lm_ids: 2041 3910 9468 2135 24916 2050 2386 2243 5779 18168 2015 2025 2057 9541 12155 7361 26874 12849 9661 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] did u hear [MASK] the [MASK] ##per poker game [SEP] stance ceramics [MASK] ram ##bo ##bi ##ggs glow athletic doo ##bee ##s adams wa ##cky headed sunny ##side dodge ##lok ##e myth ##an [MASK] ##eta [MASK] so ##uven ##irs hale [MASK] cu ##cum ##bers ##ome nato biceps kitchen ##ette genuinely di ##sin ##ter ##ested peg [MASK] pe ##can counter [MASK] ##er mac ##ab ##re claw via ##gara caught slaughter ##s [SEP]\n","INFO:tensorflow:input_ids: 101 2106 1057 2963 103 1996 103 4842 11662 2208 102 11032 17314 103 8223 5092 5638 21314 8652 5188 20160 11306 2015 5922 11333 17413 3753 11559 7363 11898 29027 2063 10661 2319 103 12928 103 2061 27346 18894 13084 103 12731 24894 17198 8462 10079 27947 3829 7585 15958 4487 11493 3334 17944 25039 103 21877 9336 4675 103 2121 6097 7875 2890 15020 3081 24864 3236 14574 2015 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 4 6 12 13 29 34 36 41 45 56 60 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2055 3393 2015 8799 1038 8458 11233 2078 8462 2015 21156 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] what do you call jay [MASK] having a leg transplant [SEP] pretend im ##balance [MASK] photon ##s photon ##s heated responsible [MASK] ice ##berg spoke ##sw ##oman [MASK] ##sw ##oman sex ##e theoretical [MASK] bounce ##r [MASK] ##sw ##oman sex [MASK] ra ##bi ra ##bi brew ##s emperors [MASK] ##ach [MASK] ##oka leg ##room playoff breath ##y breath ##y [MASK] la ##tte [SEP]\n","INFO:tensorflow:input_ids: 101 2054 2079 2017 2655 6108 103 2383 1037 4190 22291 102 9811 10047 26657 103 26383 2015 26383 2015 9685 3625 103 3256 4059 3764 26760 20778 103 26760 20778 3348 2063 9373 103 17523 2099 103 26760 20778 3348 103 10958 5638 10958 5638 24702 2015 19655 103 6776 103 12352 4190 9954 7808 3052 2100 3052 2100 103 2474 4674 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 6 15 22 28 34 37 41 49 51 60 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2480 2094 10901 3764 24379 3764 2063 13433 5506 17500 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] snatch demonstrated demonstrated bucket k ##ru [MASK] [MASK] in ##bet ##wee ##n reelection ex ##wife mari ##gua ##na adrenaline ha ##bane ##ro land [MASK] ##s hue ##hue ##hue land ##mine ##s press ##ups float ##on pit teenagers his ##w ##hen son [MASK] ##hen radial lifestyle ##s lifestyle [MASK] al ##co ##hli ##c mcc ##ona ##ugh ##ey ##s [MASK] ##oma ##te data ##set [SEP] [MASK] une ##x ##p [MASK] cage ##y g ##ya ussr ##s bi ##ber vital [MASK] ##id specialty demons [MASK] ##graph ##vo bach ##ele ##t fr ##id ##die bu ##k [MASK] ##bra ##iner pan ##t to ##ga ##eth ##er [MASK] ##upt ##ing actor school ##ed forget ##s flat ##ter patting [MASK] ##some fat ##so [MASK] [MASK] ##ss ##hort ##long ##pu ##nn ##yan ##d [MASK] [SEP]\n","INFO:tensorflow:input_ids: 101 23365 7645 7645 13610 1047 6820 103 103 1999 20915 28394 2078 17648 4654 19993 16266 19696 2532 14963 5292 27543 3217 2455 103 2015 20639 20169 20169 2455 11233 2015 2811 22264 14257 2239 6770 12908 2010 2860 10222 2365 103 10222 15255 9580 2015 9580 103 2632 3597 27766 2278 23680 7856 8953 3240 2015 103 9626 2618 2951 13462 102 103 16655 2595 2361 103 7980 2100 1043 3148 10331 2015 12170 5677 8995 103 3593 12233 7942 103 14413 6767 10384 12260 2102 10424 3593 10265 20934 2243 103 10024 26455 6090 2102 2000 3654 11031 2121 103 29441 2075 3364 2082 2098 5293 2015 4257 3334 26085 103 14045 6638 6499 103 103 4757 27794 10052 14289 10695 7054 2094 103 102\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n","INFO:tensorflow:masked_lm_positions: 7 8 24 38 42 48 58 64 68 78 80 82 84 93 102 113 117 118 126 0\n","INFO:tensorflow:masked_lm_ids: 11687 29107 11233 2365 2860 2015 2358 3436 7094 10424 10265 8285 17492 2053 9413 5580 2028 20660 15640 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] im so pissed i could punch a ba [MASK] what [SEP] para ##med ##ics alumni freight d ##ns declared la [MASK] apps panther [MASK] queen abs ##tine ##nt thy ##mes pal print ##out strongest prank ##s wholly ##ly promising braun shapes ##hi ##fted hole [MASK] hole ##y compass me pride ##hn [MASK] slab sends east fe [MASK] ##powered [SEP]\n","INFO:tensorflow:input_ids: 101 10047 2061 9421 1045 2071 8595 1037 8670 103 2054 102 11498 7583 6558 9441 8441 1040 3619 4161 2474 103 18726 15133 103 3035 14689 10196 3372 15177 7834 14412 6140 5833 10473 26418 2015 12590 2135 10015 21909 10466 4048 18915 4920 103 4920 2100 16681 2033 6620 7295 103 17584 10255 2264 10768 103 27267 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 9 21 24 37 40 45 50 52 57 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1037 4103 6947 24655 21909 2100 5558 2015 4179 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:Wrote 3783 total instances\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"UIPpaoSVWC6c","colab_type":"code","outputId":"7f7966ac-ac78-4fde-eb61-ee1d2e450ebf","executionInfo":{"status":"ok","timestamp":1556864371617,"user_tz":420,"elapsed":8178,"user":{"displayName":"ALLEN CHEN","photoUrl":"","userId":"13117472557552131577"}},"colab":{"base_uri":"https://localhost:8080/","height":3244}},"source":["# transformer_output_jokes.txt\n","# Create pretraining_data for each model output jokes\n","!python gdrive/My\\ Drive/cs182_proj/bert-master/create_pretraining_data.py \\\n","--input_file=gdrive/My\\ Drive/cs182_proj/transformer_output_jokes.txt \\\n","--output_file=gdrive/My\\ Drive/cs182_proj/processed_transformer_output_jokes.txt \\\n","--vocab_file=gdrive/My\\ Drive/cs182_proj/BERT_BASE_DIR/vocab.txt \\\n","--do_lower_case=True \\\n","--max_seq_length=128 \\\n","--max_predictions_per_seq=20 \\\n","--masked_lm_prob=0.15 \\\n","--random_seed=12345 \\\n","--dupe_factor=5"],"execution_count":0,"outputs":[{"output_type":"stream","text":["INFO:tensorflow:*** Reading from input files ***\n","INFO:tensorflow:  gdrive/My Drive/cs182_proj/transformer_output_jokes.txt\n","INFO:tensorflow:*** Writing to output files ***\n","INFO:tensorflow:  gdrive/My Drive/cs182_proj/processed_transformer_output_jokes.txt\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] did you hear about the [MASK] [MASK] who won a nobel [MASK] [SEP] he he [MASK] was outstanding outstanding in in his his field field ` ` ` ` [SEP]\n","INFO:tensorflow:input_ids: 101 2106 2017 2963 2055 1996 103 103 2040 2180 1037 10501 103 102 2002 2002 103 2001 5151 5151 1999 1999 2010 2010 2492 2492 1036 1036 1036 1036 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 6 7 12 16 21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 12665 24375 3396 2001 1999 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] im [MASK] ##ca ##use not i getting think the the first [MASK] fan is a ` [SEP] to to the the letter ##meter [MASK] course the girl [MASK] ##p iss ##he ` [SEP]\n","INFO:tensorflow:input_ids: 101 10047 103 3540 8557 2025 1045 2893 2228 1996 1996 2034 103 5470 2003 1037 1036 102 2000 2000 1996 1996 3661 22828 103 2607 1996 2611 103 2361 26354 5369 1036 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 2 4 12 24 28 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 4783 8557 2034 1997 3597 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] they they ##rer [MASK] both in a ac ##ro ##cr ##op [MASK] [MASK] ##e bed wrote ` [SEP] the a bit [MASK] ##ian ##y [MASK] ` ` ss ` ` ` [SEP]\n","INFO:tensorflow:input_ids: 101 2027 2027 14544 103 2119 1999 1037 9353 3217 26775 7361 103 103 2063 2793 2626 1036 102 1996 1037 2978 103 2937 2100 103 1036 1036 7020 1036 1036 1036 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 4 12 13 22 25 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2063 22269 7317 28616 1036 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] what ##s [MASK] difference between chinese [MASK] and racism [SEP] you jesus [MASK] jesus [MASK] ` driver in ` chickens ##ss ##ss ` [SEP]\n","INFO:tensorflow:input_ids: 101 2054 2015 103 4489 2090 2822 103 1998 14398 102 2017 4441 103 4441 103 1036 4062 1999 1036 21868 4757 4757 1036 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 3 7 13 15 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1996 2111 2031 1037 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] why don ##t black phosphate listen to country music [SEP] because ##be ##ca ##use they [MASK] are would similar similar similar similar ##y ##y baby [MASK] [SEP]\n","INFO:tensorflow:input_ids: 101 2339 2123 2102 2304 17344 4952 2000 2406 2189 102 2138 4783 3540 8557 2027 103 2024 2052 2714 2714 2714 2714 2100 2100 3336 103 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 5 16 22 26 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2111 2027 2714 1036 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] what did the road ##man [MASK] when he saw the mona lisa [SEP] palo ##s the ##mx between [MASK] and jelly i you can can ##tt j je ##le ##lly ##ly my it [MASK] ` [SEP]\n","INFO:tensorflow:input_ids: 101 2054 2106 1996 2346 2386 103 2043 2002 2387 1996 13813 7059 102 24326 2015 1996 22984 2090 103 1998 20919 1045 2017 2064 2064 4779 1046 15333 2571 9215 2135 2026 2009 103 1036 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 5 6 14 17 19 34 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2386 2360 2054 4489 9389 5980 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] did you hear about the [MASK] trapped in [MASK] nuns body [SEP] what ##s [MASK] definition of a will a a n ni ##nu ##n condo in ##m a [MASK] ex ` p ` ` [SEP]\n","INFO:tensorflow:input_ids: 101 2106 2017 2963 2055 1996 103 7567 1999 103 16752 2303 102 2054 2015 103 6210 1997 1037 2097 1037 1037 1050 9152 11231 2078 25805 1999 2213 1037 103 4654 1036 1052 1036 1036 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 5 6 9 15 18 30 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1996 2158 1037 1996 1037 1036 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] patron how come this fly is swimming in routed soup [SEP] neither ##im did a a rr ##uf expired [MASK] ` ` ` ` [SEP]\n","INFO:tensorflow:input_ids: 101 9161 2129 2272 2023 4875 2003 5742 1999 19578 11350 102 4445 5714 2106 1037 1037 25269 16093 13735 103 1036 1036 1036 1036 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 9 10 19 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2026 11350 11263 4246 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] what do reindeer say before telling [MASK] a joke [SEP] [MASK] i [MASK] don ##tt [MASK] know i its don at lot gr ##av ` [SEP]\n","INFO:tensorflow:input_ids: 101 2054 2079 29495 2360 2077 4129 103 1037 8257 102 103 1045 103 2123 4779 103 2113 1045 2049 2123 2012 2843 24665 11431 1036 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 7 11 13 16 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2017 1045 2123 2113 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] what ##s [MASK] [MASK] white and red all over [SEP] what ##s [MASK] hardest part about va ##ping telling telling parents parents you you are [MASK] [MASK] a slow good ##af co ` [SEP]\n","INFO:tensorflow:input_ids: 101 2054 2015 103 103 2317 1998 2417 2035 2058 102 2054 2015 103 18263 2112 2055 12436 4691 4129 4129 3008 3008 2017 2017 2024 103 103 1037 4030 2204 10354 2522 1036 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 3 4 13 26 27 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2304 1998 1996 2024 1037 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] a a n ni ##nu ##n condo in ##m a ` ex ` [MASK] ` joanne [SEP] what did the lesbian vampire say to the other lesbian vampire see see ya [MASK] du [MASK] de ` ` in ` [MASK] ` ` [SEP]\n","INFO:tensorflow:input_ids: 101 1037 1037 1050 9152 11231 2078 25805 1999 2213 1037 1036 4654 1036 103 1036 23459 102 2054 2106 1996 11690 4393 2360 2000 1996 2060 11690 4393 2156 2156 8038 103 4241 103 2139 1036 1036 1999 1036 103 1036 1036 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 5 14 15 16 32 34 40 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 11231 1052 1036 1036 24894 1036 2045 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] what ##s black and white and phelps [MASK] over braking can ##t get through a revolving door [SEP] a a man woman in after a ao ##tto ##otto [MASK] im ##y holy ` [SEP]\n","INFO:tensorflow:input_ids: 101 2054 2015 2304 1998 2317 1998 20475 103 2058 24427 2064 2102 2131 2083 1037 24135 2341 102 1037 1037 2158 2450 1999 2044 1037 20118 9284 23052 103 10047 2100 4151 1036 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 7 8 10 14 29 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2417 2035 1998 2083 1036 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] [MASK] ##cate just [MASK] boil put the it shocks once of [MASK] it ` ` ` [SEP] what did the two tam ##pon ##s have in common they they both are have in a a pretend pretend ##ley [MASK] girls ##y [MASK] ` [SEP]\n","INFO:tensorflow:input_ids: 101 103 16280 2074 103 26077 2404 1996 2009 28215 2320 1997 103 2009 1036 1036 1036 102 2054 2106 1996 2048 17214 26029 2015 2031 1999 2691 2027 2027 2119 2024 2031 1999 1037 1037 9811 9811 3051 103 3057 2100 103 1036 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 1 2 4 9 12 39 42 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2017 2017 2074 3109 1036 1036 4364 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] what does fiat [MASK] for [SEP] he he didn didn ##tt want [MASK] to to [MASK] get out out of ` [SEP]\n","INFO:tensorflow:input_ids: 101 2054 2515 18550 103 2005 102 2002 2002 2134 2134 4779 2215 103 2000 2000 103 2131 2041 2041 1997 1036 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 4 13 16 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 3233 2215 2131 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] you have ##lick as rev ##ol ##ve ss ##uck [MASK] ##er shi ##st [MASK] [SEP] a a salad salad [MASK] dressing ` ` th ` th ` th ` plc ` [SEP]\n","INFO:tensorflow:input_ids: 101 2017 2031 25230 2004 7065 4747 3726 7020 12722 103 2121 11895 3367 103 102 1037 1037 16521 16521 103 11225 1036 1036 16215 1036 16215 1036 16215 1036 15492 1036 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 10 14 17 20 30 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2090 1036 1037 11225 1061 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] [MASK] you have have to to look go stupid to ` ways [MASK] ` ` [SEP] [MASK] a she ##mp isn [MASK] ##cks ##cus ` t ` tam ` ` [SEP]\n","INFO:tensorflow:input_ids: 101 103 2017 2031 2031 2000 2000 2298 2175 5236 2000 1036 3971 103 1036 1036 102 103 1037 2016 8737 3475 103 10603 7874 1036 1056 1036 17214 1036 1036 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 1 4 13 17 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2017 2031 1036 1037 9932 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] patron how come this fly is swimming [MASK] my [MASK] [SEP] im ##im not a [MASK] boy ##t now [MASK] ##im ` [SEP]\n","INFO:tensorflow:input_ids: 101 9161 2129 2272 2023 4875 2003 5742 103 2026 103 102 10047 5714 2025 1037 103 2879 2102 2085 103 5714 1036 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 8 10 16 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1999 11350 1037 2021 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] how do filipino ##s count [MASK] [SEP] [MASK] by use a a st stud ##udi ##anian these these [MASK] science ` [SEP]\n","INFO:tensorflow:input_ids: 101 2129 2079 10275 2015 4175 103 102 103 2011 2224 1037 1037 2358 16054 21041 26032 2122 2122 103 2671 1036 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 6 8 19 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2769 2027 2671 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] [MASK] the directly ##thor fifty jam di pan ##mt ##ss ` ` [SEP] did you hear about the new em ##o website he [MASK] [MASK] a paul ex hurricane ve ##see [MASK] a ` [SEP]\n","INFO:tensorflow:input_ids: 101 103 1996 3495 27844 5595 9389 4487 6090 20492 4757 1036 1036 102 2106 2017 2963 2055 1996 2047 7861 2080 4037 2002 103 103 1037 2703 4654 7064 2310 19763 103 1037 1036 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 1 3 24 25 32 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1996 4654 23484 1037 1037 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] i i don don [MASK] know know i they could were not changing a ` [SEP] what [MASK] the stages of sex after marriage the first ##th no of ##m my ##s [MASK] [MASK] have me bird ` [SEP]\n","INFO:tensorflow:input_ids: 101 1045 1045 2123 2123 103 2113 2113 1045 2027 2071 2020 2025 5278 1037 1036 102 2054 103 1996 5711 1997 3348 2044 3510 1996 2034 2705 2053 1997 2213 2026 2015 103 103 2031 2033 4743 1036 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 5 14 18 33 34 37 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 4779 1037 2024 2069 3694 4743 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:Wrote 3730 total instances\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Mo4sKs-OWT-K","colab_type":"code","outputId":"33abf00f-e89d-41cd-ab4a-5a21bfd4a08e","executionInfo":{"status":"ok","timestamp":1556864421532,"user_tz":420,"elapsed":5862,"user":{"displayName":"ALLEN CHEN","photoUrl":"","userId":"13117472557552131577"}},"colab":{"base_uri":"https://localhost:8080/","height":3244}},"source":["# GPT_output_jokes.txt\n","# Create pretraining_data for each model output jokes\n","!python gdrive/My\\ Drive/cs182_proj/bert-master/create_pretraining_data.py \\\n","--input_file=gdrive/My\\ Drive/cs182_proj/GPT_output_jokes.txt \\\n","--output_file=gdrive/My\\ Drive/cs182_proj/processed_GPT_output_jokes.txt \\\n","--vocab_file=gdrive/My\\ Drive/cs182_proj/BERT_BASE_DIR/vocab.txt \\\n","--do_lower_case=True \\\n","--max_seq_length=128 \\\n","--max_predictions_per_seq=20 \\\n","--masked_lm_prob=0.15 \\\n","--random_seed=12345 \\\n","--dupe_factor=5"],"execution_count":0,"outputs":[{"output_type":"stream","text":["INFO:tensorflow:*** Reading from input files ***\n","INFO:tensorflow:  gdrive/My Drive/cs182_proj/GPT_output_jokes.txt\n","INFO:tensorflow:*** Writing to output files ***\n","INFO:tensorflow:  gdrive/My Drive/cs182_proj/processed_GPT_output_jokes.txt\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] he was an army officer . http : ##ges or buy . tiny ##pic [MASK] com orr ##us ##2 ##lc [SEP] why are you wearing ##hem on [MASK] top ? [MASK] you don ' t have to sell advertising for the dangers of alcohol . ? [MASK] ? br ##rr [MASK] [MASK] . so wear [SEP]\n","INFO:tensorflow:input_ids: 101 2002 2001 2019 2390 2961 1012 8299 1024 8449 2030 4965 1012 4714 24330 103 4012 26914 2271 2475 15472 102 2339 2024 2017 4147 29122 2006 103 2327 1029 103 2017 2123 1005 1056 2031 2000 5271 6475 2005 1996 16796 1997 6544 1012 1029 103 1029 7987 12171 103 103 1012 2061 4929 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 9 15 26 28 31 46 47 51 52 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2030 1012 2317 2115 2061 1029 1029 2243 2009 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] what smells like blue [MASK] ? [SEP] blue paint [MASK] [MASK] to rob allen . http : or or www . youtube . com or watch ? v [MASK] ##lc [SEP]\n","INFO:tensorflow:input_ids: 101 2054 14747 2066 2630 103 1029 102 2630 6773 103 103 2000 6487 5297 1012 8299 1024 2030 2030 7479 1012 7858 1012 4012 2030 3422 1029 1058 103 15472 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 5 10 11 25 29 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 6773 1012 4923 2030 2549 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] how many you alone gave [MASK] a internet provider ? [SEP] how many you look at you because [MASK] look like johnny [MASK] [MASK] ##ley and and is ##iste and not [SEP]\n","INFO:tensorflow:input_ids: 101 2129 2116 2017 2894 2435 103 1037 4274 10802 1029 102 2129 2116 2017 2298 2012 2017 2138 103 2298 2066 5206 103 103 3051 1998 1998 2003 27870 1998 2025 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 6 19 23 24 29 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2017 2017 1010 5366 2590 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] yes , we [MASK] re built up [MASK] times . it ' s allowed no ##ene [MASK] . hey folks [SEP] how near test ? the air , gu [MASK] confidence and power and go to an [MASK] up ##dating trumpets . basically there is state [MASK] both member numbers . [SEP]\n","INFO:tensorflow:input_ids: 101 2748 1010 2057 103 2128 2328 2039 103 2335 1012 2009 1005 1055 3039 2053 8625 103 1012 4931 12455 102 2129 2379 3231 1029 1996 2250 1010 19739 103 7023 1998 2373 1998 2175 2000 2019 103 2039 16616 26506 1012 10468 2045 2003 2110 103 2119 2266 3616 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 4 5 8 16 17 30 38 47 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1005 2128 1997 4953 2339 21814 4545 3595 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] what do you [MASK] an underground hip moose ? [SEP] he was an hector officer . http [MASK] or or buy . tiny [MASK] [MASK] com orr ##us ##2 ##lc [SEP]\n","INFO:tensorflow:input_ids: 101 2054 2079 2017 103 2019 5230 5099 17716 1029 102 2002 2001 2019 10590 2961 1012 8299 103 2030 2030 4965 1012 4714 103 103 4012 26914 2271 2475 15472 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 4 14 18 24 25 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2655 2390 1024 24330 1012 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] why did the per ##vert get divorced ? [SEP] rappers [MASK] what do robots grab when they have lovely teeth ? [MASK] ##gh [MASK] . what do [MASK] . . [MASK] . . . . . i [SEP]\n","INFO:tensorflow:input_ids: 101 2339 2106 1996 2566 16874 2131 9196 1029 102 28257 103 2054 2079 13507 6723 2043 2027 2031 8403 4091 1029 103 5603 103 1012 2054 2079 103 1012 1012 103 1012 1012 1012 1012 1012 1045 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 11 19 22 24 28 31 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1012 8403 10930 19585 8588 1012 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] why don [MASK] t you see any elle members in the elk ? [SEP] because [MASK] the [MASK] ##key suit . : d [MASK] wat [MASK] tried to finish each other [SEP]\n","INFO:tensorflow:input_ids: 101 2339 2123 103 1056 2017 2156 2151 15317 2372 1999 1996 18995 1029 102 2138 103 1996 103 14839 4848 1012 1024 1040 103 28194 103 2699 2000 3926 2169 2060 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 3 16 18 24 26 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1005 1997 2659 2323 2031 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] [MASK] do asian ##s have white parents [MASK] the only [MASK] ##my ' s brown ##ie baby steelers [MASK] [MASK] morning ? because love can ' [MASK] exist . ha ##ha got [SEP] make them an electric chair . carry on . [MASK] on to the [MASK] edge from the world . protect them [MASK] [SEP]\n","INFO:tensorflow:input_ids: 101 103 2079 4004 2015 2031 2317 3008 103 1996 2069 103 8029 1005 1055 2829 2666 3336 15280 103 103 2851 1029 2138 2293 2064 1005 103 4839 1012 5292 3270 2288 102 2191 2068 2019 3751 3242 1012 4287 2006 1012 103 2006 2000 1996 103 3341 2013 1996 2088 1012 4047 2068 103 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 1 8 11 19 20 27 43 47 55 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2339 1010 9389 1999 1996 1056 5376 8111 2013 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] why didn ' t yo ##da [MASK] apple ? [SEP] say i love you . another guy ' s born ##rit us [MASK] for [MASK] ? sitcom and to case math is unlimited . [SEP]\n","INFO:tensorflow:input_ids: 101 2339 2134 1005 1056 10930 2850 103 6207 1029 102 2360 1045 2293 2017 1012 2178 3124 1005 1055 2141 14778 2149 103 2005 103 1029 13130 1998 2000 2553 8785 2003 14668 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 7 18 21 23 25 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2066 1005 2005 1012 2033 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] cause he was a mor ##on ' s greatest brain surgeon . i heard that ##gin [MASK] was [MASK] ##ff ##ff ##ff ##ff ##ffy ##ne ##p . [SEP] how do [MASK] make a black nervous ? you take [MASK] to darkest [MASK] with you . i ' m going there tonight . this [MASK] to me on the elevator . [SEP]\n","INFO:tensorflow:input_ids: 101 3426 2002 2001 1037 22822 2239 1005 1055 4602 4167 9431 1012 1045 2657 2008 11528 103 2001 103 4246 4246 4246 4246 16329 2638 2361 1012 102 2129 2079 103 2191 1037 2304 6091 1029 2017 2202 103 2000 23036 103 2007 2017 1012 1045 1005 1049 2183 2045 3892 1012 2023 103 2000 2033 2006 1996 7764 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 12 17 19 22 31 39 42 52 54 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1012 5724 1052 4246 2017 2032 3482 1012 2288 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] why do jews not eat ni ##gger anymore ? [SEP] cause they would have been roasted alive . : [MASK] : i came [MASK] my party [MASK] night [MASK] [MASK] [SEP]\n","INFO:tensorflow:input_ids: 101 2339 2079 5181 2025 4521 9152 13327 4902 1029 102 3426 2027 2052 2031 2042 28115 4142 1012 1024 103 1024 1045 2234 103 2026 2283 103 2305 103 103 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 20 24 27 29 30 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1040 2000 2197 1012 1037 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] cause they would have been roasted alive . : d : [MASK] came to my party last night [MASK] a [SEP] because si ##kan ##ia rhymes with [MASK] [MASK] [MASK] [MASK] version of f1 . [SEP]\n","INFO:tensorflow:input_ids: 101 3426 2027 2052 2031 2042 28115 4142 1012 1024 1040 1024 103 2234 2000 2026 2283 2197 2305 103 1037 102 2138 9033 9126 2401 24468 2007 103 103 103 103 2544 1997 20069 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 12 19 28 29 30 31 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1045 1012 1996 22854 19277 2319 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] just [MASK] the guy who [MASK] good [MASK] . there is lots of [MASK] suggestions . [MASK] . you [SEP] three guys . three guys . i don ' t ₛ . . . auto correct [SEP]\n","INFO:tensorflow:input_ids: 101 2074 103 1996 3124 2040 103 2204 103 1012 2045 2003 7167 1997 103 15690 1012 103 1012 2017 102 2093 4364 1012 2093 4364 1012 1045 2123 1005 1056 1570 1012 1012 1012 8285 6149 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 2 6 8 14 17 31 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 3198 2003 2559 2060 4593 2113 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] what [MASK] aba ##cus say when someone told him that he had an [MASK] son ? barney . baseball team definitely fk [MASK] [MASK] about that . your dad ##mon ##key [SEP] how many polar bears does [MASK] take to [MASK] in a light ##bu ##lb ? [MASK] [MASK] . great [MASK] surely they ' ve [MASK] the same rating . please come on [SEP]\n","INFO:tensorflow:input_ids: 101 2054 103 19557 7874 2360 2043 2619 2409 2032 2008 2002 2018 2019 103 2365 1029 15377 1012 3598 2136 5791 14352 103 103 2055 2008 1012 2115 3611 8202 14839 102 2129 2116 11508 6468 2515 103 2202 2000 103 1999 1037 2422 8569 20850 1029 103 103 1012 2307 103 7543 2027 1005 2310 103 1996 2168 5790 1012 3531 2272 2006 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 2 14 23 24 38 41 48 49 52 57 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2106 10777 2098 2039 2009 11224 2074 2048 1010 2288 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] how did the hips ##ter burn his [MASK] ? [SEP] he drank his [MASK] before it was cool . rip i am so sorry . . . : d : sorry diminished [MASK] so [SEP]\n","INFO:tensorflow:input_ids: 101 2129 2106 1996 6700 3334 6402 2010 103 1029 102 2002 10749 2010 103 2077 2009 2001 4658 1012 10973 1045 2572 2061 3374 1012 1012 1012 1024 1040 1024 3374 15911 103 2061 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 8 14 21 32 33 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 4416 4157 1045 1045 2001 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] why did the deaf guy pull a knife out of his [MASK] ? [SEP] you flip it upside [MASK] try it kid that [MASK] s not the [MASK] . it , cannot be a joke [MASK] [SEP]\n","INFO:tensorflow:input_ids: 101 2339 2106 1996 12419 3124 4139 1037 5442 2041 1997 2010 103 1029 102 2017 11238 2009 14961 103 3046 2009 4845 2008 103 1055 2025 1996 103 1012 2009 1010 3685 2022 1037 8257 103 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 12 19 22 24 28 36 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 6471 1998 1012 1005 2391 2030 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] how do you get [MASK] [MASK] ireland ? [SEP] you ya ##wn in a long line , [MASK] yourself sc ##at it . i ' [unused191] sorry , i terribly sorry sorry sorry [SEP]\n","INFO:tensorflow:input_ids: 101 2129 2079 2017 2131 103 103 3163 1029 102 2017 8038 7962 1999 1037 2146 2240 1010 103 4426 8040 4017 2009 1012 1045 1005 196 3374 1010 1045 16668 3374 3374 3374 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 5 6 18 20 26 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2017 2013 2059 8040 1049 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] because his drawings were all drawings . he was a drawings artist . bad ##um ##ts ##ss [MASK] . : d sorry ##ick [MASK] [SEP] why did cows abused [MASK] ? because they were too ed ##gy for emphasis [MASK] i ' d be speechless if its [MASK] a stud . : [MASK] [MASK] ##icated can [SEP]\n","INFO:tensorflow:input_ids: 101 2138 2010 9254 2020 2035 9254 1012 2002 2001 1037 9254 3063 1012 2919 2819 3215 4757 103 1012 1024 1040 3374 6799 103 102 2339 2106 17188 16999 103 1029 2138 2027 2020 2205 3968 6292 2005 7902 103 1045 1005 1040 2022 25146 2065 2049 103 1037 16054 1012 1024 103 103 17872 2064 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 2 18 24 30 40 48 53 54 55 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2010 7421 2100 5850 1012 2025 1052 1024 1045 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] [MASK] do [MASK] get a lady to marry you [MASK] [SEP] ren ##ch it [MASK] ranch dressing . and then leave . cut her to the left . and you can ' [MASK] get [SEP]\n","INFO:tensorflow:input_ids: 101 103 2079 103 2131 1037 3203 2000 5914 2017 103 102 14916 2818 2009 103 8086 11225 1012 1998 2059 2681 1012 3013 2014 2000 1996 2187 1012 1998 2017 2064 1005 103 2131 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 1 3 10 15 33 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2129 2017 1029 1999 1056 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] give [MASK] a basketball and ask them whether they will be good or not . they don ' t [MASK] any hands to [SEP] what do you call a [MASK] ##he space infection ? a par vip ##elia ##ge sc ##am . bad ##um ##ts [MASK] reading [MASK] a charm ##ies story [MASK] bad [SEP]\n","INFO:tensorflow:input_ids: 101 2507 103 1037 3455 1998 3198 2068 3251 2027 2097 2022 2204 2030 2025 1012 2027 2123 1005 1056 103 2151 2398 2000 102 2054 2079 2017 2655 1037 103 5369 2686 8985 1029 1037 11968 21722 13902 3351 8040 3286 1012 2919 2819 3215 103 3752 103 1037 11084 3111 2466 103 2919 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 2 20 25 30 37 46 48 53 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2068 2031 2054 14397 6906 2015 2112 2138 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:Wrote 2114 total instances\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"JhpYKpqEWe9C","colab_type":"code","outputId":"0c7efb9b-4f83-4569-c05a-7c28787c4ec6","executionInfo":{"status":"ok","timestamp":1556864464273,"user_tz":420,"elapsed":6555,"user":{"displayName":"ALLEN CHEN","photoUrl":"","userId":"13117472557552131577"}},"colab":{"base_uri":"https://localhost:8080/","height":3244}},"source":["# LSTM_output_jokes.txt\n","# Create pretraining_data for each model output jokes\n","!python gdrive/My\\ Drive/cs182_proj/bert-master/create_pretraining_data.py \\\n","--input_file=gdrive/My\\ Drive/cs182_proj/LSTM_output_jokes.txt \\\n","--output_file=gdrive/My\\ Drive/cs182_proj/processed_LSTM_output_jokes.txt \\\n","--vocab_file=gdrive/My\\ Drive/cs182_proj/BERT_BASE_DIR/vocab.txt \\\n","--do_lower_case=True \\\n","--max_seq_length=128 \\\n","--max_predictions_per_seq=20 \\\n","--masked_lm_prob=0.15 \\\n","--random_seed=12345 \\\n","--dupe_factor=5"],"execution_count":0,"outputs":[{"output_type":"stream","text":["INFO:tensorflow:*** Reading from input files ***\n","INFO:tensorflow:  gdrive/My Drive/cs182_proj/LSTM_output_jokes.txt\n","INFO:tensorflow:*** Writing to output files ***\n","INFO:tensorflow:  gdrive/My Drive/cs182_proj/processed_LSTM_output_jokes.txt\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] triangles do you get when you cross an ins ##om ##nia ##c ag ##nostic and a d [MASK] ##le whole [MASK] [SEP] o see the moon ! what asha you call a man who wear the best [MASK] drink the word \" come [MASK] [SEP]\n","INFO:tensorflow:input_ids: 101 27189 2079 2017 2131 2043 2017 2892 2019 16021 5358 6200 2278 12943 28199 1998 1037 1040 103 2571 2878 103 102 1051 2156 1996 4231 999 2054 24595 2017 2655 1037 2158 2040 4929 1996 2190 103 4392 1996 2773 1000 2272 103 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 1 18 20 21 29 38 44 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2054 7274 9048 2278 2079 1998 2041 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] use i had to stop singing \" what ' s brown and white [MASK] goes ##hur alone with a slide of seven ? s [SEP] d you hear [MASK] the man who cooled to [MASK] his teeth , [MASK] i feel [MASK] i had to owned the ci ##rc [MASK] [SEP]\n","INFO:tensorflow:input_ids: 101 2224 1045 2018 2000 2644 4823 1000 2054 1005 1055 2829 1998 2317 103 3632 24572 2894 2007 1037 7358 1997 2698 1029 1055 102 1040 2017 2963 103 1996 2158 2040 12981 2000 103 2010 4091 1010 103 1045 2514 103 1045 2018 2000 3079 1996 25022 11890 103 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 12 14 16 29 35 39 42 50 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1998 1998 2188 2055 4287 2061 2066 2226 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] why don ##t insisting need a watch [SEP] both steal [MASK] [MASK] time she was talking apologized for the first time to have [MASK] do something [SEP]\n","INFO:tensorflow:input_ids: 101 2339 2123 2102 22604 2342 1037 3422 102 2119 8954 103 103 2051 2016 2001 3331 17806 2005 1996 2034 2051 2000 2031 103 2079 2242 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 4 11 12 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2308 2035 1996 2000 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] why should we call indians native americans [SEP] what ##s a sl ##uts [MASK] [MASK] e [MASK] so you have to tell you that you don ' t like it [MASK] what ' s the difference between a dead baby i [SEP]\n","INFO:tensorflow:input_ids: 101 2339 2323 2057 2655 6505 3128 4841 102 2054 2015 1037 22889 16446 103 103 1041 103 2061 2017 2031 2000 2425 2017 2008 2017 2123 1005 1056 2066 2009 103 2054 1005 1055 1996 4489 2090 1037 2757 3336 1045 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 4 14 15 17 31 40 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2655 8837 4392 1024 1012 3336 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] where did napoleon keep his armies [SEP] what do you [MASK] [MASK] ##z having a leg transplant the pigs of signals , [MASK] they don ' t [MASK] a hole [MASK] one side . what do you call a black [MASK] [SEP]\n","INFO:tensorflow:input_ids: 101 2073 2106 8891 2562 2010 8749 102 2054 2079 2017 103 103 2480 2383 1037 4190 22291 1996 14695 1997 7755 1010 103 2027 2123 1005 1056 103 1037 4920 103 2028 2217 1012 2054 2079 2017 2655 1037 2304 103 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 11 12 23 28 31 41 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2655 6108 1998 2031 1999 3124 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] what does [MASK] [MASK] [MASK] wake with but can never go to sleep with [SEP] ave a home invention that you have nothing to get [MASK] copy of the toilet . what do you poultry [SEP]\n","INFO:tensorflow:input_ids: 101 2054 2515 103 103 103 5256 2007 2021 2064 2196 2175 2000 3637 2007 102 13642 1037 2188 11028 2008 2017 2031 2498 2000 2131 103 6100 1997 1996 11848 1012 2054 2079 2017 22468 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 3 4 5 26 32 35 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1037 2158 2411 1037 2054 1039 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] s wrong with the [MASK] of a ball statue before i told her [MASK] first time in the ##tara cup of [SEP] an the joke about sex with [MASK] plane but 1790 have to take a shit when he saw a girl now if a dead [SEP]\n","INFO:tensorflow:input_ids: 101 1055 3308 2007 1996 103 1997 1037 3608 6231 2077 1045 2409 2014 103 2034 2051 1999 1996 27115 2452 1997 102 2019 1996 8257 2055 3348 2007 103 4946 2021 13393 2031 2000 2202 1037 4485 2043 2002 2387 1037 2611 2085 2065 1037 2757 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 5 10 14 19 21 29 32 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 4000 2077 1996 2088 1997 1037 2017 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] why did the se ##men cross [MASK] road [SEP] se [MASK] is [MASK] statue of the time . what do you get when [MASK] cross a soldier in the world ? a cold [SEP]\n","INFO:tensorflow:input_ids: 101 2339 2106 1996 7367 3549 2892 103 2346 102 7367 103 2003 103 6231 1997 1996 2051 1012 2054 2079 2017 2131 2043 103 2892 1037 5268 1999 1996 2088 1029 1037 3147 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 1 7 11 13 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2339 1996 2002 1037 2017 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] did you hear about [MASK] ag ##nostic d ##ys ##le ##xi [MASK] [SEP] why [MASK] the composer go to the chi ##rop ##rac ##tor [MASK] cookies me on [MASK] floors of the construction ci [MASK] ##um ##cision . . . . . . and then [MASK] call the po [SEP]\n","INFO:tensorflow:input_ids: 101 2106 2017 2963 2055 103 12943 28199 1040 7274 2571 9048 103 102 2339 103 1996 4543 2175 2000 1996 9610 18981 22648 4263 103 16324 2033 2006 103 8158 1997 1996 2810 25022 103 2819 28472 1012 1012 1012 1012 1012 1012 1998 2059 103 2655 1996 13433 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 5 12 15 25 29 32 35 46 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1996 2278 2106 1051 1996 1996 11890 2027 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] waiting [MASK] in the high school box . how do they could say it was a food to check the bed [SEP] what ##s the [MASK] between chinese people and racism iv ##e been [MASK] [MASK] . i ' m not sure if it was a hole in the face . [MASK] bolt walks into [MASK] bar [MASK] fun ##e [SEP]\n","INFO:tensorflow:input_ids: 101 3403 103 1999 1996 2152 2082 3482 1012 2129 2079 2027 2071 2360 2009 2001 1037 2833 2000 4638 1996 2793 102 2054 2015 1996 103 2090 2822 2111 1998 14398 4921 2063 2042 103 103 1012 1045 1005 1049 2025 2469 2065 2009 2001 1037 4920 1999 1996 2227 1012 103 10053 7365 2046 103 3347 103 4569 2063 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 2 26 35 36 43 52 53 56 58 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2182 4489 24858 2015 2065 1037 2158 1037 1996 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] t do when you watch tv [MASK] the human on a date [MASK] a member of the art [MASK] ##章 ##cie ##nce of course , i d [SEP] how many people does it [MASK] to save the world [SEP]\n","INFO:tensorflow:input_ids: 101 1056 2079 2043 2017 3422 2694 103 1996 2529 2006 1037 3058 103 1037 2266 1997 1996 2396 103 30458 23402 5897 1997 2607 1010 1045 1040 102 2129 2116 2111 2515 2009 103 2000 3828 1996 2088 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 2 7 13 19 20 34 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2079 1999 1029 2594 21877 2202 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] how does a ba ##bo ##on make [MASK] calls [SEP] [MASK] ? the [MASK] that was a more expensive [MASK] on the internet helped [MASK] over ? me : i [SEP]\n","INFO:tensorflow:input_ids: 101 2129 2515 1037 8670 5092 2239 2191 103 4455 102 103 1029 1996 103 2008 2001 1037 2062 6450 103 2006 1996 4274 3271 103 2058 1029 2033 1024 1045 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 8 11 14 20 25 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 3042 2482 2711 14759 2017 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] police chief so what do we know about the serial killer [SEP] or ##ld [MASK] a boat say when [MASK] couldn ' t stop [MASK] [MASK] by a disabled [MASK] on the back of [SEP]\n","INFO:tensorflow:input_ids: 101 2610 2708 2061 2054 2079 2057 2113 2055 1996 7642 6359 102 2030 6392 103 1037 4049 2360 2043 103 2481 1005 1056 2644 103 103 2011 1037 9776 103 2006 1996 2067 1997 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 15 20 25 26 30 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2007 2002 2893 2718 3602 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] [MASK] with the highway personality . i was going to prove the face to the final stairs [MASK] a ga ##r [SEP] why was the integrate arrested was [MASK] in a mexican restaurant on the point , they were a woman is all about [MASK] times [MASK] [SEP]\n","INFO:tensorflow:input_ids: 101 103 2007 1996 3307 6180 1012 1045 2001 2183 2000 6011 1996 2227 2000 1996 2345 5108 103 1037 11721 2099 102 2339 2001 1996 17409 4727 2001 103 1999 1037 4916 4825 2006 1996 2391 1010 2027 2020 1037 2450 2003 2035 2055 103 2335 103 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 1 18 26 29 37 45 47 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1060 1029 5455 3061 1010 2070 1045 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] how do you tell the difference between a met ##h head and backseat hill [MASK] ##y [SEP] ang ##ed his dick ##s into quan planet ? [MASK] were really [MASK] the other side . [MASK] doctor [SEP]\n","INFO:tensorflow:input_ids: 101 2129 2079 2017 2425 1996 4489 2090 1037 2777 2232 2132 1998 19978 2940 103 2100 102 17076 2098 2010 5980 2015 2046 24110 4774 1029 103 2020 2428 103 1996 2060 2217 1012 103 3460 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 13 15 24 27 30 35 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1037 24457 1996 2027 3305 1996 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] knock knock who ##s there ali ali who [SEP] [MASK] many [MASK] ##s counter girls [MASK] it take to change a light bulb adi ##o down [MASK] what do you call a good [MASK] when they [MASK] back ? i want to write a bucket [SEP]\n","INFO:tensorflow:input_ids: 101 7324 7324 2040 2015 2045 4862 4862 2040 102 103 2116 103 2015 4675 3057 103 2009 2202 2000 2689 1037 2422 20581 27133 2080 2091 103 2054 2079 2017 2655 1037 2204 103 2043 2027 103 2067 1029 1045 2215 2000 4339 1037 13610 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 10 12 16 27 34 37 38 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2129 9383 2515 1012 4405 2272 2067 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] what do you call it when [MASK] cat wins a [MASK] ##how [SEP] en you do not look like . [MASK] had to take your hand to the [MASK] side . i was out of the wi [SEP]\n","INFO:tensorflow:input_ids: 101 2054 2079 2017 2655 2009 2043 103 4937 5222 1037 103 14406 102 4372 2017 2079 2025 2298 2066 1012 103 2018 2000 2202 2115 2192 2000 1996 103 2217 1012 1045 2001 2041 1997 1996 15536 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 7 11 15 21 23 29 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1037 6077 2017 1045 2000 2060 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] all week over the world stuff the park bench . what do you call [MASK] [MASK] with his son [MASK] [SEP] so doctor do i [MASK] cancer or not ur ##e that you [MASK] to sit down by [MASK] construction but when [MASK] see someone says [MASK] come on the f [SEP]\n","INFO:tensorflow:input_ids: 101 2035 2733 2058 1996 2088 4933 1996 2380 6847 1012 2054 2079 2017 2655 103 103 2007 2010 2365 103 102 2061 3460 2079 1045 103 4456 2030 2025 24471 2063 2008 2017 103 2000 4133 2091 2011 103 2810 2021 2043 103 2156 2619 2758 103 2272 2006 1996 1042 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 15 16 20 26 34 39 43 47 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1037 2450 1996 2031 2031 1996 1045 1000 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] what did [MASK] [MASK] ##pon say to the other tam ##pon [SEP] the [MASK] are on the [unused839] final control . how do you know [MASK] you find the kids were the first [SEP]\n","INFO:tensorflow:input_ids: 101 2054 2106 103 103 26029 2360 2000 1996 2060 17214 26029 102 1996 103 2024 2006 1996 844 2345 2491 1012 2129 2079 2017 2113 103 2017 2424 1996 4268 2020 1996 2034 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 3 4 14 18 26 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2028 17214 2610 2482 2043 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] i [MASK] a [MASK] and [MASK] , \" it ' s like \" i don ' t have a bot of richard and now i can [MASK] t just have [SEP] that ##ll be magistrates do [MASK] [MASK] giant revolving badger ##s term ##ine ##how do you know when you cross a strip 广 and the biggest crows like a home cr ##a [SEP]\n","INFO:tensorflow:input_ids: 101 1045 103 1037 103 1998 103 1010 1000 2009 1005 1055 2066 1000 1045 2123 1005 1056 2031 1037 28516 1997 2957 1998 2085 1045 2064 103 1056 2074 2031 102 2008 3363 2022 23007 2079 103 103 5016 24135 24186 2015 2744 3170 14406 2079 2017 2113 2043 2017 2892 1037 6167 1842 1998 1996 5221 21623 2066 1037 2188 13675 2050 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 2 4 6 22 27 35 37 38 54 58 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2288 3185 2758 4424 1005 3531 2017 2202 2252 21623 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:Wrote 3756 total instances\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"47iIRdKV1wnt","colab_type":"code","outputId":"b8113442-9bc4-4c2b-c0ac-a2a7f35b3888","executionInfo":{"status":"ok","timestamp":1556872713635,"user_tz":420,"elapsed":7258,"user":{"displayName":"ALLEN CHEN","photoUrl":"","userId":"13117472557552131577"}},"colab":{"base_uri":"https://localhost:8080/","height":3244}},"source":["# LM_output_jokes.txt\n","# Create pretraining_data for each model output jokes\n","!python gdrive/My\\ Drive/cs182_proj/bert-master/create_pretraining_data.py \\\n","--input_file=gdrive/My\\ Drive/cs182_proj/LM_output_jokes.txt \\\n","--output_file=gdrive/My\\ Drive/cs182_proj/processed_LM_output_jokes.txt \\\n","--vocab_file=gdrive/My\\ Drive/cs182_proj/BERT_BASE_DIR/vocab.txt \\\n","--do_lower_case=True \\\n","--max_seq_length=128 \\\n","--max_predictions_per_seq=20 \\\n","--masked_lm_prob=0.15 \\\n","--random_seed=12345 \\\n","--dupe_factor=5"],"execution_count":0,"outputs":[{"output_type":"stream","text":["INFO:tensorflow:*** Reading from input files ***\n","INFO:tensorflow:  gdrive/My Drive/cs182_proj/LM_output_jokes.txt\n","INFO:tensorflow:*** Writing to output files ***\n","INFO:tensorflow:  gdrive/My Drive/cs182_proj/processed_LM_output_jokes.txt\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] what [MASK] [MASK] [MASK] i should buy a [MASK] or a bike [SEP] a : i don ' t know , but i don ' t know what [MASK] do [SEP]\n","INFO:tensorflow:input_ids: 101 2054 103 103 103 1045 2323 4965 1037 103 2030 1037 7997 102 1037 1024 1045 2123 1005 1056 2113 1010 2021 1045 2123 1005 1056 2113 2054 103 2079 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 2 3 4 9 29 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2079 2017 2228 11190 1045 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] i wouldn ' t pay $ 200 to have a [MASK] ##rba ##nzo bean on my face [SEP] what ##s an ex ##tro ##verted accountant a : \" i don [MASK] t know , [MASK] [MASK] ' [MASK] know , [MASK] just don ' t like it . [MASK] [SEP]\n","INFO:tensorflow:input_ids: 101 1045 2876 1005 1056 3477 1002 3263 2000 2031 1037 103 28483 25650 14068 2006 2026 2227 102 2054 2015 2019 4654 13181 26686 17907 1037 1024 1000 1045 2123 103 1056 2113 1010 103 103 1005 103 2113 1010 103 2074 2123 1005 1056 2066 2009 1012 103 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 11 22 31 35 36 38 41 49 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 11721 4654 1005 1045 2123 1056 1045 1000 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] do [MASK] royals why god created le ##pro ##sy [SEP] [MASK] titanic is the only one that can only get [MASK] of the other one [SEP]\n","INFO:tensorflow:input_ids: 101 2079 103 15426 2339 2643 2580 3393 21572 6508 102 103 20753 2003 1996 2069 2028 2008 2064 2069 2131 103 1997 1996 2060 2028 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 2 3 11 21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2017 2113 1996 2028 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] never [MASK] to the blind ##fold shooting range [SEP] [MASK] is milk so fast because it doesn ' t matter [SEP]\n","INFO:tensorflow:input_ids: 101 2196 103 2000 1996 6397 10371 5008 2846 102 103 2003 6501 2061 3435 2138 2009 2987 1005 1056 3043 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 2 10 14 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2042 2339 3435 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] when a [MASK] says what did you just say [SEP] her [MASK] [MASK] [MASK] don [MASK] t know , but i don ' t know , but i [MASK] ' t know what [MASK] laced them with , but i don ' t know what he laced them with valuation but i was trip [MASK] all day [SEP]\n","INFO:tensorflow:input_ids: 101 2043 1037 103 2758 2054 2106 2017 2074 2360 102 2014 103 103 103 2123 103 1056 2113 1010 2021 1045 2123 1005 1056 2113 1010 2021 1045 103 1005 1056 2113 2054 103 17958 2068 2007 1010 2021 1045 2123 1005 1056 2113 2054 2002 17958 2068 2007 26004 2021 1045 2001 4440 103 2035 2154 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 3 12 13 14 16 29 34 50 55 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2450 3129 1029 1045 1005 2123 2002 1010 4691 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] [MASK] noah was loading [MASK] ark where did he put the [MASK] [SEP] he was a little hoarse [SEP]\n","INFO:tensorflow:input_ids: 101 103 7240 2001 10578 103 15745 2073 2106 2002 2404 1996 103 102 2002 2001 1037 2210 21221 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 1 5 12 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2043 1996 13734 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] because they are always in [MASK] corner [SEP] nothing [MASK] they were both stuck affirmed cu ##nts [SEP]\n","INFO:tensorflow:input_ids: 101 2138 2027 2024 2467 1999 103 3420 102 2498 103 2027 2020 2119 5881 19768 12731 7666 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 6 10 15 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1996 1010 2039 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] do all girls make po [MASK] ##y [SEP] it was a great [MASK] to get a little dogg ##y [SEP]\n","INFO:tensorflow:input_ids: 101 2079 2035 3057 2191 13433 103 2100 102 2009 2001 1037 2307 103 2000 2131 1037 2210 28844 2100 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 3 6 13 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 3057 7361 2126 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] where did napoleon reprint his armies [SEP] in his [MASK] ##ee ##vies [SEP]\n","INFO:tensorflow:input_ids: 101 2073 2106 8891 25364 2010 8749 102 1999 2010 103 4402 25929 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 4 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2562 22889 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] what did the two tam ##pon ##s have in common [SEP] [MASK] big bang ? [MASK] both have a little [MASK] [SEP]\n","INFO:tensorflow:input_ids: 101 2054 2106 1996 2048 17214 26029 2015 2031 1999 2691 102 103 2502 9748 1029 103 2119 2031 1037 2210 103 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 12 16 21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 1037 2027 2978 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] facebook [MASK] [MASK] ##tagram for one billion [SEP] the time [SEP]\n","INFO:tensorflow:input_ids: 101 9130 103 103 23091 2005 2028 4551 102 1996 2051 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 2 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 23311 16021 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] guest why did you offer me a [MASK] of candy [SEP] teacher why do we put a h ##yp ##hen in a bird ##ca [MASK] because they were [MASK] nervous - rex ! ! ! ! ! ! ! ! ! [MASK] [MASK] ! ! ! [MASK] ! ! ! ! ! ! ! ! ! ! [MASK] [MASK] ! ! ! ! ! [MASK] ! ! ! ! ! ! ! ! ! ! ! ! ! [MASK] [MASK] [MASK] ! [MASK] ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! [MASK] [SEP]\n","INFO:tensorflow:input_ids: 101 4113 2339 2106 2017 3749 2033 1037 103 1997 9485 102 3836 2339 2079 2057 2404 1037 1044 22571 10222 1999 1037 4743 3540 103 2138 2027 2020 103 6091 1011 10151 999 999 999 999 999 999 999 999 999 103 103 999 999 999 103 999 999 999 999 999 999 999 999 999 999 103 103 999 999 999 999 999 103 999 999 999 999 999 999 999 999 999 999 999 999 999 103 103 103 999 103 999 999 999 999 999 999 999 999 999 999 999 999 999 999 999 999 999 999 999 999 999 999 999 999 999 999 999 103 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 8 25 29 30 42 43 47 58 59 65 79 80 81 83 84 98 111 0 0 0\n","INFO:tensorflow:masked_lm_ids: 3538 3351 2035 6091 999 999 999 999 999 999 999 999 999 999 999 999 999 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] hey who did you [MASK] for [SEP] what ##s white [MASK] drip ##s from the [MASK] a : the police officer [SEP]\n","INFO:tensorflow:input_ids: 101 4931 2040 2106 2017 103 2005 102 2054 2015 2317 103 27304 2015 2013 1996 103 1037 1024 1996 2610 2961 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 5 11 16 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 3789 1998 6112 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] q what ##s the difference between a blonde and a supermarket trolley [SEP] because he [MASK] [MASK] little hoarse [SEP]\n","INFO:tensorflow:input_ids: 101 1053 2054 2015 1996 4489 2090 1037 9081 1998 1037 17006 20820 102 2138 2002 103 103 2210 21221 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 3 16 17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2015 2001 1037 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] because you have to be a little boy [SEP] what do [MASK] [MASK] a dead baby who fights [MASK] a conde [MASK] ##ending [SEP]\n","INFO:tensorflow:input_ids: 101 2138 2017 2031 2000 2022 1037 2210 2879 102 2054 2079 103 103 1037 2757 3336 2040 9590 103 1037 24707 103 18537 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 12 13 19 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2017 2655 4126 11020 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] at [MASK] fast food [MASK] is a curry ha ##pp ##iest [SEP] what do shakes ##pe ##arian buddhist ##s eat for breakfast a synonym [SEP]\n","INFO:tensorflow:input_ids: 101 2012 103 3435 2833 103 2003 1037 15478 5292 9397 10458 102 2054 2079 10854 5051 12199 7992 2015 4521 2005 6350 1037 10675 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 2 5 8 14 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2029 4825 24575 2079 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] saxony they are always [MASK] the dark [SEP] the world [SEP]\n","INFO:tensorflow:input_ids: 101 13019 2027 2024 2467 103 1996 2601 102 1996 2088 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 1 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2138 1999 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] what do you call a sl ##ut ##ty camera [SEP] how [MASK] you tell who the ex ##tro ##verted engineers are a nu ##dis [MASK] colony ? they both [MASK] the balls to get to the other lance [SEP]\n","INFO:tensorflow:input_ids: 101 2054 2079 2017 2655 1037 22889 4904 3723 4950 102 2129 103 2017 2425 2040 1996 4654 13181 26686 6145 2024 1037 16371 10521 103 5701 1029 2027 2119 103 1996 7395 2000 2131 2000 1996 2060 9993 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 12 25 29 30 37 38 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2079 2102 2119 2031 2060 2217 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] do hamburger [MASK] [MASK] good vampires [SEP] they don ' t [MASK] a good time [SEP]\n","INFO:tensorflow:input_ids: 101 2079 24575 103 103 2204 6144 102 2027 2123 1005 1056 103 1037 2204 2051 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 3 4 12 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2015 2191 2031 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 0\n","INFO:tensorflow:*** Example ***\n","INFO:tensorflow:tokens: [CLS] [MASK] he was [MASK] fungi [SEP] to get to the other side [SEP]\n","INFO:tensorflow:input_ids: 101 103 2002 2001 103 15289 102 2000 2131 2000 1996 2060 2217 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_positions: 1 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_ids: 2138 1037 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n","INFO:tensorflow:masked_lm_weights: 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n","INFO:tensorflow:next_sentence_labels: 1\n","INFO:tensorflow:Wrote 3767 total instances\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"Ae8ujZxzWunp","colab_type":"text"},"source":["This can be used to evaluate whether the next sentence is an appropriate joke / answer to the joke's question  \n","This is used to compare models"]},{"cell_type":"code","metadata":{"id":"6XMb0N6abt7I","colab_type":"code","outputId":"de72b1fa-d0aa-4a6c-c473-b86b3f390294","executionInfo":{"status":"ok","timestamp":1556864719605,"user_tz":420,"elapsed":64792,"user":{"displayName":"ALLEN CHEN","photoUrl":"","userId":"13117472557552131577"}},"colab":{"base_uri":"https://localhost:8080/","height":5168}},"source":["# Evaluate processed_encoder_decoder_output_jokes.txt\n","!python gdrive/My\\ Drive/cs182_proj/bert-master/run_pretraining.py \\\n","--input_file=gdrive/My\\ Drive/cs182_proj/processed_encoder_decoder_output_jokes.txt \\\n","--output_dir=gdrive/My\\ Drive/cs182_proj/output/jokes_prediction \\\n","--do_train=False \\\n","--do_eval=True \\\n","--bert_config_file=gdrive/My\\ Drive/cs182_proj/BERT_BASE_DIR/bert_config.json \\\n","--init_checkpoint=gdrive/My\\ Drive/cs182_proj/BERT_BASE_DIR/bert_model.ckpt \\\n","--train_batch_size=32 \\\n","--max_seq_length=128 \\\n","--max_predictions_per_seq=20 \\\n","--num_train_steps=10000 \\\n","--num_warmup_steps=10 \\\n","--learning_rate=2e-5"],"execution_count":0,"outputs":[{"output_type":"stream","text":["INFO:tensorflow:*** Input Files ***\n","INFO:tensorflow:  gdrive/My Drive/cs182_proj/processed_encoder_decoder_output_jokes.txt\n","\n","WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n","For more information, please see:\n","  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n","  * https://github.com/tensorflow/addons\n","If you depend on functionality not listed there, please file an issue.\n","\n","WARNING:tensorflow:Estimator's model_fn (<function model_fn_builder.<locals>.model_fn at 0x7f93284ab378>) includes params argument, but params are not passed to Estimator.\n","INFO:tensorflow:Using config: {'_model_dir': 'gdrive/My Drive/cs182_proj/output/jokes_prediction', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n","graph_options {\n","  rewrite_options {\n","    meta_optimizer_iterations: ONE\n","  }\n","}\n",", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f931bf7ecf8>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=1000, num_shards=8, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None), '_cluster': None}\n","INFO:tensorflow:_TPUContext: eval_on_tpu True\n","WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n","INFO:tensorflow:***** Running evaluation *****\n","INFO:tensorflow:  Batch size = 8\n","INFO:tensorflow:Could not find trained model in model_dir: gdrive/My Drive/cs182_proj/output/jokes_prediction, running initialization to evaluate.\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Colocations handled automatically by placer.\n","WARNING:tensorflow:From gdrive/My Drive/cs182_proj/bert-master/run_pretraining.py:385: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use `tf.data.experimental.map_and_batch(...)`.\n","WARNING:tensorflow:From gdrive/My Drive/cs182_proj/bert-master/run_pretraining.py:400: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.cast instead.\n","INFO:tensorflow:Calling model_fn.\n","INFO:tensorflow:Running eval on CPU\n","INFO:tensorflow:*** Features ***\n","INFO:tensorflow:  name = input_ids, shape = (8, 128)\n","INFO:tensorflow:  name = input_mask, shape = (8, 128)\n","INFO:tensorflow:  name = masked_lm_ids, shape = (8, 20)\n","INFO:tensorflow:  name = masked_lm_positions, shape = (8, 20)\n","INFO:tensorflow:  name = masked_lm_weights, shape = (8, 20)\n","INFO:tensorflow:  name = next_sentence_labels, shape = (8, 1)\n","INFO:tensorflow:  name = segment_ids, shape = (8, 128)\n","WARNING:tensorflow:From /content/gdrive/My Drive/cs182_proj/bert-master/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use keras.layers.dense instead.\n","INFO:tensorflow:**** Trainable Variables ****\n","INFO:tensorflow:  name = bert/embeddings/word_embeddings:0, shape = (30522, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/token_type_embeddings:0, shape = (2, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/position_embeddings:0, shape = (512, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/pooler/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/pooler/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/output_bias:0, shape = (30522,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/seq_relationship/output_weights:0, shape = (2, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/seq_relationship/output_bias:0, shape = (2,), *INIT_FROM_CKPT*\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/metrics_impl.py:455: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.cast instead.\n","INFO:tensorflow:Done calling model_fn.\n","INFO:tensorflow:Starting evaluation at 2019-05-03T06:24:35Z\n","INFO:tensorflow:Graph was finalized.\n","2019-05-03 06:24:36.864281: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n","2019-05-03 06:24:36.868630: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x1cd2aa0 executing computations on platform Host. Devices:\n","2019-05-03 06:24:36.868679: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>\n","2019-05-03 06:24:37.236463: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n","2019-05-03 06:24:37.238310: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x1cd2680 executing computations on platform CUDA. Devices:\n","2019-05-03 06:24:37.238352: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): Tesla T4, Compute Capability 7.5\n","2019-05-03 06:24:37.238709: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1433] Found device 0 with properties: \n","name: Tesla T4 major: 7 minor: 5 memoryClockRate(GHz): 1.59\n","pciBusID: 0000:00:04.0\n","totalMemory: 14.73GiB freeMemory: 2.01GiB\n","2019-05-03 06:24:37.238737: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1512] Adding visible gpu devices: 0\n","2019-05-03 06:24:40.373167: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] Device interconnect StreamExecutor with strength 1 edge matrix:\n","2019-05-03 06:24:40.373235: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990]      0 \n","2019-05-03 06:24:40.373252: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1003] 0:   N \n","2019-05-03 06:24:40.376926: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:42] Overriding allow_growth setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n","2019-05-03 06:24:40.379461: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 1739 MB memory) -> physical GPU (device: 0, name: Tesla T4, pci bus id: 0000:00:04.0, compute capability: 7.5)\n","INFO:tensorflow:Running local_init_op.\n","INFO:tensorflow:Done running local_init_op.\n","2019-05-03 06:25:02.946763: I tensorflow/stream_executor/dso_loader.cc:152] successfully opened CUDA library libcublas.so.10.0 locally\n","INFO:tensorflow:Evaluation [10/100]\n","INFO:tensorflow:Evaluation [20/100]\n","INFO:tensorflow:Evaluation [30/100]\n","INFO:tensorflow:Evaluation [40/100]\n","INFO:tensorflow:Evaluation [50/100]\n","INFO:tensorflow:Evaluation [60/100]\n","INFO:tensorflow:Evaluation [70/100]\n","INFO:tensorflow:Evaluation [80/100]\n","INFO:tensorflow:Evaluation [90/100]\n","INFO:tensorflow:Evaluation [100/100]\n","INFO:tensorflow:Finished evaluation at 2019-05-03-06:25:18\n","INFO:tensorflow:Saving dict for global step 0: global_step = 0, loss = 9.861248, masked_lm_accuracy = 0.30613595, masked_lm_loss = 5.8904033, next_sentence_accuracy = 0.29, next_sentence_loss = 3.9744303\n","INFO:tensorflow:evaluation_loop marked as finished\n","INFO:tensorflow:***** Eval results *****\n","INFO:tensorflow:  global_step = 0\n","INFO:tensorflow:  loss = 9.861248\n","INFO:tensorflow:  masked_lm_accuracy = 0.30613595\n","INFO:tensorflow:  masked_lm_loss = 5.8904033\n","INFO:tensorflow:  next_sentence_accuracy = 0.29\n","INFO:tensorflow:  next_sentence_loss = 3.9744303\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"qUZsiP1bV0zs","colab_type":"code","outputId":"83aa725c-8507-4ea1-fcb1-2d2448747dab","executionInfo":{"status":"ok","timestamp":1556865007118,"user_tz":420,"elapsed":37702,"user":{"displayName":"ALLEN CHEN","photoUrl":"","userId":"13117472557552131577"}},"colab":{"base_uri":"https://localhost:8080/","height":5168}},"source":["# Evaluate processed_transformer_output_jokes.txt\n","!python gdrive/My\\ Drive/cs182_proj/bert-master/run_pretraining.py \\\n","--input_file=gdrive/My\\ Drive/cs182_proj/processed_transformer_output_jokes.txt \\\n","--output_dir=gdrive/My\\ Drive/cs182_proj/output/transformer_jokes_prediction \\\n","--do_train=False \\\n","--do_eval=True \\\n","--bert_config_file=gdrive/My\\ Drive/cs182_proj/BERT_BASE_DIR/bert_config.json \\\n","--init_checkpoint=gdrive/My\\ Drive/cs182_proj/BERT_BASE_DIR/bert_model.ckpt \\\n","--train_batch_size=32 \\\n","--max_seq_length=128 \\\n","--max_predictions_per_seq=20 \\\n","--num_train_steps=10000 \\\n","--num_warmup_steps=10 \\\n","--learning_rate=2e-5"],"execution_count":0,"outputs":[{"output_type":"stream","text":["INFO:tensorflow:*** Input Files ***\n","INFO:tensorflow:  gdrive/My Drive/cs182_proj/processed_transformer_output_jokes.txt\n","\n","WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n","For more information, please see:\n","  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n","  * https://github.com/tensorflow/addons\n","If you depend on functionality not listed there, please file an issue.\n","\n","WARNING:tensorflow:Estimator's model_fn (<function model_fn_builder.<locals>.model_fn at 0x7f7931020268>) includes params argument, but params are not passed to Estimator.\n","INFO:tensorflow:Using config: {'_model_dir': 'gdrive/My Drive/cs182_proj/output/transformer_jokes_prediction', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n","graph_options {\n","  rewrite_options {\n","    meta_optimizer_iterations: ONE\n","  }\n","}\n",", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f79235021d0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=1000, num_shards=8, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None), '_cluster': None}\n","INFO:tensorflow:_TPUContext: eval_on_tpu True\n","WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n","INFO:tensorflow:***** Running evaluation *****\n","INFO:tensorflow:  Batch size = 8\n","INFO:tensorflow:Could not find trained model in model_dir: gdrive/My Drive/cs182_proj/output/transformer_jokes_prediction, running initialization to evaluate.\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Colocations handled automatically by placer.\n","WARNING:tensorflow:From gdrive/My Drive/cs182_proj/bert-master/run_pretraining.py:385: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use `tf.data.experimental.map_and_batch(...)`.\n","WARNING:tensorflow:From gdrive/My Drive/cs182_proj/bert-master/run_pretraining.py:400: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.cast instead.\n","INFO:tensorflow:Calling model_fn.\n","INFO:tensorflow:Running eval on CPU\n","INFO:tensorflow:*** Features ***\n","INFO:tensorflow:  name = input_ids, shape = (8, 128)\n","INFO:tensorflow:  name = input_mask, shape = (8, 128)\n","INFO:tensorflow:  name = masked_lm_ids, shape = (8, 20)\n","INFO:tensorflow:  name = masked_lm_positions, shape = (8, 20)\n","INFO:tensorflow:  name = masked_lm_weights, shape = (8, 20)\n","INFO:tensorflow:  name = next_sentence_labels, shape = (8, 1)\n","INFO:tensorflow:  name = segment_ids, shape = (8, 128)\n","WARNING:tensorflow:From /content/gdrive/My Drive/cs182_proj/bert-master/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use keras.layers.dense instead.\n","INFO:tensorflow:**** Trainable Variables ****\n","INFO:tensorflow:  name = bert/embeddings/word_embeddings:0, shape = (30522, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/token_type_embeddings:0, shape = (2, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/position_embeddings:0, shape = (512, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/pooler/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/pooler/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/output_bias:0, shape = (30522,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/seq_relationship/output_weights:0, shape = (2, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/seq_relationship/output_bias:0, shape = (2,), *INIT_FROM_CKPT*\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/metrics_impl.py:455: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.cast instead.\n","INFO:tensorflow:Done calling model_fn.\n","INFO:tensorflow:Starting evaluation at 2019-05-03T06:29:43Z\n","INFO:tensorflow:Graph was finalized.\n","2019-05-03 06:29:44.180114: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n","2019-05-03 06:29:44.180339: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x2518680 executing computations on platform Host. Devices:\n","2019-05-03 06:29:44.180368: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>\n","2019-05-03 06:29:44.415016: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n","2019-05-03 06:29:44.416533: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x2518940 executing computations on platform CUDA. Devices:\n","2019-05-03 06:29:44.416593: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): Tesla T4, Compute Capability 7.5\n","2019-05-03 06:29:44.416897: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1433] Found device 0 with properties: \n","name: Tesla T4 major: 7 minor: 5 memoryClockRate(GHz): 1.59\n","pciBusID: 0000:00:04.0\n","totalMemory: 14.73GiB freeMemory: 2.01GiB\n","2019-05-03 06:29:44.416925: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1512] Adding visible gpu devices: 0\n","2019-05-03 06:29:46.599337: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] Device interconnect StreamExecutor with strength 1 edge matrix:\n","2019-05-03 06:29:46.599411: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990]      0 \n","2019-05-03 06:29:46.599427: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1003] 0:   N \n","2019-05-03 06:29:46.599780: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:42] Overriding allow_growth setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n","2019-05-03 06:29:46.599838: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 1739 MB memory) -> physical GPU (device: 0, name: Tesla T4, pci bus id: 0000:00:04.0, compute capability: 7.5)\n","INFO:tensorflow:Running local_init_op.\n","INFO:tensorflow:Done running local_init_op.\n","2019-05-03 06:29:50.831726: I tensorflow/stream_executor/dso_loader.cc:152] successfully opened CUDA library libcublas.so.10.0 locally\n","INFO:tensorflow:Evaluation [10/100]\n","INFO:tensorflow:Evaluation [20/100]\n","INFO:tensorflow:Evaluation [30/100]\n","INFO:tensorflow:Evaluation [40/100]\n","INFO:tensorflow:Evaluation [50/100]\n","INFO:tensorflow:Evaluation [60/100]\n","INFO:tensorflow:Evaluation [70/100]\n","INFO:tensorflow:Evaluation [80/100]\n","INFO:tensorflow:Evaluation [90/100]\n","INFO:tensorflow:Evaluation [100/100]\n","INFO:tensorflow:Finished evaluation at 2019-05-03-06:30:06\n","INFO:tensorflow:Saving dict for global step 0: global_step = 0, loss = 10.509312, masked_lm_accuracy = 0.23828228, masked_lm_loss = 6.1792164, next_sentence_accuracy = 0.3725, next_sentence_loss = 4.3224373\n","INFO:tensorflow:evaluation_loop marked as finished\n","INFO:tensorflow:***** Eval results *****\n","INFO:tensorflow:  global_step = 0\n","INFO:tensorflow:  loss = 10.509312\n","INFO:tensorflow:  masked_lm_accuracy = 0.23828228\n","INFO:tensorflow:  masked_lm_loss = 6.1792164\n","INFO:tensorflow:  next_sentence_accuracy = 0.3725\n","INFO:tensorflow:  next_sentence_loss = 4.3224373\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"iB88XQ-zV83-","colab_type":"code","outputId":"dda08676-8566-4f11-e0c5-7f888b143cf7","executionInfo":{"status":"ok","timestamp":1556865098156,"user_tz":420,"elapsed":35076,"user":{"displayName":"ALLEN CHEN","photoUrl":"","userId":"13117472557552131577"}},"colab":{"base_uri":"https://localhost:8080/","height":5168}},"source":["# Evaluate processed_GPT_output_jokes.txt\n","!python gdrive/My\\ Drive/cs182_proj/bert-master/run_pretraining.py \\\n","--input_file=gdrive/My\\ Drive/cs182_proj/processed_GPT_output_jokes.txt \\\n","--output_dir=gdrive/My\\ Drive/cs182_proj/output/GPT_jokes_prediction \\\n","--do_train=False \\\n","--do_eval=True \\\n","--bert_config_file=gdrive/My\\ Drive/cs182_proj/BERT_BASE_DIR/bert_config.json \\\n","--init_checkpoint=gdrive/My\\ Drive/cs182_proj/BERT_BASE_DIR/bert_model.ckpt \\\n","--train_batch_size=32 \\\n","--max_seq_length=128 \\\n","--max_predictions_per_seq=20 \\\n","--num_train_steps=10000 \\\n","--num_warmup_steps=10 \\\n","--learning_rate=2e-5"],"execution_count":0,"outputs":[{"output_type":"stream","text":["INFO:tensorflow:*** Input Files ***\n","INFO:tensorflow:  gdrive/My Drive/cs182_proj/processed_GPT_output_jokes.txt\n","\n","WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n","For more information, please see:\n","  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n","  * https://github.com/tensorflow/addons\n","If you depend on functionality not listed there, please file an issue.\n","\n","WARNING:tensorflow:Estimator's model_fn (<function model_fn_builder.<locals>.model_fn at 0x7ff4ac967268>) includes params argument, but params are not passed to Estimator.\n","INFO:tensorflow:Using config: {'_model_dir': 'gdrive/My Drive/cs182_proj/output/GPT_jokes_prediction', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n","graph_options {\n","  rewrite_options {\n","    meta_optimizer_iterations: ONE\n","  }\n","}\n",", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7ff49ee081d0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=1000, num_shards=8, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None), '_cluster': None}\n","INFO:tensorflow:_TPUContext: eval_on_tpu True\n","WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n","INFO:tensorflow:***** Running evaluation *****\n","INFO:tensorflow:  Batch size = 8\n","INFO:tensorflow:Could not find trained model in model_dir: gdrive/My Drive/cs182_proj/output/GPT_jokes_prediction, running initialization to evaluate.\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Colocations handled automatically by placer.\n","WARNING:tensorflow:From gdrive/My Drive/cs182_proj/bert-master/run_pretraining.py:385: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use `tf.data.experimental.map_and_batch(...)`.\n","WARNING:tensorflow:From gdrive/My Drive/cs182_proj/bert-master/run_pretraining.py:400: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.cast instead.\n","INFO:tensorflow:Calling model_fn.\n","INFO:tensorflow:Running eval on CPU\n","INFO:tensorflow:*** Features ***\n","INFO:tensorflow:  name = input_ids, shape = (8, 128)\n","INFO:tensorflow:  name = input_mask, shape = (8, 128)\n","INFO:tensorflow:  name = masked_lm_ids, shape = (8, 20)\n","INFO:tensorflow:  name = masked_lm_positions, shape = (8, 20)\n","INFO:tensorflow:  name = masked_lm_weights, shape = (8, 20)\n","INFO:tensorflow:  name = next_sentence_labels, shape = (8, 1)\n","INFO:tensorflow:  name = segment_ids, shape = (8, 128)\n","WARNING:tensorflow:From /content/gdrive/My Drive/cs182_proj/bert-master/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use keras.layers.dense instead.\n","INFO:tensorflow:**** Trainable Variables ****\n","INFO:tensorflow:  name = bert/embeddings/word_embeddings:0, shape = (30522, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/token_type_embeddings:0, shape = (2, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/position_embeddings:0, shape = (512, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/pooler/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/pooler/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/output_bias:0, shape = (30522,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/seq_relationship/output_weights:0, shape = (2, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/seq_relationship/output_bias:0, shape = (2,), *INIT_FROM_CKPT*\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/metrics_impl.py:455: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.cast instead.\n","INFO:tensorflow:Done calling model_fn.\n","INFO:tensorflow:Starting evaluation at 2019-05-03T06:31:15Z\n","INFO:tensorflow:Graph was finalized.\n","2019-05-03 06:31:15.822779: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n","2019-05-03 06:31:15.822992: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x2a9e680 executing computations on platform Host. Devices:\n","2019-05-03 06:31:15.823017: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>\n","2019-05-03 06:31:16.079244: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n","2019-05-03 06:31:16.079767: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x2a9e940 executing computations on platform CUDA. Devices:\n","2019-05-03 06:31:16.079807: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): Tesla T4, Compute Capability 7.5\n","2019-05-03 06:31:16.080041: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1433] Found device 0 with properties: \n","name: Tesla T4 major: 7 minor: 5 memoryClockRate(GHz): 1.59\n","pciBusID: 0000:00:04.0\n","totalMemory: 14.73GiB freeMemory: 2.01GiB\n","2019-05-03 06:31:16.080065: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1512] Adding visible gpu devices: 0\n","2019-05-03 06:31:18.223718: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] Device interconnect StreamExecutor with strength 1 edge matrix:\n","2019-05-03 06:31:18.223790: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990]      0 \n","2019-05-03 06:31:18.223807: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1003] 0:   N \n","2019-05-03 06:31:18.224043: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:42] Overriding allow_growth setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n","2019-05-03 06:31:18.224101: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 1739 MB memory) -> physical GPU (device: 0, name: Tesla T4, pci bus id: 0000:00:04.0, compute capability: 7.5)\n","INFO:tensorflow:Running local_init_op.\n","INFO:tensorflow:Done running local_init_op.\n","2019-05-03 06:31:22.303059: I tensorflow/stream_executor/dso_loader.cc:152] successfully opened CUDA library libcublas.so.10.0 locally\n","INFO:tensorflow:Evaluation [10/100]\n","INFO:tensorflow:Evaluation [20/100]\n","INFO:tensorflow:Evaluation [30/100]\n","INFO:tensorflow:Evaluation [40/100]\n","INFO:tensorflow:Evaluation [50/100]\n","INFO:tensorflow:Evaluation [60/100]\n","INFO:tensorflow:Evaluation [70/100]\n","INFO:tensorflow:Evaluation [80/100]\n","INFO:tensorflow:Evaluation [90/100]\n","INFO:tensorflow:Evaluation [100/100]\n","INFO:tensorflow:Finished evaluation at 2019-05-03-06:31:37\n","INFO:tensorflow:Saving dict for global step 0: global_step = 0, loss = 6.7752285, masked_lm_accuracy = 0.42109454, masked_lm_loss = 4.055758, next_sentence_accuracy = 0.3975, next_sentence_loss = 2.7222037\n","INFO:tensorflow:evaluation_loop marked as finished\n","INFO:tensorflow:***** Eval results *****\n","INFO:tensorflow:  global_step = 0\n","INFO:tensorflow:  loss = 6.7752285\n","INFO:tensorflow:  masked_lm_accuracy = 0.42109454\n","INFO:tensorflow:  masked_lm_loss = 4.055758\n","INFO:tensorflow:  next_sentence_accuracy = 0.3975\n","INFO:tensorflow:  next_sentence_loss = 2.7222037\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"q1d9qKPIY5hm","colab_type":"code","outputId":"8476dbcd-0567-4807-8d36-fec042b92ccf","executionInfo":{"status":"ok","timestamp":1556865178678,"user_tz":420,"elapsed":31228,"user":{"displayName":"ALLEN CHEN","photoUrl":"","userId":"13117472557552131577"}},"colab":{"base_uri":"https://localhost:8080/","height":5168}},"source":["# Evaluate processed_LSTM_output_jokes.txt\n","!python gdrive/My\\ Drive/cs182_proj/bert-master/run_pretraining.py \\\n","--input_file=gdrive/My\\ Drive/cs182_proj/processed_LSTM_output_jokes.txt \\\n","--output_dir=gdrive/My\\ Drive/cs182_proj/output/LSTM_jokes_prediction \\\n","--do_train=False \\\n","--do_eval=True \\\n","--bert_config_file=gdrive/My\\ Drive/cs182_proj/BERT_BASE_DIR/bert_config.json \\\n","--init_checkpoint=gdrive/My\\ Drive/cs182_proj/BERT_BASE_DIR/bert_model.ckpt \\\n","--train_batch_size=32 \\\n","--max_seq_length=128 \\\n","--max_predictions_per_seq=20 \\\n","--num_train_steps=10000 \\\n","--num_warmup_steps=10 \\\n","--learning_rate=2e-5"],"execution_count":0,"outputs":[{"output_type":"stream","text":["INFO:tensorflow:*** Input Files ***\n","INFO:tensorflow:  gdrive/My Drive/cs182_proj/processed_LSTM_output_jokes.txt\n","\n","WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n","For more information, please see:\n","  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n","  * https://github.com/tensorflow/addons\n","If you depend on functionality not listed there, please file an issue.\n","\n","WARNING:tensorflow:Estimator's model_fn (<function model_fn_builder.<locals>.model_fn at 0x7f4670fd8268>) includes params argument, but params are not passed to Estimator.\n","INFO:tensorflow:Using config: {'_model_dir': 'gdrive/My Drive/cs182_proj/output/LSTM_jokes_prediction', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n","graph_options {\n","  rewrite_options {\n","    meta_optimizer_iterations: ONE\n","  }\n","}\n",", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f46634b91d0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=1000, num_shards=8, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None), '_cluster': None}\n","INFO:tensorflow:_TPUContext: eval_on_tpu True\n","WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n","INFO:tensorflow:***** Running evaluation *****\n","INFO:tensorflow:  Batch size = 8\n","INFO:tensorflow:Could not find trained model in model_dir: gdrive/My Drive/cs182_proj/output/LSTM_jokes_prediction, running initialization to evaluate.\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Colocations handled automatically by placer.\n","WARNING:tensorflow:From gdrive/My Drive/cs182_proj/bert-master/run_pretraining.py:385: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use `tf.data.experimental.map_and_batch(...)`.\n","WARNING:tensorflow:From gdrive/My Drive/cs182_proj/bert-master/run_pretraining.py:400: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.cast instead.\n","INFO:tensorflow:Calling model_fn.\n","INFO:tensorflow:Running eval on CPU\n","INFO:tensorflow:*** Features ***\n","INFO:tensorflow:  name = input_ids, shape = (8, 128)\n","INFO:tensorflow:  name = input_mask, shape = (8, 128)\n","INFO:tensorflow:  name = masked_lm_ids, shape = (8, 20)\n","INFO:tensorflow:  name = masked_lm_positions, shape = (8, 20)\n","INFO:tensorflow:  name = masked_lm_weights, shape = (8, 20)\n","INFO:tensorflow:  name = next_sentence_labels, shape = (8, 1)\n","INFO:tensorflow:  name = segment_ids, shape = (8, 128)\n","WARNING:tensorflow:From /content/gdrive/My Drive/cs182_proj/bert-master/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use keras.layers.dense instead.\n","INFO:tensorflow:**** Trainable Variables ****\n","INFO:tensorflow:  name = bert/embeddings/word_embeddings:0, shape = (30522, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/token_type_embeddings:0, shape = (2, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/position_embeddings:0, shape = (512, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/pooler/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/pooler/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/output_bias:0, shape = (30522,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/seq_relationship/output_weights:0, shape = (2, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/seq_relationship/output_bias:0, shape = (2,), *INIT_FROM_CKPT*\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/metrics_impl.py:455: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.cast instead.\n","INFO:tensorflow:Done calling model_fn.\n","INFO:tensorflow:Starting evaluation at 2019-05-03T06:32:37Z\n","INFO:tensorflow:Graph was finalized.\n","2019-05-03 06:32:37.716715: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n","2019-05-03 06:32:37.716927: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x235e680 executing computations on platform Host. Devices:\n","2019-05-03 06:32:37.716952: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>\n","2019-05-03 06:32:37.974973: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n","2019-05-03 06:32:37.975822: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x235e940 executing computations on platform CUDA. Devices:\n","2019-05-03 06:32:37.975863: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): Tesla T4, Compute Capability 7.5\n","2019-05-03 06:32:37.976170: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1433] Found device 0 with properties: \n","name: Tesla T4 major: 7 minor: 5 memoryClockRate(GHz): 1.59\n","pciBusID: 0000:00:04.0\n","totalMemory: 14.73GiB freeMemory: 2.01GiB\n","2019-05-03 06:32:37.976202: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1512] Adding visible gpu devices: 0\n","2019-05-03 06:32:39.206635: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] Device interconnect StreamExecutor with strength 1 edge matrix:\n","2019-05-03 06:32:39.206701: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990]      0 \n","2019-05-03 06:32:39.206720: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1003] 0:   N \n","2019-05-03 06:32:39.206944: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:42] Overriding allow_growth setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n","2019-05-03 06:32:39.207005: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 1739 MB memory) -> physical GPU (device: 0, name: Tesla T4, pci bus id: 0000:00:04.0, compute capability: 7.5)\n","INFO:tensorflow:Running local_init_op.\n","INFO:tensorflow:Done running local_init_op.\n","2019-05-03 06:32:42.826347: I tensorflow/stream_executor/dso_loader.cc:152] successfully opened CUDA library libcublas.so.10.0 locally\n","INFO:tensorflow:Evaluation [10/100]\n","INFO:tensorflow:Evaluation [20/100]\n","INFO:tensorflow:Evaluation [30/100]\n","INFO:tensorflow:Evaluation [40/100]\n","INFO:tensorflow:Evaluation [50/100]\n","INFO:tensorflow:Evaluation [60/100]\n","INFO:tensorflow:Evaluation [70/100]\n","INFO:tensorflow:Evaluation [80/100]\n","INFO:tensorflow:Evaluation [90/100]\n","INFO:tensorflow:Evaluation [100/100]\n","INFO:tensorflow:Finished evaluation at 2019-05-03-06:32:57\n","INFO:tensorflow:Saving dict for global step 0: global_step = 0, loss = 7.344153, masked_lm_accuracy = 0.3801069, masked_lm_loss = 4.342164, next_sentence_accuracy = 0.32875, next_sentence_loss = 2.9920745\n","INFO:tensorflow:evaluation_loop marked as finished\n","INFO:tensorflow:***** Eval results *****\n","INFO:tensorflow:  global_step = 0\n","INFO:tensorflow:  loss = 7.344153\n","INFO:tensorflow:  masked_lm_accuracy = 0.3801069\n","INFO:tensorflow:  masked_lm_loss = 4.342164\n","INFO:tensorflow:  next_sentence_accuracy = 0.32875\n","INFO:tensorflow:  next_sentence_loss = 2.9920745\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"v8JiE4D62M0T","colab_type":"code","outputId":"44f86b3c-1ac3-48d2-d60d-d5d69d00c2f2","executionInfo":{"status":"ok","timestamp":1556872799209,"user_tz":420,"elapsed":24078,"user":{"displayName":"ALLEN CHEN","photoUrl":"","userId":"13117472557552131577"}},"colab":{"base_uri":"https://localhost:8080/","height":5168}},"source":["# Evaluate processed_LM_output_jokes.txt\n","!python gdrive/My\\ Drive/cs182_proj/bert-master/run_pretraining.py \\\n","--input_file=gdrive/My\\ Drive/cs182_proj/processed_LM_output_jokes.txt \\\n","--output_dir=gdrive/My\\ Drive/cs182_proj/output/LM_jokes_prediction \\\n","--do_train=False \\\n","--do_eval=True \\\n","--bert_config_file=gdrive/My\\ Drive/cs182_proj/BERT_BASE_DIR/bert_config.json \\\n","--init_checkpoint=gdrive/My\\ Drive/cs182_proj/BERT_BASE_DIR/bert_model.ckpt \\\n","--train_batch_size=32 \\\n","--max_seq_length=128 \\\n","--max_predictions_per_seq=20 \\\n","--num_train_steps=10000 \\\n","--num_warmup_steps=10 \\\n","--learning_rate=2e-5"],"execution_count":0,"outputs":[{"output_type":"stream","text":["INFO:tensorflow:*** Input Files ***\n","INFO:tensorflow:  gdrive/My Drive/cs182_proj/processed_LM_output_jokes.txt\n","\n","WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n","For more information, please see:\n","  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n","  * https://github.com/tensorflow/addons\n","If you depend on functionality not listed there, please file an issue.\n","\n","WARNING:tensorflow:Estimator's model_fn (<function model_fn_builder.<locals>.model_fn at 0x7f0e25018268>) includes params argument, but params are not passed to Estimator.\n","INFO:tensorflow:Using config: {'_model_dir': 'gdrive/My Drive/cs182_proj/output/LM_jokes_prediction', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n","graph_options {\n","  rewrite_options {\n","    meta_optimizer_iterations: ONE\n","  }\n","}\n",", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f0e174f91d0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=1000, num_shards=8, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None), '_cluster': None}\n","INFO:tensorflow:_TPUContext: eval_on_tpu True\n","WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n","INFO:tensorflow:***** Running evaluation *****\n","INFO:tensorflow:  Batch size = 8\n","INFO:tensorflow:Could not find trained model in model_dir: gdrive/My Drive/cs182_proj/output/LM_jokes_prediction, running initialization to evaluate.\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Colocations handled automatically by placer.\n","WARNING:tensorflow:From gdrive/My Drive/cs182_proj/bert-master/run_pretraining.py:385: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use `tf.data.experimental.map_and_batch(...)`.\n","WARNING:tensorflow:From gdrive/My Drive/cs182_proj/bert-master/run_pretraining.py:400: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.cast instead.\n","INFO:tensorflow:Calling model_fn.\n","INFO:tensorflow:Running eval on CPU\n","INFO:tensorflow:*** Features ***\n","INFO:tensorflow:  name = input_ids, shape = (8, 128)\n","INFO:tensorflow:  name = input_mask, shape = (8, 128)\n","INFO:tensorflow:  name = masked_lm_ids, shape = (8, 20)\n","INFO:tensorflow:  name = masked_lm_positions, shape = (8, 20)\n","INFO:tensorflow:  name = masked_lm_weights, shape = (8, 20)\n","INFO:tensorflow:  name = next_sentence_labels, shape = (8, 1)\n","INFO:tensorflow:  name = segment_ids, shape = (8, 128)\n","WARNING:tensorflow:From /content/gdrive/My Drive/cs182_proj/bert-master/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use keras.layers.dense instead.\n","INFO:tensorflow:**** Trainable Variables ****\n","INFO:tensorflow:  name = bert/embeddings/word_embeddings:0, shape = (30522, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/token_type_embeddings:0, shape = (2, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/position_embeddings:0, shape = (512, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/embeddings/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/pooler/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = bert/pooler/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/predictions/output_bias:0, shape = (30522,), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/seq_relationship/output_weights:0, shape = (2, 768), *INIT_FROM_CKPT*\n","INFO:tensorflow:  name = cls/seq_relationship/output_bias:0, shape = (2,), *INIT_FROM_CKPT*\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/metrics_impl.py:455: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.cast instead.\n","INFO:tensorflow:Done calling model_fn.\n","INFO:tensorflow:Starting evaluation at 2019-05-03T08:39:44Z\n","INFO:tensorflow:Graph was finalized.\n","2019-05-03 08:39:44.971669: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n","2019-05-03 08:39:44.972022: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x2492680 executing computations on platform Host. Devices:\n","2019-05-03 08:39:44.972060: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>\n","2019-05-03 08:39:45.190795: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n","2019-05-03 08:39:45.191210: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x2492940 executing computations on platform CUDA. Devices:\n","2019-05-03 08:39:45.191242: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): Tesla T4, Compute Capability 7.5\n","2019-05-03 08:39:45.191508: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1433] Found device 0 with properties: \n","name: Tesla T4 major: 7 minor: 5 memoryClockRate(GHz): 1.59\n","pciBusID: 0000:00:04.0\n","totalMemory: 14.73GiB freeMemory: 2.01GiB\n","2019-05-03 08:39:45.191531: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1512] Adding visible gpu devices: 0\n","2019-05-03 08:39:46.600931: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] Device interconnect StreamExecutor with strength 1 edge matrix:\n","2019-05-03 08:39:46.600996: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990]      0 \n","2019-05-03 08:39:46.601012: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1003] 0:   N \n","2019-05-03 08:39:46.601211: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:42] Overriding allow_growth setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n","2019-05-03 08:39:46.601263: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 1739 MB memory) -> physical GPU (device: 0, name: Tesla T4, pci bus id: 0000:00:04.0, compute capability: 7.5)\n","INFO:tensorflow:Running local_init_op.\n","INFO:tensorflow:Done running local_init_op.\n","2019-05-03 08:39:50.193653: I tensorflow/stream_executor/dso_loader.cc:152] successfully opened CUDA library libcublas.so.10.0 locally\n","INFO:tensorflow:Evaluation [10/100]\n","INFO:tensorflow:Evaluation [20/100]\n","INFO:tensorflow:Evaluation [30/100]\n","INFO:tensorflow:Evaluation [40/100]\n","INFO:tensorflow:Evaluation [50/100]\n","INFO:tensorflow:Evaluation [60/100]\n","INFO:tensorflow:Evaluation [70/100]\n","INFO:tensorflow:Evaluation [80/100]\n","INFO:tensorflow:Evaluation [90/100]\n","INFO:tensorflow:Evaluation [100/100]\n","INFO:tensorflow:Finished evaluation at 2019-05-03-08:39:58\n","INFO:tensorflow:Saving dict for global step 0: global_step = 0, loss = 4.8116164, masked_lm_accuracy = 0.6213873, masked_lm_loss = 2.5463052, next_sentence_accuracy = 0.4725, next_sentence_loss = 2.088344\n","INFO:tensorflow:evaluation_loop marked as finished\n","INFO:tensorflow:***** Eval results *****\n","INFO:tensorflow:  global_step = 0\n","INFO:tensorflow:  loss = 4.8116164\n","INFO:tensorflow:  masked_lm_accuracy = 0.6213873\n","INFO:tensorflow:  masked_lm_loss = 2.5463052\n","INFO:tensorflow:  next_sentence_accuracy = 0.4725\n","INFO:tensorflow:  next_sentence_loss = 2.088344\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"euyH03DGZOH7","colab_type":"code","outputId":"b1ead8c1-12b6-4641-edf1-905fde1ef1b2","executionInfo":{"status":"ok","timestamp":1556872907194,"user_tz":420,"elapsed":442,"user":{"displayName":"ALLEN CHEN","photoUrl":"","userId":"13117472557552131577"}},"colab":{"base_uri":"https://localhost:8080/","height":121}},"source":["import pandas as pd\n","model_names = ['encoder-decoder', 'Transformer', 'GPT', 'LSTM', 'LM']\n","next_sentence_acc = [0.29, 0.3725, 0.3975, 0.3287, 0.4725]\n","#next_sentence_loss = [3.97, 4.322, 2.722, 2.99, 2.088344]\n","model_acc_df = pd.DataFrame({'Model': model_names, 'Accuracy': next_sentence_acc})\n","model_acc_df = model_acc_df.sort_values(['Accuracy'], ascending = False).reset_index(drop=True)\n","print(model_acc_df)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["             Model  Accuracy\n","0               LM    0.4725\n","1              GPT    0.3975\n","2      Transformer    0.3725\n","3             LSTM    0.3287\n","4  encoder-decoder    0.2900\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"StU7RQ6Bafcr","colab_type":"code","outputId":"215fbaa6-2ff6-4464-bbc5-e6730c9ab627","executionInfo":{"status":"ok","timestamp":1556873230070,"user_tz":420,"elapsed":518,"user":{"displayName":"ALLEN CHEN","photoUrl":"","userId":"13117472557552131577"}},"colab":{"base_uri":"https://localhost:8080/","height":301}},"source":["import seaborn as sns\n","import matplotlib.pyplot as plt\n","sns.set(style=\"whitegrid\")\n","title = \"BERT Next Sentence Classificaion Accuracy Evaluation\"\n","ax = sns.barplot(x=\"Accuracy\", y=\"Model\", data = model_acc_df, orient = 'h')\n","plt.title(title)\n","plt.xlabel(\"Next Sentence Accuracy\")\n","plt.show()"],"execution_count":0,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAdwAAAEcCAYAAABtSwVBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xtcj/f/P/DHu1JSDjXnMTbWaUnn\nSOSQiHTOYdjIqTHGaMwcljnGbBgzYj7Gto+kGrLZHGemyddhphaNplKZQgf1Pr1+f/i5Pprwduh6\nJ4/77eZ2632dXs/r1bv3w+u6rvd1KYQQAkRERFStDPRdABER0fOAgUtERCQDBi4REZEMGLhEREQy\nYOASERHJgIFLREQkAwYuEekkJSUF3bp1q7btz5kzB6tXr5Zef/311/D09ISTkxOKiorg5OSEy5cv\nP9U2qOZYtWoVpk2bVi3bTk1NRZ8+fapl24+CgUuV9OzZEw4ODnBycoKbmxvGjh2LK1euSPNnzJgB\ne3t7ODk5Sf8CAgIAANnZ2bC2tpam9+zZE+vWrZPWvXsdGxsbqR0nJyd8991399QyY8YMWFtb48yZ\nM9K0rKwsWFtbP/F+6hIeeXl5mDhxIjw8PODi4gJ/f3/s2LHjidu+009qtfqJt/W0nTlzBmPGjIGr\nqyvc3d0RFhaG+Ph4WdqeN28eJkyYAABQqVRYvHgxNm7ciJMnT8LCwgInT55E69atn1ob1WHHjh2w\ntrZGcnJytbWhbw/6DKgprK2tkZWVJb12dXXFDz/8oMeKbjPSdwFU86xduxaenp6oqKjAhx9+iI8+\n+ghr1qyR5o8aNQpTpky57/rHjx+HkZERfv/9dwwfPhyvvfYaunTpgpMnT0rL9OzZE/Pnz4enp+cD\na2nUqBE+/fRTbNy48cl37BFFRUXBxsYGBw4cgLGxMTIyMnD16lXZ65DLyZMnERERgbfeegtLliyB\nhYUF/vjjD6xfvx6hoaGy1nLt2jVUVFSgffv2srb7pBISEtCoUSMkJiaiX79+srat0WhgaGgoS1sP\n+wygqnGES/dlYmKCvn37IjMz87HW79ChA9q3b4+0tLTHriEoKAh//vknfvvttyrnFxcXY+bMmfDy\n8kLXrl3xySefQKPRAADmzp2LiRMnSssuXboUb775JsrKyjBmzBgUFBRI/0PPz8+/Z9tnz55FSEgI\n6tWrByMjI9jZ2cHb21uaf+rUKQwePBiurq4ICAhASkqKNG/48OH49NNPMXjwYDg5OSEiIgKFhYUA\ngGHDhgEA3Nzc4OTkJP1HZPv27fDz84ObmxtGjRqFnJwcaXvW1tb45ptv4OvrC1dXV0RHR+Pum8Rt\n27YNfn5+cHJyQr9+/fDHH38AAPLz8zFx4kR06tQJPXv2xObNm+/b1zExMQgKCsLYsWNhaWkJhUIB\ne3t7rFixosrl161bBx8fH6nNH3/8UZqXlZWFYcOGwcXFBR4eHpg8eTIAQAiBhQsXonPnznB2dsaA\nAQOQkZEB4PbI6ZNPPsHFixfRt29fqY/eeOMNqQ/ujFrKy8uxePFi9OjRAy4uLhgyZAjKy8sBAJMm\nTUKXLl3g4uKCoUOH4vz581Jdd9q4u9969+4Nd3d3REZGVnofPKzP/y0nJwfHjx/HvHnzcOTIkXv+\nc/bTTz8hMDAQzs7O8PHxweHDhwEA169fx/vvvw8vLy+4ublh/PjxAG6PlocMGVJpG3f3wYwZMzB3\n7lyMGTMGjo6OSElJwcGDBxEUFARnZ2d4e3tj1apVldZPTU2V3rPe3t7YsWMHzpw5A09PT+nvBgD2\n7t37WKPW0aNHY8uWLZWmBQQEYO/evQCA+fPnw9vbG87OzggJCUFqamqV26nqCFTPnj1x9OhRALeP\nxAwaNAiurq7w8vLCvHnzoFQqAQBDhw4FAAQGBsLJyQnJycn3bC8zMxPDhw+Hq6sr+vfvj3379knz\nZsyYgejoaIwdOxZOTk4IDw/H33///ch9USVBdJcePXqIX375RQghRFlZmXjvvfdEVFSUNH/69Oli\n+fLlVa57+fJlYWVlJVQqlRBCiJMnTwoHBwexd+/eB7ZzP3fa+s9//iMGDx4shBDi0qVLwsrKSlpm\n/PjxYvbs2aK0tFT8888/IjQ0VHzzzTdS/b6+viI+Pl4cP35cuLu7iytXrgghhDh27Jjo2rXrA9t/\n8803xaBBg8SuXbtETk5OpXl5eXnC3d1dHDx4UGg0GnHkyBHh7u4url27JoQQYtiwYaJXr17ir7/+\nErdu3RLDhg0TS5curbKfhBDixx9/FD4+PuLChQtCpVKJ1atXi0GDBknzraysxNixY8WNGzdETk6O\n8PDwEIcOHRJCCJGcnCy8vLzE6dOnhVarFZcuXRLZ2dlCo9GI4OBgsWrVKlFRUSH+/vtv0bNnT3H4\n8OF79rWsrEzY2NiIX3/99b798e8+S05OFnl5eUKj0Yjdu3eLjh07ivz8fCGEEFOmTBFr1qwRGo1G\nlJeXi+PHjwshhDh8+LAIDg4WN27cEFqtVly4cEFa5+73VlV9ZGVlJS5duiSEEOLDDz8Uw4YNE3l5\neUKtVosTJ06IiooKIYQQcXFxori4WFRUVIj58+eLgIAAaRt3t3H06FHh7u4uzp49KyoqKsS8efPE\n66+/rlOfV+Wzzz4ToaGhQggh/P39xYYNG6R5p0+fFs7OzuLIkSNCo9GIvLw8ceHCBSGEEGPGjBHv\nvPOOuH79ulAqlSIlJUUIIUR8fLz0vq+qD6ZPny6cnZ1Famqq1M/Hjh0T6enpQqPRiLS0NNG5c2fx\n448/CiGEyM7OFo6OjmLnzp1CqVSKwsJCce7cOSGEEH5+fuLgwYNSO+PHj69U/90e9BmQkJBQ6X17\n/vx54eLiIv1uEhMTRWFhoVCpVGLDhg3C09NTlJeXCyGEWLlypZg6daoQouq/z7s/M37//Xdx8uRJ\noVKpxOXLl0Xfvn3Fl19+WWU//Xt7SqVS+Pj4iM8//1xUVFSIo0ePCkdHR5GZmSntn7u7uzh9+rRQ\nqVTi3XffFZMnT65yfx8VR7h0jwkTJsDV1RWurq745ZdfMGrUqErzN27cKM13dXXF9OnTK83v1KkT\nHBwcMGjQILz++uvw8fF5onoGDx6MK1eu4NChQ5Wm//PPPzh06BBmzpyJevXq4YUXXsCIESOwe/du\nAICpqSliYmKwePFiREVFYfbs2WjevLnO7a5YsQKurq5Ys2YNevXqhcDAQOl8clJSErp16wZvb28Y\nGBigS5cusLe3r1RjSEgIXn75ZdStWxd9+/Z94Ej/22+/xdixY9GuXTsYGRkhMjISaWlplUa5Y8aM\nQYMGDdCyZUt4eHggPT0dwO2R8ejRo+Hg4ACFQoE2bdrgxRdfxO+//47CwkK8/fbbMDY2RuvWrTFw\n4MAqzy/evHkTWq0WTZo00bl//Pz80KxZMxgYGKBfv35o06aN1D9GRkbIzc1FQUEBTExM4OrqKk0v\nLS3FX3/9BSEE2rVrh6ZNm+rcJgBotVrEx8fjgw8+QLNmzWBoaAhnZ2cYGxsDAMLCwmBubg5jY2NM\nnDgR6enpKC4uvmc7O3fuRGhoKF577TUYGxvj3XffxalTp5CdnS0tc78+r0pSUhL8/f0BAP7+/khM\nTJTmbd++HaGhoejSpQsMDAzQrFkztGvXDgUFBTh8+DCio6PRsGFD1KlTB+7u7jr3Ra9eveDi4gID\nAwOYmJjAw8MD1tbWMDAwgI2NDfr37y8dHdq1axc8PT3h7++POnXqwMLCAra2tgBuH0m6cx3F9evX\nceTIEWlfqnK/zwAfHx+kp6dL79udO3eid+/e0u8mMDAQFhYWMDIyQkREBJRKJS5evKjz/t5hb28P\nR0dHGBkZoVWrVhg0aBCOHz+u07qnT59GWVkZxo4dC2NjY3Tu3Bk9evSQPjfu7IeDgwOMjIwQEBDw\nREfp7sZzuHSP1atXS4eY9u3bh+HDh2P37t3Sh3FERMQDz98cO3YMCoUCmzdvxs6dO6FSqaQ/uMdh\nbGyM8ePHY8WKFZUOB+bm5kKtVsPLy0uaptVq0aJFC+l1x44d0apVKxQWFsLPz++R2m3YsCGmTZuG\nadOmobCwEDExMZgwYQIOHz6M3NxcfP/99zhw4IC0vFqthoeHh/T67vAyNTVFWVnZfdvKzc3FwoUL\nsWTJEmmaEAL5+fl48cUXq9xeaWkpAODKlSt46aWX7tlmTk4OCgoKpLADbp/nu/v1HQ0aNICBgQGu\nXr2Kdu3aPbBf7khMTMSXX34pfbiWlZWhqKgIwO3z3ytWrEBYWBgaNmyIkSNHIiwsDJ07d8bQoUMx\nb9485OTkwNfXF9OnT4e5ublObQJAUVERKioqqryASqPR4JNPPsH333+PwsJCGBgYSOvUr1+/0rIF\nBQV47bXXpNdmZmZo1KgR8vPz0apVKwD37/N/O3HiBLKzs9G/f38AtwP3k08+QVpaGmxtbXHlypVK\npyPuyMvLQ8OGDdGwYUOd9/9ud7/XgdthsmzZMpw/fx4qlQpKpVI6PH+/9wlwOwj9/PxQVlaGPXv2\nwNXV9YH/EbrfZ4C5uTm8vb2xe/dujB07Frt27cL8+fOl+Rs2bMD27dtRUFAAhUKBkpIS6T3zKC5e\nvIjFixfj7NmzuHXrFjQaTaXf5YMUFBSgefPm0nsDAFq2bFnpdELjxo2ln+vWrfvAv91HwcCl+zI0\nNISvry/mzJmDEydOSH+4uq47cuRI7N27F19//TVGjBjxRLWEhIRg/fr10rkgAGjevDmMjY1x7Ngx\nGBlV/VbeunUrVCoVmjZtitjYWIwbNw4AoFAoHql9S0tLREREICEhAdevX0eLFi0QGBhY6cNEV1W1\n3aJFC0RGRj7WebMWLVpUeY6pRYsWaNWqVaU+ux9TU1M4Ojpi79696NSp00OXz8nJwaxZs7Bp0yY4\nOTnB0NAQgYGB0vwmTZpIfZOamoqRI0fCzc0Nbdq0wRtvvIE33ngD165dw+TJkxEbGyud49WFhYUF\nTExMcPnyZdjY2FSat3PnTuzbtw9ffvklWrVqheLiYri5uVV57rVp06aVjiCUlZXh+vXraNasmc61\n3JGYmAghBIKCgipNT0hIgK2t7X1/R82bN8eNGzdw8+ZNNGjQoNI8U1NT6bw0AJ0u2Js6dSqGDRuG\n2NhYmJiYYMGCBVKgtWjRotIV/3dr1qwZnJycsHfvXiQlJd1z7vhR+Pv747PPPoObmxsqKiqk/4Sm\npqYiNjYWmzZtwquvvgoDA4P7/m7+ve8ajUa6BgIAPvzwQ9jZ2eHjjz+Gubk5Nm3apPNVyE2bNkVe\nXh60Wq0UuleuXEHbtm0fe591xUPKdF9CCPz000+4efOmzqOefxs7dixiY2NRUVHxRLUYGRlh4sSJ\niI2NlaY1bdoUXbp0weLFi1FSUgKtVou///5bOoR28eJFfPrpp1i6dCliYmIQGxsrHRp64YUXcP36\n9SoPNd6xdOlSZGRkQK1Wo6SkBN988w3atGkDCwsLBAQE4MCBA/j555+h0WhQUVGBlJQU5OXlPXRf\nLC0tYWBgUOk7pYMHD8a6deukC3yKi4uxZ88enfomLCwMGzduxNmzZyGEQFZWFnJycuDg4AAzMzOs\nW7cO5eXl0Gg0yMjIuO+HblRUFBISEhAbGyt9SKenp1c5krl16xYUCgUsLS0BAPHx8ZUuTtqzZ4/U\nFw0bNoRCoYCBgQHOnDmD06dPQ6VSwdTUFMbGxpVGGrowMDBAaGgoFi1ahPz8fGg0Gpw8eRJKpRKl\npaUwNjaGhYUFbt26heXLl993O3e+5pWWlgalUonly5fDwcFBGt3qqqKiAnv27MG8efOQmJgo/Zs9\nezZ27doFtVqNsLAw7NixA7/++iu0Wi3y8/ORmZmJpk2bolu3boiOjsaNGzegUqmkQ6M2NjY4f/48\n0tLSUFFRcc8FUFUpLS1Fw4YNYWJigjNnzmDXrl3SvAEDBuDo0aNITk6GWq1GUVFRpUOlgYGB2LBh\nAzIyMuDr6/tIfXA3b29v5ObmYuXKlejXr5/0+y0tLYWhoSEsLS2hVqvx2WefoaSkpMptvPzyy6io\nqMDBgwehUqnw+eefSxdF3dmWmZkZzMzMkJmZiW+++abS+o0bN77vd7YdHBxQt25dxMbGQqVSISUl\nBfv375flqnIGLt0jMjISTk5OcHZ2xqefforFixfj1VdfleZv2LCh0nfw7j6M+m/du3dHw4YNsW3b\ntieuy9/f/55zjDExMVCpVOjXrx/c3NwwadIkXL16FWq1GlFRURgzZgxsbGzQtm1bTJkyBe+99x6U\nSiXatWuH/v37w8fHB66urlVepVxeXo63334bbm5u8PHxQW5uLj7//HMAt0cLa9aswRdffIHOnTvD\n29sbGzZsgFarfeh+mJqaIjIyEkOGDIGrqytOnTqF3r17Y/To0Xj33Xfh7OwMf39/6SrWh/Hz80Nk\nZCSmTp0KZ2dnTJgwATdu3IChoSHWrl2L9PR09OrVC506dcKsWbPu+yHn7OyM//znPzh27Bh8fHzg\n7u6O2bNnV3kotH379oiIiMDgwYPh6emJjIwMODs7S/N///13hIeHw8nJCW+99RY++OADtG7dGqWl\npZg1axbc3d3Ro0cPNGrU6J5rBHQxffp0WFlZISwsDO7u7li2bBm0Wi2CgoLQsmVLdO3aFf3794ej\no+N9t+Hp6Yl33nkHEydOhJeXFy5fvlzplIWufvrpJ9StWxdBQUFo0qSJ9C80NBQajQY///wzHBwc\nsGjRIixcuBAuLi4YNmwYcnNzAdx+DxsZGcHPzw+enp74z3/+A+B26EyYMAEjRoyAr68vXFxcHlrL\n3LlzsXLlSjg5OWH16tWVTqO0bNkS69evx5dffgl3d3cEBQVVOifdu3dv5OTkoHfv3jA1NX1gOw/6\nDDA2Nkbv3r1x9OjRSueB73yToE+fPujZsydMTEzuOSR+R/369TF37lzMmjUL3bp1g6mpaaXrL6ZP\nn45du3bB2dkZs2fPvics3377bcyYMQOurq73XLNgbGyMtWvX4vDhw+jUqROio6MRExPz2IOKR6EQ\nVY3niYjouePj44N58+Y99Pvx9Hg4wiUiIvzwww9QKBQ6ncOnx8OLpoiInnPDhw/HhQsXEBMT88jn\n1El3PKRMREQkA/5XhoiISAY8pPwc02q1KC0tRZ06dR75e6lERM8rIQRUKhXMzMwe6RA8A/c5Vlpa\nKt04noiIHo2VldU9dzB7EAbuc6xOnToAbr9pnuTWi7XJ2bNnYW9vr+8yagT2RWXsj/953vtCqVQi\nIyND+gzVFQP3OXbnMLKxsTFMTEz0XE3Nwb74H/ZFZeyP/2FfPPotYnnRFBERkQwYuER30eX2ec8L\n9kVl7I//qQ19oVRpZG+Th5QJ7yxKws0ylb7LICKSzdcxQ2VvkyNcIiIiGTBwiYiIZMDAJSIikgED\nl4iISAYMXCIiIhkwcImIiGTAwCUiIpIBA5eIiEgGDFwiIiIZMHCJiIhkwMAlIiKSAQOXiIhIBgzc\nGq5nz57IyMioNG348OGwt7fH9evXpWkpKSmwtrbGkiVL5C6RiIh0wMB9RllZWWH37t3S6x07duC1\n117TY0VERPQgDNxnVFBQEJKSkgAApaWlOHHiBLp27arnqoiI6H74PNxnVOvWrWFiYoLMzEycOnUK\nPj4+MDIyglKp1HdpRERUBY5wn2FBQUFISEhAYmIigoOD9V0OERE9AAP3Gda3b1/s3r0bZWVlsLa2\n1nc5RET0ADyk/AwzMzNDVFQUGjdurO9SiIjoIRi4z4CRI0fC0NBQet2oUSPp5379+umjJCIiekQM\n3Bpu//79Oi87ceLEaqyEiIieBM/hEhERyYCBS0REJAMGLhERkQwYuERERDJg4BIREcmAgUtERCQD\nBi4REZEMGLhEREQyYOASERHJgIFLREQkAwYuERGRDBi4REREMuDDCwgr3g+EiYmJvssgIpKNUqWB\ncR3Dhy/4FHGES3SXEydO6LuEGoN9URn7439qQ1/IHbYAA5eIiEgWDFwiIiIZMHCJiIhkwMAlIiKS\nAQOXiIhIBgxcIiIiGTBwiYiIZMDAJbqLi4uLvkuoMdgXlbE/AK1ape8Snmm80xTh7BczgPJifZdB\nRDWcy3ux+i7hmcYRLhERkQwYuERERDJg4BIREcmAgUtERCQDBi4REZEMGLhEREQyYOASERHJgIFL\nREQkAwYuERGRDBi4REREMmDgEhERyYD3Uq4hVCoV1q5di127dsHIyAiGhoZo27YtJk2ahDNnzmDh\nwoV48cUXoVKp0K5dO3z00Uf44IMPkJ2dDQBIT0+HlZUVDAwM0LhxY2zYsEHPe0RERHdj4NYQ77//\nPsrLyxEXF4cGDRpACIFDhw7h4sWLAABPT0+sXLkSWq0WkydPxueff47Vq1dL61tbW+Pbb7+FmZmZ\nvnaBiIgegIFbA1y6dAk//fQTDh06hAYNGgAAFAoFunfvDgDYsWOHtKyBgQE8PDxw6NAhfZRKRESP\niedwa4Bz586hTZs2aNiw4UOXVSqV2L9/P2xtbWWojIiInhaOcGugCxcuYOrUqSgvL0fXrl1hZ2eH\no0ePIjAwEADg7OyMcePG6blKIiJ6FAzcGsDOzg5ZWVm4efMmGjRogPbt2yMpKQlbtmzB2bNnAfzv\nHC4RET2beEi5Bmjbti169eqFWbNmobi4WJpeVlamx6qIiOhp4gi3hli0aBHWrFmDsLAwGBkZoUGD\nBmjatCnGjh2LP//8U9/lERHRE2Lg1hDGxsaYPHkyJk+efM+81157DSEhIQ9cn6FMRFSz8ZAyERGR\nDBi4REREMmDgEhERyYCBS0REJAMGLhERkQwYuERERDJg4BIREcmAgUtERCQDBi4REZEMGLhEREQy\nYOASERHJgIFLREQkAz68gGA/bjFMTEz0XQYR1XBatQoGRnX0XcYziyNcorucOHFC3yXUGOyLytgf\nYNg+IQYuERGRDBi4REREMmDgEhERyYCBS0REJAMGLhERkQwYuERERDJg4BIREcmAgUt0FxcXF32X\nUGOwLyqrDf2hVKv0XcJzjXeaIkyLi0axqlTfZRBRNds0coW+S3iucYRLREQkAwYuERGRDB54SPn1\n11+HQqF46Ea2bt361AoiIiKqjR4YuOHh4XLVQUREVKs9MHCDg4PlqoOIiKhW0/kcrhAC27Ztwxtv\nvIEBAwYAAI4fP47k5ORqK46IiKi20DlwV6xYge3bt2PQoEG4cuUKAKB58+aIjY2ttuKIiIhqC50D\nNyEhAWvXrkX//v2lC6latWqFy5cvV1txREREtYXOgavRaGBmZgYAUuCWlpaiXr161VMZERFRLaJz\n4Hp7e2PRokVQKpUAbp/TXbFiBXr06FFtxREREdUWOgfu+++/j6tXr8LFxQXFxcVwcnJCbm4upk2b\nVp31ERER1Qo630vZ3Nwcq1evxj///IPc3Fy0aNECTZo0qc7aiIiIao0HBq5Wq71nmqWlJSwtLSvN\nNzDQ/x0iw8PDoVQqoVKpcOnSJbz66qsAADs7OyxatKha296yZQu2bNkCExMT/Pe//0XdunWrtT0i\nInr2PDBw7ezsdLq1Y1pa2lMr6HHFxcUBALKzsxEaGoqkpKQql9NoNDA0NHyqbW/evBmffvop7Ozs\ndF5HCAGtVvvUawEAtVoNIyM+CIqIqCZ54Kfyvn37pJ8PHjyIH374AePGjUPLli2Rm5uL9evXw9fX\nt9qLfBJHjx5FTEwMrKyskJ6ejqlTp6KoqAhbtmyBWq2GQqHAjBkz4OHhAQDo1q0bwsLCcOTIEfzz\nzz8YPXo0Xn/9dWg0GkRHR+O3336DsbEx6tevj61bt2LSpEnIzc3F1KlT0aFDB8TExCA+Ph6bNm0C\nALRt2xbR0dGwtLREXFwc9uzZg7p16+Lvv//G8uXLMXfuXDg6OuLUqVPIzc3FyJEjYWFhga+//hpX\nr17FjBkzpD4+efIkli9fjtLS24/Se+edd+Dt7Y2srCwMGTIEAwYMQEpKCoYOHcrbchIR1TAPDNwX\nX3xR+nnTpk2Ij49HgwYNAAAvv/wy7O3tERoaitdff716q3xCf/75J+bNmwcHBwcAQFFREYKCggAA\nFy5cwOjRo3Hw4EFpeaVSiW3btuHy5csICAhASEgIMjIycPLkSSQnJ8PAwAA3btwAAKxcuRLdunXD\nZ599hnbt2iE9PR0rVqxAfHw8mjRpgo8//hgLFizAxx9/DAA4deoUvvvuO7Rq1Upqr6CgAFu3bkVB\nQQH69OmDiIgI/Pe//8X//d//Ydq0afD19cX169cRHR2N2NhYNG7cGPn5+QgPD5fu9HXt2jU4Ozvj\n/fffl6NLiYjoEel83LG4uBi3bt2SAhcAysvLUVxcXC2FPU3t2rWTwhYAsrKyMHXqVBQUFMDQ0BD5\n+fkoLCyUzk33798fANC6dWuYmZkhPz8fbdq0wa1btzBr1ix4eHjc9+tQx44dQ/fu3aULygYNGoSB\nAwdK893c3CqFLQD4+fnBwMAAzZs3R/369aURrb29PXJycqBSqXDixAlkZ2dj1KhR0noKhQKXL19G\nvXr1UK9ePfTp0+cp9BYREVUHnQM3ODgYI0eOxJtvvonmzZsjLy8PX3311TPxgIN/35xjypQpmDNn\nDnr06AGNRoOOHTtK3y8GAGNjY+lnAwMDqNVqNGzYELt378axY8fw66+/YtmyZUhMTMQLL7zwRLVU\n1d6d13cuRtNqtRBCwM7ODps3b75n/aysLN6AhIiohtP58uKoqCgMHz4cycnJWLx4MXbv3o2hQ4ci\nKiqqOuurFsXFxdIoc9u2bVCpVA9d59q1a6ioqIC3tzeioqJgamqKnJyce5br1KkTDh48iGvXrgG4\nfTGXp6fnE9fs7OyMzMxMHD9+XJp2+vTpJ94uERHJQ+cRroGBAYYMGYIhQ4ZUZz2ymDlzJsaNG4eG\nDRvC29sb9evXf+g6OTk5mDt3LtRqNbRaLXr16oUOHTrcs5yNjQ3eeecdjBgxAgDQpk0bzJs374lr\ntrS0xJo1a7B06VLcvHkTarUarVu3xhdffPHE2yYiouqnEEIIXReOj49HUlIS8vPz0axZMwQGBiI0\nNLQ666NqVFFRgbNnz2JTWjwgj5esAAAan0lEQVSKVaX6LoeIqtmmkSueynZOnDgBFxeXp7KtZ9Gd\nz057e3uYmJjovJ7OI9zPP/8ciYmJiIiIkL4WFBsbi4KCArz11luPVTQREdHzQufAjYuLw1dffVXp\nq0JeXl4YNmwYA5eIiOghdL5o6tatW9LXZu5o1KgRysvLn3pRREREtY3Ogdu1a1dMmzYNf/31F8rL\ny5GZmYkZM2bAy8urOusjIiKqFXQO3Dlz5sDMzAwBAQFwcnJCUFAQ6tati9mzZ1dnfURERLXCQ8/h\n5ubmSj9PnjwZEydORFFRESwsLGBoaIiSkpJKd58iIiKiez00cHv27HnfJwYJIaBQKGrE04KIiIhq\nsocGro2NDcrLyxEcHIyAgAA0bdpUjrqIiIhqlYcGbmJiIjIyMpCQkIAhQ4agXbt2CAwMhK+vLx+0\nTkREpCOdLpqysrLC9OnTsX//fowYMQIHDx6El5cX/vjjj+quj4iIqFbQ+SplALh06RKOHz+OU6dO\nwdbWlhdLERER6eihh5SvX7+O3bt3IyEhAaWlpQgMDMSWLVvQsmVLOeojIiKqFR4auF27dkWrVq0Q\nGBiIjh07Arj9/NWsrCxpmc6dO1dfhVTtloXPfaQbcBPRs0mpVsHYqI6+y3huPTRwmzRpgoqKCmzb\ntg3btm27Z75CocC+ffuqpTgiuT3vT0G5G/uistrQHwxb/Xpo4O7fv1+OOoiIiGq1R7poioiIiB4P\nA5eIiEgGDFwiIiIZMHCJiIhkwMAlIiKSAQOXiIhIBgxcors869+zfJrYF5U9zf7QKFVPbVv07Hjo\n93Cp9jvwbhS0xcX6LoPoudFv85f6LoH0gCNcIiIiGTBwiYiIZMDAJSIikgEDl4iISAYMXCIiIhkw\ncImIiGTAwCUiIpIBA5eIiEgGDFwiIiIZMHCJiIhkwMAlIiKSAQNXJj179kRGRkalaceOHUN4eDgC\nAwPh5+eHN954A1qtFhMmTEBgYCACAwNhbW2NAQMGIDAwEKNGjQIAWFtbIzQ0tNK2Vq1aBWtraxw4\ncEC2fSIiIt3x4QV6olarMWnSJGzevBk2NjYAgHPnzkGhUGD16tXSctbW1vj2229hZmZWaX0hBC5c\nuID27dtDCIFdu3bByspK1n0gIiLdcYSrJ6WlpSgrK0Pjxo2laXZ2dlAoFDqtHxwcjB07dgAAUlJS\nYGVlhUaNGlVLrURE9OQYuHrSsGFDDBw4EL6+voiMjMS6detw5coVndfv27cv9u3bB41Gg4SEBAQH\nB1djtURE9KQYuHo0Z84cJCUloVevXvj999/h7++PS5cu6bRuvXr14OjoiB9//BEnTpxAt27dqrdY\nIiJ6IgxcPWvdujXCw8OxatUqODk5PdJFT8HBwZg7dy58fHxgZMTT8URENRkDV09KS0tx5MgRCCEA\nADdv3kR2djZatWql8zY8PDwwbtw4DB06tLrKJCKip4TDIhmNHDkShoaGAICKigo4Ozvjo48+gomJ\nCTQaDQYMGIDevXvrvD2FQoGIiIjqKpeIiJ4iBq5M9u/f/1jr/fnnnzpNA4CvvvrqsdogIqLqx0PK\nREREMmDgEhERyYCBS0REJAMGLhERkQwYuERERDJg4BIREcmAgUtERCQDBi4REZEMGLhEREQyYOAS\nERHJgIFLREQkAwYuERGRDPjwAkKP5UthYmKi7zKInhsapQqGxnX0XQbJjCNcorucOHFC3yXUGOyL\nyp5mfzBsn08MXCIiIhkwcImIiGTAwCUiIpIBA5eIiEgGDFwiIiIZMHCJiIhkwMAlIiKSAQOX6C4u\nLi76LqHGeB77Qq3S6LsEqsV4pynC6mV7UF6m1ncZRHo3c0GYvkugWowjXCIiIhkwcImIiGTAwCUi\nIpIBA5eIiEgGDFwiIiIZMHCJiIhkwMAlIiKSAQOXiIhIBgxcIiIiGTBwiYiIZMDAJSIikgEDl4iI\nSAa1OnBTUlIQEhJSLdsePnw4Dhw4UC3b/jdra2uUlpbK0hYREVWPWh24j0ujefYf0aVW8+k/REQ1\niWyP5zt9+jSWLVsmjdQmTZqE9u3bIzQ0FIMHD8ahQ4dw69YtLFiwAK6urgCAAwcOYNWqVVCr1TAw\nMMDixYthY2ODw4cPY/ny5dBoNLC0tMS8efPQpk0bAMAnn3yC5ORkNGjQAO7u7pVqSEhIwNdffw2N\nRgNzc3N8+OGHeOWVV7Bjxw589913MDMzQ1ZWFpYuXQpbW9tK6164cAHvv/8+ysrKYGVlhYqKCmle\nQUEB5s+fj9zcXFRUVKB///6IjIwEAGRmZmLBggW4evUqACAiIgLBwcHIysrCnDlzUFhYCCMjI0yZ\nMgXdunUDAOzduxfLly+HiYkJfH19H9qP3bt3R3Z2NkJDQxESEoJjx45h4MCBGDJkyFP53RER0ZOT\nJXBv3ryJuXPnYt26dWjatCkKCgoQFhaGL774AtevX4ejoyOmTJmC7777DsuWLcO3336LixcvYtas\nWdi6dSvatm0LpVIJpVKJa9eu4b333sOWLVvQvn17xMXFYdq0aYiLi8P+/fuxf/9+JCYmom7dupgw\nYYJUQ2pqKvbs2YOtW7fC2NgYhw4dwsyZM/Htt98CuB1kSUlJeOmll6rch/feew/Dhw9HcHAwTp06\nVSnMpk+fjvHjx8PNzQ1KpRIjRoxAhw4d4OHhgfHjx2Py5Mnw8/MDABQVFQEApk2bhoEDByI8PBwX\nLlzA0KFDsWfPHmi1WsyePRvffPMNXnnlFaxfv/6h/bhr1y4AwPXr19GhQwdMnz796f4CiYjoickS\nuCdPnkR2djbGjBkjTVMoFFCr1ahXrx569OgBAHB0dMSSJUsAAEePHkW3bt3Qtm1bAICxsTGMjY3x\n22+/wcbGBu3btwcAhIaGIjo6GiUlJUhJSUG/fv1gZmYGAAgLC8OaNWsAAPv370d6ejrCw8MBAEII\n3Lx5U6rH2dn5vmFbUlKCjIwMBAYGSnVaWVkBAMrKyvDbb7+hsLBQWr60tBSZmZlo2rQp1Gq1FLYA\nYGFhgZKSEqSlpSE0NBQA0L59e9ja2uLUqVMQQsDOzg6vvPIKAGDQoEFYtmzZA/sxKysLFhYWMDEx\nqdQWERHVHLIErhAC1tbW2Lp1a6Xp2dnZMDY2ll4bGBhU27lHIQRCQ0PxzjvvVDn/TkgDwJ9//on3\n3nsPAODh4YFJkybdd7tarRYKhQLbt29HnTp1Ks07f/78U6j8f+7Xj8DtvjQ1NYVCoXiqbRIR0dMh\ny0VTTk5OyMrKwrFjx6RpZ86cgRDivut06dIFhw8fxqVLlwAASqUSJSUlcHR0RHp6OjIzMwHcPi9r\nZ2cHc3NzdOrUCXv27EFZWRk0Gg3i4+Ol7fXs2RNJSUnIy8sDcPvCqLNnz1bZtrW1NZKSkpCUlISZ\nM2fC3NwcVlZW2Llzp1R7RkYGAMDc3BwuLi5Yt26dtP6VK1dw9epVvPzyyzAyMsKePXukeUVFRTA3\nN4etrS0SEhIA3D7Pm56eDkdHRzg6OuLcuXPSfsfFxT1RPxIRUc0gywi3YcOGWLNmDZYuXYqFCxdC\npVKhdevWmD179n3Xadu2LT766CNMmTIFGo0GhoaGWLx4MaytrRETE4Np06ZBrVbD0tISS5cuBQD0\n6NEDp06dQmBgoHTRVH5+PgDAzc0NkydPxltvvQWNRgOVSoW+ffvC3t5ep32IiYnB+++/j/Xr18PK\nygodOnSQ5i1btgyLFi3CgAEDANweLS9YsABNmjTBmjVrMG/ePKxZswYKhQIREREICgrCsmXLMGfO\nHGzatAlGRkaIiYmBpaUlAOCjjz5CZGQk6tatW+miqfv149q1ax/tF0JERLJTCA6PnlsVFRU4e/Ys\nDn1/GeVl/BoR0cwFYfedd+LECbi4uMhYTc31vPfFnc9Oe3t7mJiY6Lwev4dLREQkAwYuERGRDBi4\nREREMmDgEhERyYCBS0REJAMGLhERkQwYuERERDJg4BIREcmAgUtERCQDBi4REZEMGLhEREQyYOAS\nERHJQJanBVHNNmGa3yPdgJuotlKrNDCqY6jvMqiW4giX6C4nTpzQdwk1xvPYFwxbqk4MXCIiIhnw\nkPJz7M6jkJVKpZ4rqVkqKir0XUKNwb6ojP3xP89zX9z5zHzUx8nzAfTPseLiYmRkZOi7DCKiZ5KV\nlRXq16+v8/IM3OeYVqtFaWkp6tSpA4VCoe9yiIieCUIIqFQqmJmZwcBA9zOzDFwiIiIZ8KIpIiIi\nGTBwiYiIZMDAJSIikgEDl4iISAYMXCIiIhkwcImIiGTAwCUiIpIBA/c5cPHiRQwaNAh9+vTBoEGD\ncOnSpXuW0Wg0iI6Oho+PD3r37o24uDj5C5WJLv1x5MgRhISEwN7eHkuWLJG/SJno0herV69G//79\nMWDAAISEhODnn3+Wv1CZ6NIf8fHxGDBgAAIDAzFgwABs3rxZ/kJloEtf3PHXX3+hY8eOtfpv5akQ\nVOsNHz5cJCYmCiGESExMFMOHD79nmYSEBBERESE0Go24du2a6Nq1q7h8+bLcpcpCl/64dOmSOHfu\nnFi+fLlYvHix3CXKRpe+OHz4sCgrKxNCCJGWliZcXFzErVu3ZK1TLrr0R3FxsdBqtdLP3bt3F2lp\nabLWKQdd+kIIIdRqtRg2bJh49913a/XfytPAEW4td+3aNZw7dw7+/v4AAH9/f5w7dw6FhYWVlktO\nTkZ4eDgMDAxgaWkJHx8ffP/99/oouVrp2h9t2rSBra0tjIxq7/M9dO2Lrl27wtTUFABgbW0NIQSu\nX78ue73VTdf+MDc3l26FWl5eDpVKVetujaprXwDAunXr0L17d7Rt21bmKp89DNxa7sqVK2jWrBkM\nDW8/59PQ0BBNmzbFlStX7lmuZcuW0usWLVogLy9P1lrloGt/PA8epy8SExPx0ksvoXnz5nKVKZtH\n6Y99+/ahf//+6NGjB0aPHg1ra2u5y61WuvZFeno6jhw5ghEjRuihymcPA5eIdPLbb79hxYoV+Pjj\nj/Vdit716tULu3fvxg8//ICkpCT89ddf+i5JdiqVCrNnz0Z0dLQUzPRgtfd4GQG4PVLNz8+HRqOB\noaEhNBoNCgoK0KJFi3uWy83NhYODA4B7R7y1ha798Tx4lL44efIkoqKisGbNGrzyyit6qLb6Pc57\no2XLlujQoQMOHjxYq/pFl764evUq/v77b4wdOxYAcPPmTQghUFJSgo8++khfpddoHOHWci+88AJs\nbW2xa9cuAMCuXbtga2sLS0vLSsv17dsXcXFx0Gq1KCwsxE8//YQ+ffroo+RqpWt/PA907YszZ85g\nypQpWLlyJV577TV9lCoLXfsjMzNT+rmwsBApKSmwsrKStdbqpktftGzZEikpKdi/fz/279+PN998\nEwMHDmTYPgAfz/ccyMzMxIwZM3Dz5k00aNAAS5YswSuvvIIxY8Zg0qRJ6NChAzQaDebNm4dffvkF\nADBmzBgMGjRIz5VXD136IzU1Fe+++y5KSkoghED9+vWxYMECdO3aVd/lP1W69EVoaChycnLQrFkz\nab2YmJhad94S0K0/Fi5ciF9++QVGRkYQQiA8PBzDhw/Xd+lPnS59cbdVq1ahrKwM06dP11PFNR8D\nl4iISAY8pExERCQDBi4REZEMGLhEREQyYOASERHJgIFLREQkAwYuERGRDBi4RHrSs2dPdO7cGWVl\nZdK0uLi4p/KdzlWrVmHatGkPXCY1NRWDBw+Gi4sL3N3dMXjwYJw5c+aJ296xYweGDBnyxNupLsOH\nD4ebmxuUSqW+S6HnDAOXSI+0Wq1enqdaUlKCyMhIDBs2DL/99hsOHz6Mt99+G8bGxrLXIqfs7Gyk\npqZCoVBg3759sratVqtlbY9qHgYukR6NGjUKGzduxM2bN6ucn5mZiZEjR8Ld3R19+vRBcnIyAECp\nVCIwMBBfffUVAECj0WDw4MH47LPPcPjwYXzxxRfYs2cPnJycEBAQcM92L168COD2Y9cMDQ1Rt25d\neHl5wcbGRlpm+/bt8PPzg5ubG0aNGoWcnBxpnrW1Nb755hv4+vrC1dUV0dHREEIgMzMTc+fOxalT\np+Dk5ARXV1ep3iVLlqB79+7w9PTEnDlzUF5eDgBISUlBt27dsHHjRnTu3BleXl6Ij4+X2iovL8fi\nxYvRo0cPuLi4YMiQIdK6p06dwuDBg+Hq6oqAgACkpKQ8sL8TExPRsWNHBAcHIzExsdK8B7Vz52iA\nq6srvL29sWPHDgC3R8txcXHSNv49ure2tsbWrVvh6+sLX19fAMD8+fPh7e0NZ2dnhISEIDU1VVpe\no9Fg7dq18PHxgZOTE0JCQnDlyhVER0dj8eLFleqNjIzEpk2bHri/VMPo7Um8RM+5Hj16iF9++UVM\nmDBBLF++XAghxLZt28SwYcOEEEKUlpaKbt26ie3btwuVSiX++OMP4e7uLs6fPy+EEOLPP/8Urq6u\n4sKFC2LNmjUiPDxcqNVqIYQQK1euFFOnTr1v28XFxcLd3V2899574uDBg+L69euV5v/444/Cx8dH\nXLhwQahUKrF69WoxaNAgab6VlZUYO3asuHHjhsjJyREeHh7i0KFDQggh4uPjxeDBgyttb8GCBWLc\nuHGiqKhIFBcXi3Hjxolly5YJIYQ4duyYsLW1FZ9++qlQKpXi4MGDwsHBQarpww8/FMOGDRN5eXlC\nrVaLEydOiIqKCpGXlyfc3d3FwYMHhUajEUeOHBHu7u7i2rVr991vHx8fsWXLFvH7778LOzs7cfXq\nVWne/drJzs4Wjo6OYufOnUKpVIrCwkJx7tw5IYQQw4YNE9u2bZO28e99t7KyEiNGjBBFRUXi1q1b\nQojbD3MvLCwUKpVKbNiwQXh6eory8nIhhBDr168X/v7+IjMzU2i1WpGWliYKCwvF6dOnRZcuXYRG\noxFCCHHt2jXh4OBQqX6q+TjCJdKzSZMmYcuWLfc83PvgwYN48cUXERoaCiMjI9jZ2aFPnz74/vvv\nAQBWVlZ46623MH78eGzcuBExMTE6PybN3NwcX3/9NRQKBWbPno3OnTsjMjIS//zzDwDg22+/xdix\nY9GuXTsYGRkhMjISaWlplUa5Y8aMQYMGDdCyZUt4eHggPT29yraEENi2bRtmzpyJRo0awdzcHOPG\njcPu3bulZYyMjDBhwgTUqVMH3t7eqFevHi5evAitVov4+Hh88MEH0vNZnZ2dYWxsjKSkJHTr1g3e\n3t4wMDBAly5dYG9vj0OHDlVZR2pqKnJzc+Hn5wd7e3u0bt1aujn/g9rZtWsXPD094e/vjzp16sDC\nwgK2trY69TMAjB07Fo0aNULdunUBAIGBgbCwsICRkREiIiKgVCqlIw5xcXF455138Morr0ChUMDG\nxgYWFhZwcHBA/fr18euvvwIAkpOT4e7ujsaNG+tcB+kfH89HpGdWVlbo3r071q1bh3bt2knTc3Jy\ncObMGemwLHD7kOPdh4iDgoLwySefwNfXF23btn2kdtu1aycdpszMzERUVBQWLlyI5cuXIzc3FwsX\nLsSSJUuk5YUQyM/Px4svvggAaNKkiTTP1NQUpaWlVbZTWFiIW7duISQkpNK2tFqt9LpRo0YwMvrf\nx5GpqSnKyspQVFSEiooKtG7d+p7t5ubm4vvvv8eBAwekaWq1Gh4eHlXWkZiYiC5dukhPvPH390dC\nQgJGjBjxwHauXLmCl156qcpt6uLfj/fbsGEDtm/fjoKCAigUCpSUlKCoqAgAkJeXd9+2goOD8d13\n36FLly747rvv8MYbbzx2TaQfDFyiGmDSpEkIDg5GRESENK1FixZwc3PDl19+ed/1oqOj0aNHDxw5\ncgSpqalSOCsUikdqv127dggJCcF///tfqe3IyMgqz/8+zL/btrCwQN26dbF79+5KTxzShYWFBUxM\nTHD58uVK55fv1BgYGIj58+c/dDvl5eXYs2cPtFotunTpAuD2eeWbN28iPT0dVlZWD2znfldvm5qa\n4tatW9LrO0cI7nZ3f6SmpiI2NhabNm3Cq6++CgMDA7i5uUH8/2fING/eHH///XeVj/sLCAiAv78/\n0tPTkZmZCR8fn4fuN9UsPKRMVAO0adMG/fr1ky6CAoDu3bvj0qVLSExMhEqlgkqlwpkzZ6TnsSYm\nJuKPP/7AokWLMGvWLMyYMUMaZb7wwgvIycmpNIq8W2ZmJjZu3Ii8vDwAt0dxu3btQseOHQEAgwcP\nxrp163D+/HkAQHFxMfbs2aPTvrzwwgvIz8+XvnZjYGCA8PBwLFy4ENeuXQMA5Ofn4+eff37otgwM\nDBAaGopFixZJD0Q/efIklEolAgICcODAAfz888/QaDSoqKhASkqKtE93++mnn2BoaIjdu3cjMTER\niYmJSE5OhqurKxITEx/YzoABA3D06FEkJydDrVajqKgIaWlpAABbW1v8+OOPuHXrFrKysrB9+/YH\n7k9paSkMDQ1haWkJtVqNzz77DCUlJdL88PBwrFixApcuXYIQAunp6dLot3nz5ujQoQOioqLg6+sr\nHaKmZwcDl6iGmDBhQqXv5Jqbm2PDhg1ITk5G165d4eXlhWXLlkGpVCI3NxeLFi3CkiVLYGZmhgED\nBsDe3h6LFi0CAPTt2xcA4OHhgeDg4HvaMjc3x+nTpxEeHg5HR0cMHDgQVlZWmDFjBgCgd+/eGD16\nNN599104OzvD398fhw8f1mk/OnXqhPbt28PLy0s6vBsVFYU2bdpg4MCBcHZ2xogRI6Tzlg8zffp0\nWFlZISwsDO7u7li2bBm0Wi1atGiBNWvW4IsvvkDnzp3h7e2NDRs2VPmfjISEBISEhKBly5Zo0qSJ\n9G/o0KHYuXMn1Gr1fdtp2bIl1q9fjy+//BLu7u4ICgqSzle/+eabqFOnDjw9PTF9+nQMGDDggfvi\n5eWFrl27ok+fPujZsydMTEwqHXIeOXIk/Pz8EBERAWdnZ3zwwQeoqKiQ5gcFBSEjIwOBgYE69R3V\nLHweLhHRM+L48eOIiorCgQMHHvm0AekfR7hERM8AlUqFzZs3IywsjGH7jGLgEhHVcJmZmXBzc8PV\nq1cxYsQIfZdDj4mHlImIiGTAES4REZEMGLhEREQyYOASERHJgIFLREQkAwYuERGRDBi4REREMvh/\niyV9e5bfJwoAAAAASUVORK5CYII=\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"code","metadata":{"id":"UfyK6zxTcPLB","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]}]}