006-eager_execution.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# tensorflow教程-eager模式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "\n",
    "import tensorflow as tf\n",
    "tf.executing_eagerly()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.eager模式下运算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[9.]]\n",
      "tf.Tensor(\n",
      "[[1 9]\n",
      " [3 6]], shape=(2, 2), dtype=int32)\n",
      "tf.Tensor(\n",
      "[[ 3 11]\n",
      " [ 5  8]], shape=(2, 2), dtype=int32)\n",
      "tf.Tensor(\n",
      "[[ 3 99]\n",
      " [15 48]], shape=(2, 2), dtype=int32)\n",
      "[[ 3 99]\n",
      " [15 48]]\n"
     ]
    }
   ],
   "source": [
    "# 在eager模式下可以直接进行运算\n",
    "x = [[3.]]\n",
    "m = tf.matmul(x, x)\n",
    "print(m.numpy())\n",
    "\n",
    "a = tf.constant([[1,9],[3,6]])\n",
    "print(a)\n",
    "\n",
    "b = tf.add(a, 2)\n",
    "print(b)\n",
    "print(a*b)\n",
    "\n",
    "import numpy as np\n",
    "s = np.multiply(a,b)\n",
    "print(s)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.动态控制流"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "2\n",
      "Fizz\n",
      "4\n",
      "Buzz\n",
      "Fizz\n",
      "7\n",
      "8\n",
      "Fizz\n",
      "Buzz\n",
      "11\n",
      "Fizz\n",
      "13\n",
      "14\n",
      "FizzBuzz\n",
      "16\n"
     ]
    }
   ],
   "source": [
    "def fizzbuzz(max_num):\n",
    "    counter = tf.constant(0)\n",
    "    max_num = tf.convert_to_tensor(max_num)\n",
    "    for num in range(1, max_num.numpy()+1):\n",
    "        num = tf.constant(num)\n",
    "        if int(num % 3) == 0 and int(num % 5) == 0:\n",
    "            print('FizzBuzz')\n",
    "        elif int(num % 3) == 0:\n",
    "            print('Fizz')\n",
    "        elif int(num % 5) == 0:\n",
    "            print('Buzz')\n",
    "        else:\n",
    "            print(num.numpy())\n",
    "        counter += 1\n",
    "fizzbuzz(16)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.构建模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 如果必须强制执行该层，则在构造函数中设置self.dynamic = True：\n",
    "class MySimpleLayer(tf.keras.layers.Layer):\n",
    "    def __init__(self, output_units):\n",
    "        super(MySimpleLayer, self).__init__()\n",
    "        self.output_units = output_units\n",
    "        self.dynamic = True\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        self.kernel = self.add_variable(\n",
    "        \"kernel\", [input_shape[-1], self.output_units])\n",
    "\n",
    "    def call(self, input):\n",
    "        return tf.matmul(input, self.kernel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构造一个模型\n",
    "class MNISTModel(tf.keras.Model):\n",
    "    def __init__(self):\n",
    "        super(MNISTModel, self).__init__()\n",
    "        self.dense1 = tf.keras.layers.Dense(units=10)\n",
    "        self.dense2 = tf.keras.layers.Dense(units=10)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        \"\"\"Run the model.\"\"\"\n",
    "        result = self.dense1(inputs)\n",
    "        result = self.dense2(result)\n",
    "        result = self.dense2(result)  # reuse variables from dense2 layer\n",
    "        return result\n",
    "\n",
    "model = MNISTModel()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.使用eager模式训练\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([[2.]], shape=(1, 1), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# 计算梯度\n",
    "w = tf.Variable([[1.0]])\n",
    "with tf.GradientTape() as tape:\n",
    "    loss = w*w\n",
    "grad = tape.gradient(loss, w)\n",
    "print(grad)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logits:  [[-0.05677134  0.00057287  0.00242344  0.04665339 -0.07706797  0.00323912\n",
      "   0.01508885 -0.07409166 -0.01701452  0.05076526]]\n",
      "........................................"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Loss [entropy]')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 训练一个模型\n",
    "(mnist_images, mnist_labels), _ = tf.keras.datasets.mnist.load_data()\n",
    "\n",
    "dataset = tf.data.Dataset.from_tensor_slices(\n",
    "  (tf.cast(mnist_images[...,tf.newaxis]/255, tf.float32),\n",
    "   tf.cast(mnist_labels,tf.int64)))\n",
    "dataset = dataset.shuffle(1000).batch(32)\n",
    "mnist_model = tf.keras.Sequential([\n",
    "  tf.keras.layers.Conv2D(16,[3,3], activation='relu', \n",
    "                         input_shape=(None, None, 1)),\n",
    "  tf.keras.layers.Conv2D(16,[3,3], activation='relu'),\n",
    "  tf.keras.layers.GlobalAveragePooling2D(),\n",
    "  tf.keras.layers.Dense(10)\n",
    "])\n",
    "for images,labels in dataset.take(1):\n",
    "    print(\"Logits: \", mnist_model(images[0:1]).numpy())\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam()\n",
    "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
    "\n",
    "loss_history = []\n",
    "for (batch, (images, labels)) in enumerate(dataset.take(400)):\n",
    "    if batch % 10 == 0:\n",
    "        print('.', end='')\n",
    "    with tf.GradientTape() as tape:\n",
    "        logits = mnist_model(images, training=True)\n",
    "        loss_value = loss_object(labels, logits)\n",
    "\n",
    "    loss_history.append(loss_value.numpy().mean())\n",
    "    grads = tape.gradient(loss_value, mnist_model.trainable_variables)\n",
    "    optimizer.apply_gradients(zip(grads, mnist_model.trainable_variables))\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_history)\n",
    "plt.xlabel('Batch #')\n",
    "plt.ylabel('Loss [entropy]')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.变量求导优化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial loss: 68.801\n",
      "Loss at step 000: 66.129\n",
      "Loss at step 020: 30.134\n",
      "Loss at step 040: 14.026\n",
      "Loss at step 060: 6.816\n",
      "Loss at step 080: 3.589\n",
      "Loss at step 100: 2.144\n",
      "Loss at step 120: 1.497\n",
      "Loss at step 140: 1.207\n",
      "Loss at step 160: 1.078\n",
      "Loss at step 180: 1.020\n",
      "Loss at step 200: 0.994\n",
      "Loss at step 220: 0.982\n",
      "Loss at step 240: 0.977\n",
      "Loss at step 260: 0.974\n",
      "Loss at step 280: 0.973\n",
      "Final loss: 0.973\n",
      "W = 3.0180840492248535, B = 2.0045783519744873\n"
     ]
    }
   ],
   "source": [
    "class MyModel(tf.keras.Model):\n",
    "    def __init__(self):\n",
    "        super(MyModel, self).__init__()\n",
    "        self.W = tf.Variable(5., name='weight')\n",
    "        self.B = tf.Variable(10., name='bias')\n",
    "    def call(self, inputs):\n",
    "        return inputs * self.W + self.B\n",
    "\n",
    "# A toy dataset of points around 3 * x + 2\n",
    "NUM_EXAMPLES = 2000\n",
    "training_inputs = tf.random.normal([NUM_EXAMPLES])\n",
    "noise = tf.random.normal([NUM_EXAMPLES])\n",
    "training_outputs = training_inputs * 3 + 2 + noise\n",
    "\n",
    "# The loss function to be optimized\n",
    "def loss(model, inputs, targets):\n",
    "    error = model(inputs) - targets\n",
    "    return tf.reduce_mean(tf.square(error))\n",
    "\n",
    "def grad(model, inputs, targets):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss_value = loss(model, inputs, targets)\n",
    "    return tape.gradient(loss_value, [model.W, model.B])\n",
    "\n",
    "# Define:\n",
    "# 1. A model.\n",
    "# 2. Derivatives of a loss function with respect to model parameters.\n",
    "# 3. A strategy for updating the variables based on the derivatives.\n",
    "model = MyModel()\n",
    "optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)\n",
    "\n",
    "print(\"Initial loss: {:.3f}\".format(loss(model, training_inputs, training_outputs)))\n",
    "\n",
    "# Training loop\n",
    "for i in range(300):\n",
    "    grads = grad(model, training_inputs, training_outputs)\n",
    "    optimizer.apply_gradients(zip(grads, [model.W, model.B]))\n",
    "    if i % 20 == 0:\n",
    "        print(\"Loss at step {:03d}: {:.3f}\".format(i, loss(model, training_inputs, training_outputs)))\n",
    "\n",
    "print(\"Final loss: {:.3f}\".format(loss(model, training_inputs, training_outputs)))\n",
    "print(\"W = {}, B = {}\".format(model.W.numpy(), model.B.numpy()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.eager模式下的对象"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 变量即对象\n",
    "if tf.test.is_gpu_available():\n",
    "    with tf.device(\"gpu:0\"):\n",
    "        v = tf.Variable(tf.random.normal([1000, 1000]))\n",
    "        v = None  # v no longer takes up GPU memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.Variable 'Variable:0' shape=() dtype=float32, numpy=1.0>\n"
     ]
    }
   ],
   "source": [
    "# 对象保存\n",
    "x = tf.Variable(6.0)\n",
    "checkpoint = tf.train.Checkpoint(x=x)\n",
    "\n",
    "x.assign(1.0)\n",
    "checkpoint.save('./ckpt/')\n",
    "\n",
    "x.assign(8.0)\n",
    "checkpoint.restore(tf.train.latest_checkpoint('./ckpt/'))\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x7f97116a5f98>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 模型保持\n",
    "import os\n",
    "model = tf.keras.Sequential([\n",
    "  tf.keras.layers.Conv2D(16,[3,3], activation='relu'),\n",
    "  tf.keras.layers.GlobalAveragePooling2D(),\n",
    "  tf.keras.layers.Dense(10)\n",
    "])\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)\n",
    "checkpoint_dir = './ck_model_dir'\n",
    "if not os.path.exists(checkpoint_dir):\n",
    "    os.makedirs(checkpoint_dir)\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n",
    "root = tf.train.Checkpoint(optimizer=optimizer,\n",
    "                           model=model)\n",
    "\n",
    "root.save(checkpoint_prefix)\n",
    "root.restore(tf.train.latest_checkpoint(checkpoint_dir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(2.5, shape=(), dtype=float32)\n",
      "tf.Tensor(5.5, shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# 面向对象的指标\n",
    "m = tf.keras.metrics.Mean('loss')\n",
    "m(0)\n",
    "m(5)\n",
    "print(m.result())  # => 2.5\n",
    "m([8, 9])\n",
    "print(m.result())  # => 5.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7.高级自动分化主题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 动态模型\n",
    "\n",
    "def line_search_step(fn, init_x, rate=1.0):\n",
    "    with tf.GradientTape() as tape:\n",
    "        # Variables are automatically recorded, but manually watch a tensor\n",
    "        tape.watch(init_x)\n",
    "        value = fn(init_x)\n",
    "    grad = tape.gradient(value, init_x)\n",
    "    grad_norm = tf.reduce_sum(grad * grad)\n",
    "    init_value = value\n",
    "    while value > init_value - rate * grad_norm:\n",
    "        x = init_x - rate * grad\n",
    "        value = fn(x)\n",
    "        rate /= 2.0\n",
    "    return x, value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 自定义梯度\n",
    "@tf.custom_gradient\n",
    "def clip_gradient_by_norm(x, norm):\n",
    "    y = tf.identity(x)\n",
    "    def grad_fn(dresult):\n",
    "        return [tf.clip_by_norm(dresult, norm), None]\n",
    "    return y, grad_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5\n",
      "nan\n"
     ]
    }
   ],
   "source": [
    "# 自定义梯度可以提供数值稳定的梯度\n",
    "def log1pexp(x):\n",
    "    return tf.math.log(1 + tf.exp(x))\n",
    "\n",
    "def grad_log1pexp(x):\n",
    "    with tf.GradientTape() as tape:\n",
    "        tape.watch(x)\n",
    "        value = log1pexp(x)\n",
    "    return tape.gradient(value, x)\n",
    "print(grad_log1pexp(tf.constant(0.)).numpy())\n",
    "# However, x = 100 fails because of numerical instability.\n",
    "print(grad_log1pexp(tf.constant(100.)).numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这里，log1pexp函数可以使用自定义梯度求导进行分析简化。 下面的实现重用了在前向传递期间计算的tf.exp（x）的值 - 通过消除冗余计算使其更有效："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "@tf.custom_gradient\n",
    "def log1pexp(x):\n",
    "  e = tf.exp(x)\n",
    "  def grad(dy):\n",
    "    return dy * (1 - 1 / (1 + e))\n",
    "  return tf.math.log(1 + e), grad\n",
    "\n",
    "def grad_log1pexp(x):\n",
    "  with tf.GradientTape() as tape:\n",
    "    tape.watch(x)\n",
    "    value = log1pexp(x)\n",
    "  return tape.gradient(value, x)\n",
    "print(grad_log1pexp(tf.constant(0.)).numpy())\n",
    "print(grad_log1pexp(tf.constant(100.)).numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time to multiply a (1000, 1000) matrix by itself 200 times:\n",
      "CPU: 1.5692901611328125 secs\n",
      "GPU: not found\n"
     ]
    }
   ],
   "source": [
    "## 8.使用gpu提升性能\n",
    "import time\n",
    "\n",
    "def measure(x, steps):\n",
    "  # TensorFlow initializes a GPU the first time it's used, exclude from timing.\n",
    "  tf.matmul(x, x)\n",
    "  start = time.time()\n",
    "  for i in range(steps):\n",
    "    x = tf.matmul(x, x)\n",
    "  # tf.matmul can return before completing the matrix multiplication\n",
    "  # (e.g., can return after enqueing the operation on a CUDA stream).\n",
    "  # The x.numpy() call below will ensure that all enqueued operations\n",
    "  # have completed (and will also copy the result to host memory,\n",
    "  # so we're including a little more than just the matmul operation\n",
    "  # time).\n",
    "  _ = x.numpy()\n",
    "  end = time.time()\n",
    "  return end - start\n",
    "\n",
    "shape = (1000, 1000)\n",
    "steps = 200\n",
    "print(\"Time to multiply a {} matrix by itself {} times:\".format(shape, steps))\n",
    "\n",
    "# Run on CPU:\n",
    "with tf.device(\"/cpu:0\"):\n",
    "  print(\"CPU: {} secs\".format(measure(tf.random.normal(shape), steps)))\n",
    "\n",
    "# Run on GPU, if available:\n",
    "if tf.test.is_gpu_available():\n",
    "  with tf.device(\"/gpu:0\"):\n",
    "    print(\"GPU: {} secs\".format(measure(tf.random.normal(shape), steps)))\n",
    "else:\n",
    "  print(\"GPU: not found\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "if tf.test.is_gpu_available():\n",
    "  x = tf.random.normal([10, 10])\n",
    "\n",
    "  x_gpu0 = x.gpu()\n",
    "  x_cpu = x.cpu()\n",
    "\n",
    "  _ = tf.matmul(x_cpu, x_cpu)    # Runs on CPU\n",
    "  _ = tf.matmul(x_gpu0, x_gpu0)  # Runs on GPU:0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}