Mario.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Afreen89/GameLearning/blob/main/Mario.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U2y1bPXOJrJm"
      },
      "source": [
        "# 1. Setup Mario"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UI1EShzUJrJo",
        "outputId": "bfe1f0b2-6d15-426a-f94f-94b248b88387"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting gym_super_mario_bros==7.3.0\n",
            "  Downloading gym_super_mario_bros-7.3.0-py2.py3-none-any.whl (198 kB)\n",
            "\u001b[K     |████████████████████████████████| 198 kB 14.7 MB/s \n",
            "\u001b[?25hCollecting nes_py\n",
            "  Downloading nes_py-8.1.8.tar.gz (76 kB)\n",
            "\u001b[K     |████████████████████████████████| 76 kB 3.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: gym>=0.17.2 in /usr/local/lib/python3.7/dist-packages (from nes_py) (0.17.3)\n",
            "Requirement already satisfied: numpy>=1.18.5 in /usr/local/lib/python3.7/dist-packages (from nes_py) (1.21.6)\n",
            "Requirement already satisfied: pyglet<=1.5.11,>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from nes_py) (1.5.0)\n",
            "Requirement already satisfied: tqdm>=4.48.2 in /usr/local/lib/python3.7/dist-packages (from nes_py) (4.64.0)\n",
            "Requirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from gym>=0.17.2->nes_py) (1.3.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from gym>=0.17.2->nes_py) (1.4.1)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from pyglet<=1.5.11,>=1.4.0->nes_py) (0.16.0)\n",
            "Building wheels for collected packages: nes-py\n",
            "  Building wheel for nes-py (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for nes-py: filename=nes_py-8.1.8-cp37-cp37m-linux_x86_64.whl size=438696 sha256=3782e1bf8532146cef39b3107051859bd03e7fee0ed581dc6801fcbbc3547b57\n",
            "  Stored in directory: /root/.cache/pip/wheels/f2/05/1f/608f15ab43187096eb5f3087506419c2d9772e97000f3ba025\n",
            "Successfully built nes-py\n",
            "Installing collected packages: nes-py, gym-super-mario-bros\n",
            "Successfully installed gym-super-mario-bros-7.3.0 nes-py-8.1.8\n"
          ]
        }
      ],
      "source": [
        "!pip install gym_super_mario_bros==7.3.0 nes_py"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Sc8uX7tjJrJp"
      },
      "outputs": [],
      "source": [
        "# Import the game\n",
        "import gym_super_mario_bros\n",
        "# Import the Joypad wrapper\n",
        "from nes_py.wrappers import JoypadSpace\n",
        "# Import the SIMPLIFIED controls\n",
        "from gym_super_mario_bros.actions import SIMPLE_MOVEMENT"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "08Wc-yTwJrJr"
      },
      "outputs": [],
      "source": [
        "# Setup game\n",
        "env = gym_super_mario_bros.make('SuperMarioBros-v0')\n",
        "env = JoypadSpace(env, SIMPLE_MOVEMENT)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FhHIEsNcJrJr",
        "outputId": "af4cd3d4-fec3-4654-c294-0bb1a69b8847"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/gym_super_mario_bros/smb_env.py:148: RuntimeWarning: overflow encountered in ubyte_scalars\n",
            "  return (self.ram[0x86] - self.ram[0x071c]) % 256\n"
          ]
        }
      ],
      "source": [
        "# Create a flag - restart or not\n",
        "done = True\n",
        "# Loop through each frame in the game\n",
        "for step in range(100000): \n",
        "    # Start the game to begin with \n",
        "    if done: \n",
        "        # Start the gamee\n",
        "        env.reset()\n",
        "    # Do random actions\n",
        "    state, reward, done, info = env.step(env.action_space.sample())\n",
        "    # Show the game on the screen\n",
        "    # env.render()\n",
        "# Close the game\n",
        "env.close()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A-Q5Vk4MJrJs"
      },
      "source": [
        "# 2. Preprocess Environment"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "igziqkSqJrJt",
        "outputId": "5afc177e-79a2-452f-c1f4-407d6f34902b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in links: https://download.pytorch.org/whl/cu113/torch_stable.html\n",
            "Collecting torch==1.10.1+cu113\n",
            "  Downloading https://download.pytorch.org/whl/cu113/torch-1.10.1%2Bcu113-cp37-cp37m-linux_x86_64.whl (1821.5 MB)\n",
            "\u001b[K     |██████████████▋                 | 834.1 MB 1.5 MB/s eta 0:11:20tcmalloc: large alloc 1147494400 bytes == 0x39972000 @  0x7f2c71130615 0x592b76 0x4df71e 0x59afff 0x515655 0x549576 0x593fce 0x548ae9 0x51566f 0x549576 0x593fce 0x548ae9 0x5127f1 0x598e3b 0x511f68 0x598e3b 0x511f68 0x598e3b 0x511f68 0x4bc98a 0x532e76 0x594b72 0x515600 0x549576 0x593fce 0x548ae9 0x5127f1 0x549576 0x593fce 0x5118f8 0x593dd7\n",
            "\u001b[K     |██████████████████▌             | 1055.7 MB 1.4 MB/s eta 0:08:52tcmalloc: large alloc 1434370048 bytes == 0x7dfc8000 @  0x7f2c71130615 0x592b76 0x4df71e 0x59afff 0x515655 0x549576 0x593fce 0x548ae9 0x51566f 0x549576 0x593fce 0x548ae9 0x5127f1 0x598e3b 0x511f68 0x598e3b 0x511f68 0x598e3b 0x511f68 0x4bc98a 0x532e76 0x594b72 0x515600 0x549576 0x593fce 0x548ae9 0x5127f1 0x549576 0x593fce 0x5118f8 0x593dd7\n",
            "\u001b[K     |███████████████████████▌        | 1336.2 MB 1.3 MB/s eta 0:06:25tcmalloc: large alloc 1792966656 bytes == 0x2dfa000 @  0x7f2c71130615 0x592b76 0x4df71e 0x59afff 0x515655 0x549576 0x593fce 0x548ae9 0x51566f 0x549576 0x593fce 0x548ae9 0x5127f1 0x598e3b 0x511f68 0x598e3b 0x511f68 0x598e3b 0x511f68 0x4bc98a 0x532e76 0x594b72 0x515600 0x549576 0x593fce 0x548ae9 0x5127f1 0x549576 0x593fce 0x5118f8 0x593dd7\n",
            "\u001b[K     |█████████████████████████████▊  | 1691.1 MB 1.2 MB/s eta 0:01:52tcmalloc: large alloc 2241208320 bytes == 0x6dbe2000 @  0x7f2c71130615 0x592b76 0x4df71e 0x59afff 0x515655 0x549576 0x593fce 0x548ae9 0x51566f 0x549576 0x593fce 0x548ae9 0x5127f1 0x598e3b 0x511f68 0x598e3b 0x511f68 0x598e3b 0x511f68 0x4bc98a 0x532e76 0x594b72 0x515600 0x549576 0x593fce 0x548ae9 0x5127f1 0x549576 0x593fce 0x5118f8 0x593dd7\n",
            "\u001b[K     |████████████████████████████████| 1821.5 MB 1.3 MB/s eta 0:00:01tcmalloc: large alloc 1821458432 bytes == 0xf3544000 @  0x7f2c7112f1e7 0x4a3940 0x4a39cc 0x592b76 0x4df71e 0x59afff 0x515655 0x549576 0x593fce 0x511e2c 0x549576 0x593fce 0x511e2c 0x549576 0x593fce 0x511e2c 0x549576 0x593fce 0x511e2c 0x549576 0x593fce 0x511e2c 0x593dd7 0x511e2c 0x549576 0x593fce 0x548ae9 0x5127f1 0x549576 0x593fce 0x548ae9\n",
            "tcmalloc: large alloc 2276827136 bytes == 0x15fe58000 @  0x7f2c71130615 0x592b76 0x4df71e 0x59afff 0x515655 0x549576 0x593fce 0x511e2c 0x549576 0x593fce 0x511e2c 0x549576 0x593fce 0x511e2c 0x549576 0x593fce 0x511e2c 0x549576 0x593fce 0x511e2c 0x593dd7 0x511e2c 0x549576 0x593fce 0x548ae9 0x5127f1 0x549576 0x593fce 0x548ae9 0x5127f1 0x549576\n",
            "\u001b[K     |████████████████████████████████| 1821.5 MB 6.0 kB/s \n",
            "\u001b[?25hCollecting torchvision==0.11.2+cu113\n",
            "  Downloading https://download.pytorch.org/whl/cu113/torchvision-0.11.2%2Bcu113-cp37-cp37m-linux_x86_64.whl (24.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 24.6 MB 3.9 MB/s \n",
            "\u001b[?25hCollecting torchaudio===0.10.1+cu113\n",
            "  Downloading https://download.pytorch.org/whl/cu113/torchaudio-0.10.1%2Bcu113-cp37-cp37m-linux_x86_64.whl (2.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9 MB 42.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch==1.10.1+cu113) (4.2.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchvision==0.11.2+cu113) (1.21.6)\n",
            "Requirement already satisfied: pillow!=8.3.0,>=5.3.0 in /usr/local/lib/python3.7/dist-packages (from torchvision==0.11.2+cu113) (7.1.2)\n",
            "Installing collected packages: torch, torchvision, torchaudio\n",
            "  Attempting uninstall: torch\n",
            "    Found existing installation: torch 1.11.0+cu113\n",
            "    Uninstalling torch-1.11.0+cu113:\n",
            "      Successfully uninstalled torch-1.11.0+cu113\n",
            "  Attempting uninstall: torchvision\n",
            "    Found existing installation: torchvision 0.12.0+cu113\n",
            "    Uninstalling torchvision-0.12.0+cu113:\n",
            "      Successfully uninstalled torchvision-0.12.0+cu113\n",
            "  Attempting uninstall: torchaudio\n",
            "    Found existing installation: torchaudio 0.11.0+cu113\n",
            "    Uninstalling torchaudio-0.11.0+cu113:\n",
            "      Successfully uninstalled torchaudio-0.11.0+cu113\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "torchtext 0.12.0 requires torch==1.11.0, but you have torch 1.10.1+cu113 which is incompatible.\u001b[0m\n",
            "Successfully installed torch-1.10.1+cu113 torchaudio-0.10.1+cu113 torchvision-0.11.2+cu113\n"
          ]
        }
      ],
      "source": [
        "# Install pytorch\n",
        "!pip install torch==1.10.1+cu113 torchvision==0.11.2+cu113 torchaudio===0.10.1+cu113 -f https://download.pytorch.org/whl/cu113/torch_stable.html"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "zANVK54oJrJt",
        "outputId": "3fd2ec80-2c27-4177-9248-354f7573ec5a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting stable-baselines3[extra]\n",
            "  Downloading stable_baselines3-1.5.0-py3-none-any.whl (177 kB)\n",
            "\u001b[?25l\r\u001b[K     |█▉                              | 10 kB 19.5 MB/s eta 0:00:01\r\u001b[K     |███▊                            | 20 kB 24.7 MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 30 kB 30.2 MB/s eta 0:00:01\r\u001b[K     |███████▍                        | 40 kB 25.6 MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 51 kB 19.1 MB/s eta 0:00:01\r\u001b[K     |███████████                     | 61 kB 21.2 MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 71 kB 22.3 MB/s eta 0:00:01\r\u001b[K     |██████████████▊                 | 81 kB 23.9 MB/s eta 0:00:01\r\u001b[K     |████████████████▋               | 92 kB 25.8 MB/s eta 0:00:01\r\u001b[K     |██████████████████▍             | 102 kB 25.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████▎           | 112 kB 25.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▏         | 122 kB 25.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 133 kB 25.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▉      | 143 kB 25.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▋    | 153 kB 25.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▌  | 163 kB 25.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▍| 174 kB 25.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 177 kB 25.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch>=1.8.1 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.10.1+cu113)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.21.6)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (3.2.2)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.3.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.3.5)\n",
            "Collecting gym==0.21\n",
            "  Downloading gym-0.21.0.tar.gz (1.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.5 MB 47.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tensorboard>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (2.8.0)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (7.1.2)\n",
            "Collecting autorom[accept-rom-license]~=0.4.2\n",
            "  Downloading AutoROM-0.4.2-py3-none-any.whl (16 kB)\n",
            "Collecting ale-py~=0.7.4\n",
            "  Downloading ale_py-0.7.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.6 MB 45.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: opencv-python in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (4.1.2.30)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (5.4.8)\n",
            "Requirement already satisfied: importlib_metadata>=4.8.1 in /usr/local/lib/python3.7/dist-packages (from gym==0.21->stable-baselines3[extra]) (4.11.3)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.7/dist-packages (from ale-py~=0.7.4->stable-baselines3[extra]) (5.7.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (7.1.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2.23.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (4.64.0)\n",
            "Collecting AutoROM.accept-rom-license\n",
            "  Downloading AutoROM.accept-rom-license-0.4.2.tar.gz (9.8 kB)\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib_metadata>=4.8.1->gym==0.21->stable-baselines3[extra]) (3.8.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib_metadata>=4.8.1->gym==0.21->stable-baselines3[extra]) (4.2.0)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.35.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (3.3.6)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.8.1)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.0.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.0.1)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (0.37.1)\n",
            "Requirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (3.17.3)\n",
            "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.44.0)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (57.4.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (0.4.6)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (0.6.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from absl-py>=0.4->tensorboard>=2.2.0->stable-baselines3[extra]) (1.15.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (4.8)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (4.2.4)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->stable-baselines3[extra]) (1.3.1)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (0.4.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2021.10.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2.10)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->stable-baselines3[extra]) (3.2.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (1.4.2)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (0.11.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (3.0.8)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->stable-baselines3[extra]) (2022.1)\n",
            "Building wheels for collected packages: gym, AutoROM.accept-rom-license\n",
            "  Building wheel for gym (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gym: filename=gym-0.21.0-py3-none-any.whl size=1616825 sha256=eb5cffb63549278b9ec46e05031be69772e9b2ddaebcf3a217f6211965fe7224\n",
            "  Stored in directory: /root/.cache/pip/wheels/76/ee/9c/36bfe3e079df99acf5ae57f4e3464ff2771b34447d6d2f2148\n",
            "  Building wheel for AutoROM.accept-rom-license (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for AutoROM.accept-rom-license: filename=AutoROM.accept_rom_license-0.4.2-py3-none-any.whl size=441027 sha256=c49fda181f26422c56a41e07f5df7870894e6c24cf06a0fe1fd0e78fa4b77fbf\n",
            "  Stored in directory: /root/.cache/pip/wheels/87/67/2e/6147e7912fe37f5408b80d07527dab807c1d25f5c403a9538a\n",
            "Successfully built gym AutoROM.accept-rom-license\n",
            "Installing collected packages: gym, AutoROM.accept-rom-license, autorom, stable-baselines3, ale-py\n",
            "  Attempting uninstall: gym\n",
            "    Found existing installation: gym 0.17.3\n",
            "    Uninstalling gym-0.17.3:\n",
            "      Successfully uninstalled gym-0.17.3\n",
            "Successfully installed AutoROM.accept-rom-license-0.4.2 ale-py-0.7.5 autorom-0.4.2 gym-0.21.0 stable-baselines3-1.5.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "gym"
                ]
              }
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "# Install stable baselines for RL stuff\n",
        "!pip install stable-baselines3[extra]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "M9NPUWrnJrJt"
      },
      "outputs": [],
      "source": [
        "# Import Frame Stacker Wrapper and GrayScaling Wrapper\n",
        "from gym.wrappers import GrayScaleObservation\n",
        "# Import Vectorization Wrappers\n",
        "from stable_baselines3.common.vec_env import VecFrameStack, DummyVecEnv\n",
        "# Import Matplotlib to show the impact of frame stacking\n",
        "from matplotlib import pyplot as plt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jTJ0R4F5JrJu"
      },
      "outputs": [],
      "source": [
        "# 1. Create the base environment\n",
        "env = gym_super_mario_bros.make('SuperMarioBros-v0')\n",
        "# 2. Simplify the controls \n",
        "env = JoypadSpace(env, SIMPLE_MOVEMENT)\n",
        "# 3. Grayscale\n",
        "env = GrayScaleObservation(env, keep_dim=True)\n",
        "# 4. Wrap inside the Dummy Environment\n",
        "env = DummyVecEnv([lambda: env])\n",
        "# 5. Stack the frames\n",
        "env = VecFrameStack(env, 4, channels_order='last')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6PoI8aqYJrJv"
      },
      "outputs": [],
      "source": [
        "state = env.reset()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xC0UaT_vJrJv"
      },
      "outputs": [],
      "source": [
        "state, reward, done, info = env.step([5])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kCfE39urJrJv"
      },
      "outputs": [],
      "source": [
        "plt.figure(figsize=(20,16))\n",
        "for idx in range(state.shape[3]):\n",
        "    plt.subplot(1,4,idx+1)\n",
        "    plt.imshow(state[0][:,:,idx])\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lgEDSrijJrJw"
      },
      "source": [
        "# 3. Train the RL Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LawAzYv3JrJw"
      },
      "outputs": [],
      "source": [
        "# Import os for file path management\n",
        "import os \n",
        "# Import PPO for algos\n",
        "from stable_baselines3 import PPO1\n",
        "# Import Base Callback for saving models\n",
        "from stable_baselines3.common.callbacks import BaseCallback"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": true
        },
        "tags": [],
        "id": "75w-4kGUJrJw"
      },
      "outputs": [],
      "source": [
        "class TrainAndLoggingCallback(BaseCallback):\n",
        "\n",
        "    def __init__(self, check_freq, save_path, verbose=1):\n",
        "        super(TrainAndLoggingCallback, self).__init__(verbose)\n",
        "        self.check_freq = check_freq\n",
        "        self.save_path = save_path\n",
        "\n",
        "    def _init_callback(self):\n",
        "        if self.save_path is not None:\n",
        "            os.makedirs(self.save_path, exist_ok=True)\n",
        "\n",
        "    def _on_step(self):\n",
        "        if self.n_calls % self.check_freq == 0:\n",
        "            model_path = os.path.join(self.save_path, 'best_model_{}'.format(self.n_calls))\n",
        "            self.model.save(model_path)\n",
        "\n",
        "        return True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rKvpsoXvJrJx"
      },
      "outputs": [],
      "source": [
        "CHECKPOINT_DIR = './train/'\n",
        "LOG_DIR = './logs/'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": true
        },
        "tags": [],
        "id": "hbWgxgFpJrJx"
      },
      "outputs": [],
      "source": [
        "# Setup model saving callback\n",
        "callback = TrainAndLoggingCallback(check_freq=10000, save_path=CHECKPOINT_DIR)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eGQSXqpUJrJx"
      },
      "outputs": [],
      "source": [
        "model = PPO1('MlpPolicy', env, verbose=1, tensorboard_log=LOG_DIR, learning_rate=0.000001, \n",
        "            n_steps=512, seed = 200, gamma = 0.99, full_tensorboard_log=True)\n",
        "\n",
        "## For Model 2: \n",
        "'''\n",
        "model = PPO1('MlpPolicy', env, verbose=1, tensorboard_log=LOG_DIR, learning_rate=0.000001, \n",
        "            n_steps=512, seed = 100, gamma = 0.99, full_tensorboard_log=True)\n",
        "'''\n",
        "\n",
        "## For Model 3: \n",
        "'''\n",
        "model = PPO('CnnPolicy', env, verbose=1, tensorboard_log=LOG_DIR, learning_rate=0.000001, \n",
        "            n_steps=512, seed = 10) \n",
        "'''\n",
        "\n",
        "## For Model 4: \n",
        "'''\n",
        "model\n",
        "model = PPO('CnnPolicy', env, verbose=1, tensorboard_log=LOG_DIR, learning_rate=0.000001,\n",
        "          n_steps=512, seed =20 )\n",
        "'''\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "tags": [],
        "id": "H_sXEPfsJrJx"
      },
      "outputs": [],
      "source": [
        "# Train the AI model, this is where the AI model starts to learn\n",
        "model.learn(total_timesteps=1000000, callback=callback)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xQtb_iOjJrJy"
      },
      "outputs": [],
      "source": [
        "model.save('thisisatestmodel')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Oqlgy2d_JrJy"
      },
      "source": [
        "# 4. Test it Out"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "22oKntd7JrJy"
      },
      "outputs": [],
      "source": [
        "# Load model\n",
        "model = PPO.load('./train/best_model_1000000')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5nqinR4ZJrJy"
      },
      "outputs": [],
      "source": [
        "state = env.reset()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lN3TDaEYJrJy"
      },
      "outputs": [],
      "source": [
        "# Start the game \n",
        "state = env.reset()\n",
        "# Loop through the game\n",
        "while True: \n",
        "    \n",
        "    action, _ = model.predict(state)\n",
        "    state, reward, done, info = env.step(action)\n",
        "    env.render()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NYrLpNnxJrJz"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "mario",
      "language": "python",
      "name": "mario"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "colab": {
      "provenance": [],
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}