prepare_boundaries_for_mapbox.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/Vizzuality/soils-revealed-data/blob/master/prepare_boundaries_for_mapbox.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "t2X9xzZDnzNy"
   },
   "source": [
    "# Prepare data for the Soils Revealed project\n",
    "\n",
    "https://github.com/Vizzuality/soils-revealed-data\n",
    "\n",
    "`Edward P. Morris (vizzuality.)`\n",
    "\n",
    "## Description\n",
    "This notebook transforms vector boundaries into MapBox tiles format (MBTILES) using tippecanoe and uploads the resulting tiles to MapBox.\n",
    "\n",
    "```\n",
    "MIT License\n",
    "\n",
    "Copyright (c) 2020 Vizzuality\n",
    "\n",
    "Permission is hereby granted, free of charge, to any person obtaining a copy\n",
    "of this software and associated documentation files (the \"Software\"), to deal\n",
    "in the Software without restriction, including without limitation the rights\n",
    "to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n",
    "copies of the Software, and to permit persons to whom the Software is\n",
    "furnished to do so, subject to the following conditions:\n",
    "\n",
    "The above copyright notice and this permission notice shall be included in all\n",
    "copies or substantial portions of the Software.\n",
    "\n",
    "THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n",
    "IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n",
    "FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n",
    "AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n",
    "LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n",
    "OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n",
    "SOFTWARE.\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "tIq19PbHlM9p"
   },
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "AtG1BvWXpb2Y"
   },
   "source": [
    "## Linux dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "icWiEaZlkLs-",
    "outputId": "a28353c0-0c07-4ac6-8897-5c0de6fcc391"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading package lists...\n",
      "Building dependency tree...\n",
      "Reading state information...\n",
      "The following additional packages will be installed:\n",
      "  docutils-common python3-botocore python3-certifi python3-chardet\n",
      "  python3-colorama python3-dateutil python3-docutils python3-idna\n",
      "  python3-jmespath python3-pkg-resources python3-pyasn1 python3-requests\n",
      "  python3-roman python3-rsa python3-s3transfer python3-six python3-urllib3\n",
      "  python3-yaml sgml-base xml-core\n",
      "Suggested packages:\n",
      "  docutils-doc fonts-linuxlibertine | ttf-linux-libertine texlive-lang-french\n",
      "  texlive-latex-base texlive-latex-recommended python3-setuptools\n",
      "  python3-cryptography python3-openssl python3-socks sgml-base-doc debhelper\n",
      "Recommended packages:\n",
      "  python3-pil python3-pygments\n",
      "The following NEW packages will be installed:\n",
      "  awscli docutils-common python3-botocore python3-certifi python3-chardet\n",
      "  python3-colorama python3-dateutil python3-docutils python3-idna\n",
      "  python3-jmespath python3-pkg-resources python3-pyasn1 python3-requests\n",
      "  python3-roman python3-rsa python3-s3transfer python3-six python3-urllib3\n",
      "  python3-yaml sgml-base xml-core\n",
      "0 upgraded, 21 newly installed, 0 to remove and 25 not upgraded.\n",
      "Need to get 3,629 kB of archives.\n",
      "After this operation, 37.1 MB of additional disk space will be used.\n",
      "Get:1 http://archive.ubuntu.com/ubuntu bionic/main amd64 sgml-base all 1.29 [12.3 kB]\n",
      "Get:2 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-yaml amd64 3.12-1build2 [109 kB]\n",
      "Get:3 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-six all 1.11.0-2 [11.4 kB]\n",
      "Get:4 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-dateutil all 2.6.1-1 [52.3 kB]\n",
      "Get:5 http://archive.ubuntu.com/ubuntu bionic/main amd64 xml-core all 0.18 [21.3 kB]\n",
      "Get:6 http://archive.ubuntu.com/ubuntu bionic/main amd64 docutils-common all 0.14+dfsg-3 [156 kB]\n",
      "Get:7 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-roman all 2.0.0-3 [8,624 B]\n",
      "Get:8 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-docutils all 0.14+dfsg-3 [363 kB]\n",
      "Get:9 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-jmespath all 0.9.3-1ubuntu1 [18.7 kB]\n",
      "Get:10 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-certifi all 2018.1.18-2 [144 kB]\n",
      "Get:11 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-pkg-resources all 39.0.1-2 [98.8 kB]\n",
      "Get:12 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-chardet all 3.0.4-1 [80.3 kB]\n",
      "Get:13 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-idna all 2.6-1 [32.5 kB]\n",
      "Get:14 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 python3-urllib3 all 1.22-1ubuntu0.18.04.1 [86.0 kB]\n",
      "Get:15 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 python3-requests all 2.18.4-2ubuntu0.1 [58.3 kB]\n",
      "Get:16 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-botocore all 1.8.48+repack-1 [1,764 kB]\n",
      "Get:17 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-colorama all 0.3.7-1 [14.9 kB]\n",
      "Get:18 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-pyasn1 all 0.4.2-3 [46.8 kB]\n",
      "Get:19 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-rsa all 3.4.2-1 [29.0 kB]\n",
      "Get:20 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-s3transfer all 0.1.13-1 [39.8 kB]\n",
      "Get:21 http://archive.ubuntu.com/ubuntu bionic/universe amd64 awscli all 1.14.44-1ubuntu1 [482 kB]\n",
      "Fetched 3,629 kB in 1s (3,281 kB/s)\n",
      "Selecting previously unselected package sgml-base.\n",
      "(Reading database ... 144568 files and directories currently installed.)\n",
      "Preparing to unpack .../00-sgml-base_1.29_all.deb ...\n",
      "Unpacking sgml-base (1.29) ...\n",
      "Selecting previously unselected package python3-yaml.\n",
      "Preparing to unpack .../01-python3-yaml_3.12-1build2_amd64.deb ...\n",
      "Unpacking python3-yaml (3.12-1build2) ...\n",
      "Selecting previously unselected package python3-six.\n",
      "Preparing to unpack .../02-python3-six_1.11.0-2_all.deb ...\n",
      "Unpacking python3-six (1.11.0-2) ...\n",
      "Selecting previously unselected package python3-dateutil.\n",
      "Preparing to unpack .../03-python3-dateutil_2.6.1-1_all.deb ...\n",
      "Unpacking python3-dateutil (2.6.1-1) ...\n",
      "Selecting previously unselected package xml-core.\n",
      "Preparing to unpack .../04-xml-core_0.18_all.deb ...\n",
      "Unpacking xml-core (0.18) ...\n",
      "Selecting previously unselected package docutils-common.\n",
      "Preparing to unpack .../05-docutils-common_0.14+dfsg-3_all.deb ...\n",
      "Unpacking docutils-common (0.14+dfsg-3) ...\n",
      "Selecting previously unselected package python3-roman.\n",
      "Preparing to unpack .../06-python3-roman_2.0.0-3_all.deb ...\n",
      "Unpacking python3-roman (2.0.0-3) ...\n",
      "Selecting previously unselected package python3-docutils.\n",
      "Preparing to unpack .../07-python3-docutils_0.14+dfsg-3_all.deb ...\n",
      "Unpacking python3-docutils (0.14+dfsg-3) ...\n",
      "Selecting previously unselected package python3-jmespath.\n",
      "Preparing to unpack .../08-python3-jmespath_0.9.3-1ubuntu1_all.deb ...\n",
      "Unpacking python3-jmespath (0.9.3-1ubuntu1) ...\n",
      "Selecting previously unselected package python3-certifi.\n",
      "Preparing to unpack .../09-python3-certifi_2018.1.18-2_all.deb ...\n",
      "Unpacking python3-certifi (2018.1.18-2) ...\n",
      "Selecting previously unselected package python3-pkg-resources.\n",
      "Preparing to unpack .../10-python3-pkg-resources_39.0.1-2_all.deb ...\n",
      "Unpacking python3-pkg-resources (39.0.1-2) ...\n",
      "Selecting previously unselected package python3-chardet.\n",
      "Preparing to unpack .../11-python3-chardet_3.0.4-1_all.deb ...\n",
      "Unpacking python3-chardet (3.0.4-1) ...\n",
      "Selecting previously unselected package python3-idna.\n",
      "Preparing to unpack .../12-python3-idna_2.6-1_all.deb ...\n",
      "Unpacking python3-idna (2.6-1) ...\n",
      "Selecting previously unselected package python3-urllib3.\n",
      "Preparing to unpack .../13-python3-urllib3_1.22-1ubuntu0.18.04.1_all.deb ...\n",
      "Unpacking python3-urllib3 (1.22-1ubuntu0.18.04.1) ...\n",
      "Selecting previously unselected package python3-requests.\n",
      "Preparing to unpack .../14-python3-requests_2.18.4-2ubuntu0.1_all.deb ...\n",
      "Unpacking python3-requests (2.18.4-2ubuntu0.1) ...\n",
      "Selecting previously unselected package python3-botocore.\n",
      "Preparing to unpack .../15-python3-botocore_1.8.48+repack-1_all.deb ...\n",
      "Unpacking python3-botocore (1.8.48+repack-1) ...\n",
      "Selecting previously unselected package python3-colorama.\n",
      "Preparing to unpack .../16-python3-colorama_0.3.7-1_all.deb ...\n",
      "Unpacking python3-colorama (0.3.7-1) ...\n",
      "Selecting previously unselected package python3-pyasn1.\n",
      "Preparing to unpack .../17-python3-pyasn1_0.4.2-3_all.deb ...\n",
      "Unpacking python3-pyasn1 (0.4.2-3) ...\n",
      "Selecting previously unselected package python3-rsa.\n",
      "Preparing to unpack .../18-python3-rsa_3.4.2-1_all.deb ...\n",
      "Unpacking python3-rsa (3.4.2-1) ...\n",
      "Selecting previously unselected package python3-s3transfer.\n",
      "Preparing to unpack .../19-python3-s3transfer_0.1.13-1_all.deb ...\n",
      "Unpacking python3-s3transfer (0.1.13-1) ...\n",
      "Selecting previously unselected package awscli.\n",
      "Preparing to unpack .../20-awscli_1.14.44-1ubuntu1_all.deb ...\n",
      "Unpacking awscli (1.14.44-1ubuntu1) ...\n",
      "Setting up python3-yaml (3.12-1build2) ...\n",
      "Setting up python3-roman (2.0.0-3) ...\n",
      "Setting up python3-idna (2.6-1) ...\n",
      "Setting up python3-six (1.11.0-2) ...\n",
      "Setting up python3-colorama (0.3.7-1) ...\n",
      "Setting up python3-certifi (2018.1.18-2) ...\n",
      "Setting up sgml-base (1.29) ...\n",
      "Setting up python3-pkg-resources (39.0.1-2) ...\n",
      "Setting up python3-pyasn1 (0.4.2-3) ...\n",
      "Setting up python3-chardet (3.0.4-1) ...\n",
      "Setting up python3-urllib3 (1.22-1ubuntu0.18.04.1) ...\n",
      "Setting up python3-jmespath (0.9.3-1ubuntu1) ...\n",
      "Setting up xml-core (0.18) ...\n",
      "Setting up python3-dateutil (2.6.1-1) ...\n",
      "Setting up python3-rsa (3.4.2-1) ...\n",
      "Setting up python3-requests (2.18.4-2ubuntu0.1) ...\n",
      "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
      "Processing triggers for shared-mime-info (1.9-2) ...\n",
      "Processing triggers for sgml-base (1.29) ...\n",
      "Setting up docutils-common (0.14+dfsg-3) ...\n",
      "Processing triggers for sgml-base (1.29) ...\n",
      "Setting up python3-docutils (0.14+dfsg-3) ...\n",
      "update-alternatives: using /usr/share/docutils/scripts/python3/rst-buildhtml to provide /usr/bin/rst-buildhtml (rst-buildhtml) in auto mode\n",
      "update-alternatives: using /usr/share/docutils/scripts/python3/rst2html to provide /usr/bin/rst2html (rst2html) in auto mode\n",
      "update-alternatives: using /usr/share/docutils/scripts/python3/rst2html4 to provide /usr/bin/rst2html4 (rst2html4) in auto mode\n",
      "update-alternatives: using /usr/share/docutils/scripts/python3/rst2html5 to provide /usr/bin/rst2html5 (rst2html5) in auto mode\n",
      "update-alternatives: using /usr/share/docutils/scripts/python3/rst2latex to provide /usr/bin/rst2latex (rst2latex) in auto mode\n",
      "update-alternatives: using /usr/share/docutils/scripts/python3/rst2man to provide /usr/bin/rst2man (rst2man) in auto mode\n",
      "update-alternatives: using /usr/share/docutils/scripts/python3/rst2odt to provide /usr/bin/rst2odt (rst2odt) in auto mode\n",
      "update-alternatives: using /usr/share/docutils/scripts/python3/rst2odt_prepstyles to provide /usr/bin/rst2odt_prepstyles (rst2odt_prepstyles) in auto mode\n",
      "update-alternatives: using /usr/share/docutils/scripts/python3/rst2pseudoxml to provide /usr/bin/rst2pseudoxml (rst2pseudoxml) in auto mode\n",
      "update-alternatives: using /usr/share/docutils/scripts/python3/rst2s5 to provide /usr/bin/rst2s5 (rst2s5) in auto mode\n",
      "update-alternatives: using /usr/share/docutils/scripts/python3/rst2xetex to provide /usr/bin/rst2xetex (rst2xetex) in auto mode\n",
      "update-alternatives: using /usr/share/docutils/scripts/python3/rst2xml to provide /usr/bin/rst2xml (rst2xml) in auto mode\n",
      "update-alternatives: using /usr/share/docutils/scripts/python3/rstpep2html to provide /usr/bin/rstpep2html (rstpep2html) in auto mode\n",
      "Setting up python3-botocore (1.8.48+repack-1) ...\n",
      "Setting up python3-s3transfer (0.1.13-1) ...\n",
      "Setting up awscli (1.14.44-1ubuntu1) ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING: apt does not have a stable CLI interface. Use with caution in scripts.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "# Install AWS CLI (for MapBox uploads)\n",
    "apt install --no-install-recommends -y -q awscli"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "JxymUmrwkfRd",
    "outputId": "0e8ba682-55ed-43a1-e6f6-0e409c036445"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading package lists...\n",
      "Building dependency tree...\n",
      "Reading state information...\n",
      "build-essential is already the newest version (12.4ubuntu1).\n",
      "zlib1g-dev is already the newest version (1:1.2.11.dfsg-0ubuntu2).\n",
      "zlib1g-dev set to manually installed.\n",
      "The following additional packages will be installed:\n",
      "  libsqlite3-0\n",
      "Suggested packages:\n",
      "  sqlite3-doc\n",
      "The following NEW packages will be installed:\n",
      "  libsqlite3-dev\n",
      "The following packages will be upgraded:\n",
      "  libsqlite3-0\n",
      "1 upgraded, 1 newly installed, 0 to remove and 24 not upgraded.\n",
      "Need to get 1,131 kB of archives.\n",
      "After this operation, 2,136 kB of additional disk space will be used.\n",
      "Get:1 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libsqlite3-0 amd64 3.22.0-1ubuntu0.3 [498 kB]\n",
      "Get:2 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libsqlite3-dev amd64 3.22.0-1ubuntu0.3 [632 kB]\n",
      "Fetched 1,131 kB in 1s (975 kB/s)\n",
      "(Reading database ... 147715 files and directories currently installed.)\n",
      "Preparing to unpack .../libsqlite3-0_3.22.0-1ubuntu0.3_amd64.deb ...\n",
      "Unpacking libsqlite3-0:amd64 (3.22.0-1ubuntu0.3) over (3.22.0-1ubuntu0.1) ...\n",
      "Selecting previously unselected package libsqlite3-dev:amd64.\n",
      "Preparing to unpack .../libsqlite3-dev_3.22.0-1ubuntu0.3_amd64.deb ...\n",
      "Unpacking libsqlite3-dev:amd64 (3.22.0-1ubuntu0.3) ...\n",
      "Setting up libsqlite3-0:amd64 (3.22.0-1ubuntu0.3) ...\n",
      "Setting up libsqlite3-dev:amd64 (3.22.0-1ubuntu0.3) ...\n",
      "Processing triggers for libc-bin (2.27-3ubuntu1) ...\n",
      "/sbin/ldconfig.real: /usr/local/lib/python3.6/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",
      "\n",
      "Get:1 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/ InRelease [3,626 B]\n",
      "Ign:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
      "Ign:3 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
      "Hit:4 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
      "Get:5 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease [21.3 kB]\n",
      "Hit:6 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
      "Hit:7 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
      "Get:8 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
      "Get:9 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
      "Get:10 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/ Packages [88.1 kB]\n",
      "Get:11 http://ppa.launchpad.net/marutter/c2d4u3.5/ubuntu bionic InRelease [15.4 kB]\n",
      "Get:13 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
      "Get:14 http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu bionic InRelease [15.4 kB]\n",
      "Get:16 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic/main amd64 Packages [37.4 kB]\n",
      "Get:17 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [839 kB]\n",
      "Get:18 http://ppa.launchpad.net/marutter/c2d4u3.5/ubuntu bionic/main Sources [1,811 kB]\n",
      "Get:19 http://archive.ubuntu.com/ubuntu bionic-updates/restricted amd64 Packages [59.0 kB]\n",
      "Get:20 http://security.ubuntu.com/ubuntu bionic-security/multiverse amd64 Packages [8,213 B]\n",
      "Get:21 http://security.ubuntu.com/ubuntu bionic-security/restricted amd64 Packages [44.6 kB]\n",
      "Get:22 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [889 kB]\n",
      "Get:23 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [1,183 kB]\n",
      "Get:24 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [1,372 kB]\n",
      "Get:25 http://archive.ubuntu.com/ubuntu bionic-updates/multiverse amd64 Packages [12.6 kB]\n",
      "Get:26 http://ppa.launchpad.net/marutter/c2d4u3.5/ubuntu bionic/main amd64 Packages [874 kB]\n",
      "Get:27 http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu bionic/main amd64 Packages [16.3 kB]\n",
      "Fetched 7,541 kB in 4s (2,145 kB/s)\n",
      "Reading package lists...\n",
      "Hit:1 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/ InRelease\n",
      "Ign:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
      "Ign:3 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
      "Hit:4 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
      "Hit:5 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
      "Hit:6 http://security.ubuntu.com/ubuntu bionic-security InRelease\n",
      "Hit:7 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
      "Hit:8 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
      "Hit:9 http://archive.ubuntu.com/ubuntu bionic-updates InRelease\n",
      "Hit:10 http://archive.ubuntu.com/ubuntu bionic-backports InRelease\n",
      "Hit:11 http://ppa.launchpad.net/marutter/c2d4u3.5/ubuntu bionic InRelease\n",
      "Hit:13 http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu bionic InRelease\n",
      "Reading package lists...\n",
      "Building dependency tree...\n",
      "Reading state information...\n",
      "60 packages can be upgraded. Run 'apt list --upgradable' to see them.\n",
      "Reading package lists...\n",
      "Building dependency tree...\n",
      "Reading state information...\n",
      "The following additional packages will be installed:\n",
      "  cpp-5 gcc-5 gcc-5-base libasan2 libgcc-5-dev libisl15 libmpx0\n",
      "  libstdc++-5-dev\n",
      "Suggested packages:\n",
      "  gcc-5-locales g++-5-multilib gcc-5-doc libstdc++6-5-dbg gcc-5-multilib\n",
      "  libgcc1-dbg libgomp1-dbg libitm1-dbg libatomic1-dbg libasan2-dbg\n",
      "  liblsan0-dbg libtsan0-dbg libubsan0-dbg libcilkrts5-dbg libmpx0-dbg\n",
      "  libquadmath0-dbg libstdc++-5-doc\n",
      "The following NEW packages will be installed:\n",
      "  cpp-5 g++-5 gcc-5 gcc-5-base libasan2 libgcc-5-dev libisl15 libmpx0\n",
      "  libstdc++-5-dev\n",
      "0 upgraded, 9 newly installed, 0 to remove and 60 not upgraded.\n",
      "Need to get 29.1 MB of archives.\n",
      "After this operation, 100 MB of additional disk space will be used.\n",
      "Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 gcc-5-base amd64 5.5.0-12ubuntu1 [17.1 kB]\n",
      "Get:2 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libisl15 amd64 0.18-4 [548 kB]\n",
      "Get:3 http://archive.ubuntu.com/ubuntu bionic/universe amd64 cpp-5 amd64 5.5.0-12ubuntu1 [7,785 kB]\n",
      "Get:4 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libasan2 amd64 5.5.0-12ubuntu1 [264 kB]\n",
      "Get:5 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libmpx0 amd64 5.5.0-12ubuntu1 [9,888 B]\n",
      "Get:6 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libgcc-5-dev amd64 5.5.0-12ubuntu1 [2,224 kB]\n",
      "Get:7 http://archive.ubuntu.com/ubuntu bionic/universe amd64 gcc-5 amd64 5.5.0-12ubuntu1 [8,357 kB]\n",
      "Get:8 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libstdc++-5-dev amd64 5.5.0-12ubuntu1 [1,415 kB]\n",
      "Get:9 http://archive.ubuntu.com/ubuntu bionic/universe amd64 g++-5 amd64 5.5.0-12ubuntu1 [8,450 kB]\n",
      "Fetched 29.1 MB in 2s (14.8 MB/s)\n",
      "Selecting previously unselected package gcc-5-base:amd64.\n",
      "(Reading database ... 147724 files and directories currently installed.)\n",
      "Preparing to unpack .../0-gcc-5-base_5.5.0-12ubuntu1_amd64.deb ...\n",
      "Unpacking gcc-5-base:amd64 (5.5.0-12ubuntu1) ...\n",
      "Selecting previously unselected package libisl15:amd64.\n",
      "Preparing to unpack .../1-libisl15_0.18-4_amd64.deb ...\n",
      "Unpacking libisl15:amd64 (0.18-4) ...\n",
      "Selecting previously unselected package cpp-5.\n",
      "Preparing to unpack .../2-cpp-5_5.5.0-12ubuntu1_amd64.deb ...\n",
      "Unpacking cpp-5 (5.5.0-12ubuntu1) ...\n",
      "Selecting previously unselected package libasan2:amd64.\n",
      "Preparing to unpack .../3-libasan2_5.5.0-12ubuntu1_amd64.deb ...\n",
      "Unpacking libasan2:amd64 (5.5.0-12ubuntu1) ...\n",
      "Selecting previously unselected package libmpx0:amd64.\n",
      "Preparing to unpack .../4-libmpx0_5.5.0-12ubuntu1_amd64.deb ...\n",
      "Unpacking libmpx0:amd64 (5.5.0-12ubuntu1) ...\n",
      "Selecting previously unselected package libgcc-5-dev:amd64.\n",
      "Preparing to unpack .../5-libgcc-5-dev_5.5.0-12ubuntu1_amd64.deb ...\n",
      "Unpacking libgcc-5-dev:amd64 (5.5.0-12ubuntu1) ...\n",
      "Selecting previously unselected package gcc-5.\n",
      "Preparing to unpack .../6-gcc-5_5.5.0-12ubuntu1_amd64.deb ...\n",
      "Unpacking gcc-5 (5.5.0-12ubuntu1) ...\n",
      "Selecting previously unselected package libstdc++-5-dev:amd64.\n",
      "Preparing to unpack .../7-libstdc++-5-dev_5.5.0-12ubuntu1_amd64.deb ...\n",
      "Unpacking libstdc++-5-dev:amd64 (5.5.0-12ubuntu1) ...\n",
      "Selecting previously unselected package g++-5.\n",
      "Preparing to unpack .../8-g++-5_5.5.0-12ubuntu1_amd64.deb ...\n",
      "Unpacking g++-5 (5.5.0-12ubuntu1) ...\n",
      "Setting up libisl15:amd64 (0.18-4) ...\n",
      "Setting up gcc-5-base:amd64 (5.5.0-12ubuntu1) ...\n",
      "Setting up libmpx0:amd64 (5.5.0-12ubuntu1) ...\n",
      "Setting up libasan2:amd64 (5.5.0-12ubuntu1) ...\n",
      "Setting up libgcc-5-dev:amd64 (5.5.0-12ubuntu1) ...\n",
      "Setting up cpp-5 (5.5.0-12ubuntu1) ...\n",
      "Setting up libstdc++-5-dev:amd64 (5.5.0-12ubuntu1) ...\n",
      "Setting up gcc-5 (5.5.0-12ubuntu1) ...\n",
      "Setting up g++-5 (5.5.0-12ubuntu1) ...\n",
      "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
      "Processing triggers for libc-bin (2.27-3ubuntu1) ...\n",
      "/sbin/ldconfig.real: /usr/local/lib/python3.6/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",
      "\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o geojson.o geojson.cpp\n",
      "cc -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -c -o jsonpull/jsonpull.o jsonpull/jsonpull.c\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o tile.o tile.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o pool.o pool.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o mbtiles.o mbtiles.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o geometry.o geometry.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o projection.o projection.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o memfile.o memfile.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o mvt.o mvt.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o serial.o serial.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o main.o main.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o text.o text.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o dirtiles.o dirtiles.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o plugin.o plugin.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o read_json.o read_json.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o write_json.o write_json.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o geobuf.o geobuf.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o evaluator.o evaluator.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o geocsv.o geocsv.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o csv.o csv.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o geojson-loop.o geojson-loop.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o enumerate.o enumerate.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o decode.o decode.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o tile-join.o tile-join.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o unit.o unit.cpp\n",
      "g++-5 -MMD  -I/usr/local/include -I. -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -c -o jsontool.o jsontool.cpp\n",
      "g++-5  -L/usr/local/lib -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -o tippecanoe-enumerate enumerate.o  -lsqlite3\n",
      "g++-5  -L/usr/local/lib -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -o tippecanoe-json-tool jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o  -lm -lz -lsqlite3 -lpthread\n",
      "g++-5  -L/usr/local/lib -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -o tippecanoe-decode decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o  -lm -lz -lsqlite3\n",
      "g++-5  -L/usr/local/lib -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -o tile-join tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o  -lm -lz -lsqlite3 -lpthread\n",
      "g++-5  -L/usr/local/lib -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -o tippecanoe geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o  -lm -lz -lsqlite3 -lpthread\n",
      "g++-5  -L/usr/local/lib -g -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -O3 -DNDEBUG  -std=c++11 -o unit unit.o text.o  -lm -lz -lsqlite3 -lpthread\n",
      "mkdir -p /usr/local/bin\n",
      "mkdir -p /usr/local/share/man/man1/\n",
      "cp tippecanoe /usr/local/bin/tippecanoe\n",
      "cp tippecanoe-enumerate /usr/local/bin/tippecanoe-enumerate\n",
      "cp tippecanoe-decode /usr/local/bin/tippecanoe-decode\n",
      "cp tippecanoe-json-tool /usr/local/bin/tippecanoe-json-tool\n",
      "cp tile-join /usr/local/bin/tile-join\n",
      "cp man/tippecanoe.1 /usr/local/share/man/man1//tippecanoe.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING: apt does not have a stable CLI interface. Use with caution in scripts.\n",
      "\n",
      "make: *** No targets specified and no makefile found.  Stop.\n",
      "make: *** No rule to make target 'install'.  Stop.\n",
      "\n",
      "WARNING: apt does not have a stable CLI interface. Use with caution in scripts.\n",
      "\n",
      "\n",
      "WARNING: apt does not have a stable CLI interface. Use with caution in scripts.\n",
      "\n",
      "Cloning into 'tippecanoe'...\n",
      "mbtiles.cpp: In function ‘void tilestats(const std::map<std::__cxx11::basic_string<char>, layermap_entry>&, size_t, json_writer&)’:\n",
      "mbtiles.cpp:133:7: warning: variable ‘first’ set but not used [-Wunused-but-set-variable]\n",
      "  bool first = true;\n",
      "       ^\n",
      "mbtiles.cpp: In function ‘void mbtiles_write_metadata(sqlite3*, const char*, const char*, int, int, double, double, double, double, double, double, int, const char*, const std::map<std::__cxx11::basic_string<char>, layermap_entry>&, bool, const char*, bool, const std::map<std::__cxx11::basic_string<char>, std::__cxx11::basic_string<char> >&, const string&, const string&)’:\n",
      "mbtiles.cpp:512:9: warning: variable ‘first’ set but not used [-Wunused-but-set-variable]\n",
      "    bool first = true;\n",
      "         ^\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "# Install tippecanoe (for MapBox mbtiles)\n",
    "apt install --no-install-recommends -q -y build-essential libsqlite3-dev zlib1g-dev\n",
    "make\n",
    "make install\n",
    "add-apt-repository -y ppa:ubuntu-toolchain-r/test\n",
    "apt update -q -y\n",
    "apt install --no-install-recommends -q -y g++-5\n",
    "export CXX=g++-5\n",
    "\n",
    "git clone https://github.com/mapbox/tippecanoe.git\n",
    "cd tippecanoe\n",
    "make -j\n",
    "make install"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "np5LlVLE1n2T",
    "outputId": "ed25d154-23af-4cc5-c83a-2f9ef5520100"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tippecanoe: invalid option -- 'h'\n",
      "Usage: tippecanoe [options] [file.json ...]\n",
      "  Output tileset\n",
      "         --output=output.mbtiles [--output-to-directory=...] [--force]\n",
      "         [--allow-existing]\n",
      "  Tileset description and attribution\n",
      "         [--name=...] [--attribution=...] [--description=...]\n",
      "  Input files and layer names\n",
      "         [--layer=...] [--named-layer=...]\n",
      "  Parallel processing of input\n",
      "         [--read-parallel]\n",
      "  Projection of input\n",
      "         [--projection=...]\n",
      "  Zoom levels\n",
      "         [--maximum-zoom=...] [--minimum-zoom=...]\n",
      "         [--extend-zooms-if-still-dropping] [--one-tile=...]\n",
      "  Tile resolution\n",
      "         [--full-detail=...] [--low-detail=...] [--minimum-detail=...]\n",
      "  Filtering feature attributes\n",
      "         [--exclude=...] [--include=...] [--exclude-all]\n",
      "  Modifying feature attributes\n",
      "         [--attribute-type=...] [--attribute-description=...]\n",
      "         [--accumulate-attribute=...] [--empty-csv-columns-are-null]\n",
      "         [--convert-stringified-ids-to-numbers]\n",
      "         [--use-attribute-for-id=...]\n",
      "  Filtering features by attributes\n",
      "         [--feature-filter-file=...] [--feature-filter=...]\n",
      "  Dropping a fixed fraction of features by zoom level\n",
      "         [--drop-rate=...] [--base-zoom=...] [--drop-lines]\n",
      "         [--drop-polygons] [--cluster-distance=...]\n",
      "  Dropping or merging a fraction of features to keep under tile size limits\n",
      "         [--drop-densest-as-needed] [--drop-fraction-as-needed]\n",
      "         [--drop-smallest-as-needed] [--coalesce-densest-as-needed]\n",
      "         [--coalesce-fraction-as-needed]\n",
      "         [--coalesce-smallest-as-needed] [--force-feature-limit]\n",
      "         [--cluster-densest-as-needed]\n",
      "  Dropping tightly overlapping features\n",
      "         [--gamma=...] [--increase-gamma-as-needed]\n",
      "  Line and polygon simplification\n",
      "         [--simplification=...] [--no-line-simplification]\n",
      "         [--simplify-only-low-zooms] [--no-tiny-polygon-reduction]\n",
      "         [--no-simplification-of-shared-nodes]\n",
      "  Attempts to improve shared polygon boundaries\n",
      "         [--detect-shared-borders] [--grid-low-zooms]\n",
      "  Controlling clipping to tile boundaries\n",
      "         [--buffer=...] [--no-clipping] [--no-duplication]\n",
      "  Reordering features within each tile\n",
      "         [--preserve-input-order] [--reorder] [--coalesce]\n",
      "         [--reverse] [--hilbert]\n",
      "  Adding calculated attributes\n",
      "         [--calculate-feature-density] [--generate-ids]\n",
      "  Trying to correct bad source geometry\n",
      "         [--detect-longitude-wraparound] [--use-source-polygon-winding]\n",
      "         [--reverse-source-polygon-winding] [--clip-bounding-box=...]\n",
      "  Filtering tile contents\n",
      "         [--prefilter=...] [--postfilter=...]\n",
      "  Setting or disabling tile size limits\n",
      "         [--maximum-tile-bytes=...] [--maximum-tile-features=...]\n",
      "         [--no-feature-limit] [--no-tile-size-limit]\n",
      "         [--no-tile-compression] [--no-tile-stats]\n",
      "         [--tile-stats-attributes-limit=...]\n",
      "         [--tile-stats-sample-values-limit=...] [--tile-stats-values-limit=...]\n",
      "  Temporary storage\n",
      "         [--temporary-directory=...]\n",
      "  Progress indicator\n",
      "         [--quiet] [--no-progress-indicator] [--progress-interval=...]\n",
      "         [--version]\n"
     ]
    }
   ],
   "source": [
    "!tippecanoe -h"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "LGWeJje13Tcw"
   },
   "source": [
    "## Python packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 416
    },
    "colab_type": "code",
    "id": "u1vfiMCVkqmi",
    "outputId": "a8c958ac-d446-4e06-9436-d9909b1a8503"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting mapbox\n",
      "  Downloading https://files.pythonhosted.org/packages/5b/de/8dbc8e6615e2b09159efe698116d652e2d6867c71f5ae86745ff3e495ec5/mapbox-0.18.0-py2.py3-none-any.whl\n",
      "Collecting polyline>=1.3.1\n",
      "  Downloading https://files.pythonhosted.org/packages/0c/4a/67edcfd960ff64221782531c867d862acc6a4e85b382a291bcb820dcde72/polyline-1.4.0-py2.py3-none-any.whl\n",
      "Requirement already satisfied: python-dateutil>=2.5.0 in /usr/local/lib/python3.6/dist-packages (from mapbox) (2.8.1)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from mapbox) (2.21.0)\n",
      "Requirement already satisfied: boto3>=1.4 in /usr/local/lib/python3.6/dist-packages (from mapbox) (1.12.39)\n",
      "Requirement already satisfied: uritemplate>=2.0 in /usr/local/lib/python3.6/dist-packages (from mapbox) (3.0.1)\n",
      "Requirement already satisfied: cachecontrol in /usr/local/lib/python3.6/dist-packages (from mapbox) (0.12.6)\n",
      "Collecting iso3166\n",
      "  Downloading https://files.pythonhosted.org/packages/a0/42/15d2ef2211ddb26deb810a21b084ee6f3d1bc7248e884dcabb5edc04b649/iso3166-1.0.1-py2.py3-none-any.whl\n",
      "Requirement already satisfied: six>=1.8.0 in /usr/local/lib/python3.6/dist-packages (from polyline>=1.3.1->mapbox) (1.12.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->mapbox) (2020.4.5.1)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->mapbox) (3.0.4)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->mapbox) (2.8)\n",
      "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->mapbox) (1.24.3)\n",
      "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3>=1.4->mapbox) (0.3.3)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3>=1.4->mapbox) (0.9.5)\n",
      "Requirement already satisfied: botocore<1.16.0,>=1.15.39 in /usr/local/lib/python3.6/dist-packages (from boto3>=1.4->mapbox) (1.15.39)\n",
      "Requirement already satisfied: msgpack>=0.5.2 in /usr/local/lib/python3.6/dist-packages (from cachecontrol->mapbox) (1.0.0)\n",
      "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.39->boto3>=1.4->mapbox) (0.15.2)\n",
      "Installing collected packages: polyline, iso3166, mapbox\n",
      "Successfully installed iso3166-1.0.1 mapbox-0.18.0 polyline-1.4.0\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "# Install mapbox python package\n",
    "pip install mapbox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "rZ_-XBWMp-9R",
    "outputId": "a0afaa1f-40a4-4b90-a7a9-b65429ace514"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Package                  Version        \n",
      "------------------------ ---------------\n",
      "absl-py                  0.9.0          \n",
      "alabaster                0.7.12         \n",
      "albumentations           0.1.12         \n",
      "altair                   4.1.0          \n",
      "asgiref                  3.2.7          \n",
      "astor                    0.8.1          \n",
      "astropy                  4.0.1.post1    \n",
      "astunparse               1.6.3          \n",
      "atari-py                 0.2.6          \n",
      "atomicwrites             1.3.0          \n",
      "attrs                    19.3.0         \n",
      "audioread                2.1.8          \n",
      "autograd                 1.3            \n",
      "awscli                   1.14.44        \n",
      "Babel                    2.8.0          \n",
      "backcall                 0.1.0          \n",
      "beautifulsoup4           4.6.3          \n",
      "bleach                   3.1.4          \n",
      "blis                     0.4.1          \n",
      "bokeh                    1.4.0          \n",
      "boto                     2.49.0         \n",
      "boto3                    1.12.39        \n",
      "botocore                 1.15.39        \n",
      "Bottleneck               1.3.2          \n",
      "branca                   0.4.0          \n",
      "bs4                      0.0.1          \n",
      "CacheControl             0.12.6         \n",
      "cachetools               3.1.1          \n",
      "catalogue                1.0.0          \n",
      "certifi                  2020.4.5.1     \n",
      "cffi                     1.14.0         \n",
      "chainer                  6.5.0          \n",
      "chardet                  3.0.4          \n",
      "click                    7.1.1          \n",
      "cloudpickle              1.3.0          \n",
      "cmake                    3.12.0         \n",
      "cmdstanpy                0.4.0          \n",
      "colorama                 0.3.7          \n",
      "colorlover               0.3.0          \n",
      "community                1.0.0b1        \n",
      "contextlib2              0.5.5          \n",
      "convertdate              2.2.0          \n",
      "coverage                 3.7.1          \n",
      "coveralls                0.5            \n",
      "crcmod                   1.7            \n",
      "cufflinks                0.17.3         \n",
      "cvxopt                   1.2.4          \n",
      "cvxpy                    1.0.31         \n",
      "cycler                   0.10.0         \n",
      "cymem                    2.0.3          \n",
      "Cython                   0.29.16        \n",
      "daft                     0.0.4          \n",
      "dask                     2.12.0         \n",
      "dataclasses              0.7            \n",
      "datascience              0.10.6         \n",
      "decorator                4.4.2          \n",
      "defusedxml               0.6.0          \n",
      "descartes                1.1.0          \n",
      "dill                     0.3.1.1        \n",
      "distributed              1.25.3         \n",
      "Django                   3.0.5          \n",
      "dlib                     19.18.0        \n",
      "docopt                   0.6.2          \n",
      "docutils                 0.15.2         \n",
      "dopamine-rl              1.0.5          \n",
      "earthengine-api          0.1.217        \n",
      "easydict                 1.9            \n",
      "ecos                     2.0.7.post1    \n",
      "editdistance             0.5.3          \n",
      "en-core-web-sm           2.2.5          \n",
      "entrypoints              0.3            \n",
      "ephem                    3.7.7.1        \n",
      "et-xmlfile               1.0.1          \n",
      "fa2                      0.3.5          \n",
      "fancyimpute              0.4.3          \n",
      "fastai                   1.0.60         \n",
      "fastdtw                  0.3.4          \n",
      "fastprogress             0.2.3          \n",
      "fastrlock                0.4            \n",
      "fbprophet                0.6            \n",
      "feather-format           0.4.0          \n",
      "featuretools             0.4.1          \n",
      "filelock                 3.0.12         \n",
      "firebase-admin           4.0.1          \n",
      "fix-yahoo-finance        0.0.22         \n",
      "Flask                    1.1.2          \n",
      "folium                   0.8.3          \n",
      "fsspec                   0.7.2          \n",
      "future                   0.16.0         \n",
      "gast                     0.3.3          \n",
      "GDAL                     2.2.2          \n",
      "gdown                    3.6.4          \n",
      "gensim                   3.6.0          \n",
      "geographiclib            1.50           \n",
      "geopy                    1.17.0         \n",
      "gin-config               0.3.0          \n",
      "glob2                    0.7            \n",
      "google                   2.0.3          \n",
      "google-api-core          1.16.0         \n",
      "google-api-python-client 1.7.12         \n",
      "google-auth              1.7.2          \n",
      "google-auth-httplib2     0.0.3          \n",
      "google-auth-oauthlib     0.4.1          \n",
      "google-cloud-bigquery    1.21.0         \n",
      "google-cloud-core        1.0.3          \n",
      "google-cloud-datastore   1.8.0          \n",
      "google-cloud-firestore   1.6.2          \n",
      "google-cloud-language    1.2.0          \n",
      "google-cloud-storage     1.18.1         \n",
      "google-cloud-translate   1.5.0          \n",
      "google-colab             1.0.0          \n",
      "google-pasta             0.2.0          \n",
      "google-resumable-media   0.4.1          \n",
      "googleapis-common-protos 1.51.0         \n",
      "googledrivedownloader    0.4            \n",
      "graphviz                 0.10.1         \n",
      "grpcio                   1.28.1         \n",
      "gspread                  3.0.1          \n",
      "gspread-dataframe        3.0.5          \n",
      "gym                      0.17.1         \n",
      "h5py                     2.10.0         \n",
      "HeapDict                 1.0.1          \n",
      "holidays                 0.9.12         \n",
      "html5lib                 1.0.1          \n",
      "httpimport               0.5.18         \n",
      "httplib2                 0.17.2         \n",
      "httplib2shim             0.0.3          \n",
      "humanize                 0.5.1          \n",
      "hyperopt                 0.1.2          \n",
      "ideep4py                 2.0.0.post3    \n",
      "idna                     2.8            \n",
      "image                    1.5.30         \n",
      "imageio                  2.4.1          \n",
      "imagesize                1.2.0          \n",
      "imbalanced-learn         0.4.3          \n",
      "imblearn                 0.0            \n",
      "imgaug                   0.2.9          \n",
      "importlib-metadata       1.6.0          \n",
      "imutils                  0.5.3          \n",
      "inflect                  2.1.0          \n",
      "intel-openmp             2020.0.133     \n",
      "intervaltree             2.1.0          \n",
      "ipykernel                4.10.1         \n",
      "ipython                  5.5.0          \n",
      "ipython-genutils         0.2.0          \n",
      "ipython-sql              0.3.9          \n",
      "ipywidgets               7.5.1          \n",
      "iso3166                  1.0.1          \n",
      "itsdangerous             1.1.0          \n",
      "jax                      0.1.62         \n",
      "jaxlib                   0.1.42         \n",
      "jdcal                    1.4.1          \n",
      "jedi                     0.17.0         \n",
      "jieba                    0.42.1         \n",
      "Jinja2                   2.11.2         \n",
      "jmespath                 0.9.5          \n",
      "joblib                   0.14.1         \n",
      "jpeg4py                  0.1.4          \n",
      "jsonschema               2.6.0          \n",
      "jupyter                  1.0.0          \n",
      "jupyter-client           5.3.4          \n",
      "jupyter-console          5.2.0          \n",
      "jupyter-core             4.6.3          \n",
      "kaggle                   1.5.6          \n",
      "kapre                    0.1.3.1        \n",
      "Keras                    2.3.1          \n",
      "Keras-Applications       1.0.8          \n",
      "Keras-Preprocessing      1.1.0          \n",
      "keras-vis                0.4.1          \n",
      "kiwisolver               1.2.0          \n",
      "knnimpute                0.1.0          \n",
      "librosa                  0.6.3          \n",
      "lightgbm                 2.2.3          \n",
      "llvmlite                 0.31.0         \n",
      "lmdb                     0.98           \n",
      "lucid                    0.3.8          \n",
      "LunarCalendar            0.0.9          \n",
      "lxml                     4.2.6          \n",
      "mapbox                   0.18.0         \n",
      "Markdown                 3.2.1          \n",
      "MarkupSafe               1.1.1          \n",
      "matplotlib               3.2.1          \n",
      "matplotlib-venn          0.11.5         \n",
      "missingno                0.4.2          \n",
      "mistune                  0.8.4          \n",
      "mizani                   0.6.0          \n",
      "mkl                      2019.0         \n",
      "mlxtend                  0.14.0         \n",
      "more-itertools           8.2.0          \n",
      "moviepy                  0.2.3.5        \n",
      "mpmath                   1.1.0          \n",
      "msgpack                  1.0.0          \n",
      "multiprocess             0.70.9         \n",
      "multitasking             0.0.9          \n",
      "murmurhash               1.0.2          \n",
      "music21                  5.5.0          \n",
      "natsort                  5.5.0          \n",
      "nbconvert                5.6.1          \n",
      "nbformat                 5.0.5          \n",
      "networkx                 2.4            \n",
      "nibabel                  3.0.2          \n",
      "nltk                     3.2.5          \n",
      "notebook                 5.2.2          \n",
      "np-utils                 0.5.12.1       \n",
      "numba                    0.48.0         \n",
      "numexpr                  2.7.1          \n",
      "numpy                    1.18.2         \n",
      "nvidia-ml-py3            7.352.0        \n",
      "oauth2client             4.1.3          \n",
      "oauthlib                 3.1.0          \n",
      "okgrade                  0.4.3          \n",
      "opencv-contrib-python    4.1.2.30       \n",
      "opencv-python            4.1.2.30       \n",
      "openpyxl                 2.5.9          \n",
      "opt-einsum               3.2.0          \n",
      "osqp                     0.6.1          \n",
      "packaging                20.3           \n",
      "palettable               3.3.0          \n",
      "pandas                   1.0.3          \n",
      "pandas-datareader        0.8.1          \n",
      "pandas-gbq               0.11.0         \n",
      "pandas-profiling         1.4.1          \n",
      "pandocfilters            1.4.2          \n",
      "parso                    0.7.0          \n",
      "pathlib                  1.0.1          \n",
      "patsy                    0.5.1          \n",
      "pexpect                  4.8.0          \n",
      "pickleshare              0.7.5          \n",
      "Pillow                   7.0.0          \n",
      "pip                      19.3.1         \n",
      "pip-tools                4.5.1          \n",
      "plac                     1.1.3          \n",
      "plotly                   4.4.1          \n",
      "plotnine                 0.6.0          \n",
      "pluggy                   0.7.1          \n",
      "polyline                 1.4.0          \n",
      "portpicker               1.3.1          \n",
      "prefetch-generator       1.0.1          \n",
      "preshed                  3.0.2          \n",
      "prettytable              0.7.2          \n",
      "progressbar2             3.38.0         \n",
      "prometheus-client        0.7.1          \n",
      "promise                  2.3            \n",
      "prompt-toolkit           1.0.18         \n",
      "protobuf                 3.10.0         \n",
      "psutil                   5.4.8          \n",
      "psycopg2                 2.7.6.1        \n",
      "ptvsd                    5.0.0a12       \n",
      "ptyprocess               0.6.0          \n",
      "py                       1.8.1          \n",
      "pyarrow                  0.14.1         \n",
      "pyasn1                   0.4.8          \n",
      "pyasn1-modules           0.2.8          \n",
      "pycocotools              2.0.0          \n",
      "pycparser                2.20           \n",
      "pydata-google-auth       0.3.0          \n",
      "pydot                    1.3.0          \n",
      "pydot-ng                 2.0.0          \n",
      "pydotplus                2.0.2          \n",
      "PyDrive                  1.3.1          \n",
      "pyemd                    0.5.1          \n",
      "pyglet                   1.5.0          \n",
      "Pygments                 2.1.3          \n",
      "pygobject                3.26.1         \n",
      "pymc3                    3.7            \n",
      "PyMeeus                  0.3.7          \n",
      "pymongo                  3.10.1         \n",
      "pymystem3                0.2.0          \n",
      "PyOpenGL                 3.1.5          \n",
      "pyparsing                2.4.7          \n",
      "pyrsistent               0.16.0         \n",
      "pysndfile                1.3.8          \n",
      "PySocks                  1.7.1          \n",
      "pystan                   2.19.1.1       \n",
      "pytest                   3.6.4          \n",
      "python-apt               1.6.5+ubuntu0.2\n",
      "python-chess             0.23.11        \n",
      "python-dateutil          2.8.1          \n",
      "python-louvain           0.14           \n",
      "python-slugify           4.0.0          \n",
      "python-utils             2.4.0          \n",
      "pytz                     2018.9         \n",
      "PyWavelets               1.1.1          \n",
      "PyYAML                   3.13           \n",
      "pyzmq                    19.0.0         \n",
      "qtconsole                4.7.2          \n",
      "QtPy                     1.9.0          \n",
      "regex                    2019.12.20     \n",
      "requests                 2.21.0         \n",
      "requests-oauthlib        1.3.0          \n",
      "resampy                  0.2.2          \n",
      "retrying                 1.3.3          \n",
      "roman                    2.0.0          \n",
      "rpy2                     3.2.7          \n",
      "rsa                      4.0            \n",
      "s3fs                     0.4.2          \n",
      "s3transfer               0.3.3          \n",
      "scikit-image             0.16.2         \n",
      "scikit-learn             0.22.2.post1   \n",
      "scipy                    1.4.1          \n",
      "screen-resolution-extra  0.0.0          \n",
      "scs                      2.1.2          \n",
      "seaborn                  0.10.0         \n",
      "Send2Trash               1.5.0          \n",
      "setuptools               46.1.3         \n",
      "setuptools-git           1.2            \n",
      "Shapely                  1.7.0          \n",
      "simplegeneric            0.8.1          \n",
      "six                      1.12.0         \n",
      "sklearn                  0.0            \n",
      "sklearn-pandas           1.8.0          \n",
      "smart-open               1.11.1         \n",
      "snowballstemmer          2.0.0          \n",
      "sortedcontainers         2.1.0          \n",
      "spacy                    2.2.4          \n",
      "Sphinx                   1.8.5          \n",
      "sphinxcontrib-websupport 1.2.1          \n",
      "SQLAlchemy               1.3.16         \n",
      "sqlparse                 0.3.1          \n",
      "srsly                    1.0.2          \n",
      "statsmodels              0.10.2         \n",
      "sympy                    1.1.1          \n",
      "tables                   3.4.4          \n",
      "tabulate                 0.8.7          \n",
      "tbb                      2020.0.133     \n",
      "tblib                    1.6.0          \n",
      "tensorboard              2.2.0          \n",
      "tensorboard-plugin-wit   1.6.0.post3    \n",
      "tensorboardcolab         0.0.22         \n",
      "tensorflow               2.2.0rc3       \n",
      "tensorflow-addons        0.8.3          \n",
      "tensorflow-datasets      2.1.0          \n",
      "tensorflow-estimator     2.2.0rc0       \n",
      "tensorflow-gcs-config    2.1.8          \n",
      "tensorflow-hub           0.8.0          \n",
      "tensorflow-metadata      0.21.2         \n",
      "tensorflow-privacy       0.2.2          \n",
      "tensorflow-probability   0.9.0          \n",
      "termcolor                1.1.0          \n",
      "terminado                0.8.3          \n",
      "testpath                 0.4.4          \n",
      "text-unidecode           1.3            \n",
      "textblob                 0.15.3         \n",
      "textgenrnn               1.4.1          \n",
      "Theano                   1.0.4          \n",
      "thinc                    7.4.0          \n",
      "toolz                    0.10.0         \n",
      "torch                    1.4.0          \n",
      "torchsummary             1.5.1          \n",
      "torchtext                0.3.1          \n",
      "torchvision              0.5.0          \n",
      "tornado                  4.5.3          \n",
      "tqdm                     4.38.0         \n",
      "traitlets                4.3.3          \n",
      "tweepy                   3.6.0          \n",
      "typeguard                2.7.1          \n",
      "typing                   3.6.6          \n",
      "typing-extensions        3.6.6          \n",
      "tzlocal                  1.5.1          \n",
      "umap-learn               0.4.1          \n",
      "uritemplate              3.0.1          \n",
      "urllib3                  1.24.3         \n",
      "vega-datasets            0.8.0          \n",
      "wasabi                   0.6.0          \n",
      "wcwidth                  0.1.9          \n",
      "webencodings             0.5.1          \n",
      "Werkzeug                 1.0.1          \n",
      "wheel                    0.34.2         \n",
      "widgetsnbextension       3.5.1          \n",
      "wordcloud                1.5.0          \n",
      "wrapt                    1.12.1         \n",
      "xarray                   0.15.1         \n",
      "xgboost                  0.90           \n",
      "xkit                     0.0.0          \n",
      "xlrd                     1.1.0          \n",
      "xlwt                     1.3.0          \n",
      "yellowbrick              0.9.1          \n",
      "zict                     2.0.0          \n",
      "zipp                     3.1.0          \n"
     ]
    }
   ],
   "source": [
    "!pip list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "IK4kYzml3bDD"
   },
   "source": [
    "## Authorisation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "wtgLnjQj3CEv"
   },
   "source": [
    "### Google cloud storage\n",
    "\n",
    "Either use user authorisation or a service account, save credentials to your drive or upload."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oY_ymKY_oIY8"
   },
   "outputs": [],
   "source": [
    "# For auth WITHOUT service account\n",
    "#from google.colab import auth\n",
    "#auth.authenticate_user()\n",
    "\n",
    "# https://cloud.google.com/resource-manager/docs/creating-managing-projects\n",
    "#project_id = \"soc-platform\"\n",
    "#!gcloud config set project {project_id}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 124
    },
    "colab_type": "code",
    "id": "YGXp6bBR16Cp",
    "outputId": "58f504ff-b3a9-4105-b014-feebf0ba9ff2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "# Mount drive\n",
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dCFS8FOnzScr"
   },
   "outputs": [],
   "source": [
    "# Copy GC credentials to home (place in your GDrive, and connect Drive)\n",
    "!cp \"/content/drive/My Drive/soc-platform-6a9bf204638c.json\" \"/root/.soc-platform-6a9bf204638c.json\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "03Tbqeq9zSc0",
    "outputId": "70b1e793-0f76-4bc8-c772-99f296c8d3c1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Activated service account credentials for: [edward-morris-vizzuality@soc-platform.iam.gserviceaccount.com]\n"
     ]
    }
   ],
   "source": [
    "# Auth WITH service account\n",
    "!gcloud auth activate-service-account \\\n",
    "  edward-morris-vizzuality@soc-platform.iam.gserviceaccount.com  \\\n",
    "          --key-file=/root/.soc-platform-6a9bf204638c.json --project=\"soc-platform\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "PA4tPZ4-zSc9",
    "outputId": "ec1322fe-8660-418d-c34d-6e59e8bb4e47"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gs://vizz-data-transfer/SOC_maps/\n"
     ]
    }
   ],
   "source": [
    "# Test GC auth\n",
    "!gsutil ls \"gs://vizz-data-transfer\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "751xkIHo3KJJ"
   },
   "source": [
    "### MapBox\n",
    "\n",
    "Create a JSON file and add it to your drive or upload:\n",
    "```\n",
    "{\"MB_USER:\"user-name\", \"MB_TOKEN\":\"token\"}\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JZDk-V7s2_05"
   },
   "outputs": [],
   "source": [
    "# Copy GC credentials to home (place in your GDrive, and connect Drive)\n",
    "!cp \"/content/drive/My Drive/copernicus-forests-mapbox.json\" \"/root/.copernicus-forests-mapbox.json\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "cellView": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 69
    },
    "colab_type": "code",
    "id": "o9QRHh6_k1WW",
    "outputId": "a32e410d-d874-42f0-f04d-24486d53030a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100   857  100   857    0     0   2158      0 --:--:-- --:--:-- --:--:--  2158\n"
     ]
    }
   ],
   "source": [
    "# Set up Mapbox (S3) credentials as environmental variables\n",
    "import json\n",
    "import os\n",
    "\n",
    "# Set user and token as environment variables\n",
    "c = json.loads(open(\"/root/.copernicus-forests-mapbox.json\").read())\n",
    "os.environ['MB_USER'] = c['MB_USER']\n",
    "os.environ['MB_TOKEN'] = c['MB_TOKEN']\n",
    "\n",
    "# Make call to mapbox api and save return to file\n",
    "!curl -X POST https://api.mapbox.com/uploads/v1/${MB_USER}/credentials?access_token=${MB_TOKEN} > credentials.json\n",
    "r = json.loads(open(\"credentials.json\").read())\n",
    "#print(r)\n",
    "\n",
    "# Set credentials as environ variables\n",
    "os.environ['MB_BUCKET'] = r['bucket']\n",
    "os.environ['MB_KEY'] = r['key']\n",
    "os.environ['AWS_ACCESS_KEY_ID'] = r['accessKeyId']\n",
    "os.environ['AWS_SECRET_ACCESS_KEY'] = r['secretAccessKey']\n",
    "os.environ['AWS_SESSION_TOKEN'] = r['sessionToken']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Vb3bFXTp0JFz"
   },
   "source": [
    "# Utils"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-l-NVtcE8KdV"
   },
   "source": [
    "## copy_gcs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "h1KygWllicHk"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import subprocess\n",
    "\n",
    "def copy_gcs(source_list, dest_list, opts=\"\"):\n",
    "  \"\"\"\n",
    "  Use gsutil to copy each corresponding item in source_list\n",
    "  to dest_list\n",
    "  \"\"\"\n",
    "  for s, d  in zip(source_list, dest_list):\n",
    "    cmd = f\"gsutil -m cp -r {opts} {s} {d}\"\n",
    "    print(f\"Processing: {cmd}\")\n",
    "    r = subprocess.call(cmd, shell=True)\n",
    "    if r == 0:\n",
    "        print(\"Task created\")\n",
    "    else:\n",
    "        print(\"Task failed\")\n",
    "  print(\"Finished copy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "gTOD9R2KyFJ_"
   },
   "source": [
    "## upload_to_mapbox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7f7HSk1QlJqG"
   },
   "outputs": [],
   "source": [
    "# Upload task for mapbox\n",
    "import os\n",
    "from mapbox import Uploader\n",
    "\n",
    "def upload_to_mapbox(file_path, tileset_name):\n",
    "  \"\"\"\n",
    "  Given a local file path and a MapBox tileset name\n",
    "  push to MapBox AWS S3 staging and create MapBox upload task\n",
    "  \"\"\"\n",
    "  username = os.getenv(\"MB_USER\")\n",
    "  my_token = os.getenv(\"MB_TOKEN\")\n",
    "  u = Uploader(access_token=my_token)  # handles authentication\n",
    "  tileset = f\"{username}.{tileset_name}\"  # name your tileset\n",
    "  job = u.upload(open(file_path, 'rb'), tileset)  # upload happens here\n",
    "  # job = u.create(url, tileset, name=tileset_name)  # starts the tiling job\n",
    "  status = job.status_code\n",
    "  print(status)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nbLJIdsD8ODk"
   },
   "source": [
    "## create_mbtiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nJvWJrhI8Ukz"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import subprocess\n",
    "\n",
    "def create_mbtiles(source_path, dest_path, layer_name, opts=\"-zg --drop-densest-as-needed --extend-zooms-if-still-dropping --force --read-parallel\"):\n",
    "  \"\"\"\n",
    "  Use tippecanoe to to create a MBTILE at dest_path from source_path.\n",
    "  layer_name is used for the name of the layer in the MBTILE.\n",
    "  Regex file path (/*.geojson) is supported for source_path.\n",
    "  \"\"\"\n",
    "  cmd = f\"tippecanoe -o {dest_path} -l {layer_name} {opts} {source_path}\"\n",
    "  print(f\"Processing: {cmd}\")\n",
    "  r = subprocess.call(cmd, shell=True)\n",
    "  if r == 0:\n",
    "      print(\"Task created\")\n",
    "  else:\n",
    "      print(\"Task failed\")\n",
    "  print(\"Finished processing\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fyuSh-zhrFDW"
   },
   "source": [
    "# Process data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "2O5npVkK6OTB"
   },
   "source": [
    "## Create MBTILES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 89
    },
    "colab_type": "code",
    "id": "vph6TQXrQiGZ",
    "outputId": "20578209-98aa-43c9-f959-7563663a90d9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing: tippecanoe -o '/content/drive/My Drive/copernicus-forests/SWE-bv-spp.mbtiles' -l SWE_biovar_species -zg --drop-densest-as-needed --extend-zooms-if-still-dropping --force --read-parallel '/content/drive/My Drive/copernicus-forests/SWE_zonal_biovar_ISEA-3-HEXAGON_grid.geojson'\n",
      "Task created\n",
      "Finished processing\n"
     ]
    }
   ],
   "source": [
    "layer_name = \"SWE_biovar_species\"\n",
    "source_path = \"'/content/drive/My Drive/copernicus-forests/SWE_zonal_biovar_ISEA-3-HEXAGON_grid.geojson'\"\n",
    "dest_path = \"'/content/drive/My Drive/copernicus-forests/SWE-bv-spp.mbtiles'\"\n",
    "create_mbtiles(source_path, dest_path, layer_name, opts=\"-zg --drop-densest-as-needed --extend-zooms-if-still-dropping --force --read-parallel\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RVmTctOP6a1O"
   },
   "source": [
    "## Upload to MapBox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 69
    },
    "colab_type": "code",
    "id": "VLHGuMw44Nyl",
    "outputId": "b684ad24-e2d2-48dd-d786-f724f0de2657"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['/content/drive/My Drive/copernicus-forests/SWE-bv-spp.mbtiles']\n",
      "/content/drive/My Drive/copernicus-forests/SWE-bv-spp.mbtiles\n",
      "201\n"
     ]
    }
   ],
   "source": [
    "# Add to Mapbox\n",
    "import glob \n",
    "import os\n",
    "path = '/content/drive/My Drive/copernicus-forests/'\n",
    "files = [f for f in glob.glob(path + \"**/*.mbtiles\", recursive=True)]\n",
    "print(files)\n",
    "for f in files:\n",
    "  print(f)\n",
    "  upload_to_mapbox(f, os.path.splitext(os.path.basename(f))[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HPnRrznOFM-W"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "include_colab_link": true,
   "name": "prepare_boundaries_for_mapbox.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}