1_DownloadDataImpress.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1dc31f11-4111-42af-8d8f-d3d61b1e4c70",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Read CE points and export as table to GEE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b9b5946-3aae-408d-ad48-ebce29b40c21",
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84154a59-b6d9-422e-a9d1-00f56948a0d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import geopandas as gpd\n",
    "from config import config_imp as conf\n",
    "import ee\n",
    "\n",
    "ee.Initialize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "201647e0-bdda-451e-9544-177c88000851",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(conf.data_file)\n",
    "data.columns.to_list();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed480156-aeaa-4e16-ae42-6b33a936b644",
   "metadata": {},
   "outputs": [],
   "source": [
    "geodata = gpd.GeoDataFrame(data, geometry=gpd.points_from_xy(data[\"lon\"], data[\"lat\"]))\n",
    "geodata;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad4ea653-c2fb-4c32-934f-9c0c6ff34554",
   "metadata": {},
   "outputs": [],
   "source": [
    "geodata.__geo_interface__.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20080027-1d90-448b-b4f7-b3b978d36037",
   "metadata": {},
   "outputs": [],
   "source": [
    "fc = ee.FeatureCollection(geodata.__geo_interface__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9e3a9d7-c46a-451c-9590-255517eb7ed8",
   "metadata": {},
   "outputs": [],
   "source": [
    "task = ee.batch.Export.table.toAsset(\n",
    "    fc, \"kenya_impress\", \"projects/planet-afk/assets/kenya_impress\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fae894a3-79ac-4958-b2a8-ecfc75997e03",
   "metadata": {},
   "outputs": [],
   "source": [
    "task.start()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4010d476-f453-410a-9404-0db18cebebd2",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Download patches from planet using GEE\n",
    "\n",
    "Based on https://gist.github.com/gorelick-google/4c015b79119ef85313b8bef6d654e2d9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e38ba11-829a-4189-a9b0-665862c3bfc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import ee\n",
    "import logging\n",
    "import multiprocessing\n",
    "import requests\n",
    "import shutil\n",
    "import backoff\n",
    "from retry import retry\n",
    "from pathlib import Path\n",
    "import concurrent.futures\n",
    "\n",
    "\n",
    "ee.Initialize(opt_url=\"https://earthengine-highvolume.googleapis.com\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab6985e5-f7cf-4d05-ba3d-db89606f445e",
   "metadata": {},
   "outputs": [],
   "source": [
    "@retry(tries=10, delay=1, backoff=2)\n",
    "def getResult(_, feature):\n",
    "    \"\"\"Handle the HTTP requests to download an image.\"\"\"\n",
    "\n",
    "    # Generate the desired image from the given point.\n",
    "    region = ee.Feature(feature).geometry()\n",
    "    name = feature[\"properties\"][\"id\"]\n",
    "\n",
    "    image = (\n",
    "        ee.ImageCollection(\"projects/planet-nicfi/assets/basemaps/africa\")\n",
    "        .filter(ee.Filter.date(\"2022-06-01\", \"2022-08-31\"))\n",
    "        .mean()\n",
    "    ).clip(region)\n",
    "\n",
    "    # Fetch the URL from which to download the image.\n",
    "    url = image.getDownloadURL(\n",
    "        {\"scale\": 4.77, \"region\": image.geometry(), \"format\": \"GEO_TIFF\"}\n",
    "    )\n",
    "\n",
    "    # Handle downloading the actual pixels.\n",
    "    r = requests.get(url, stream=True)\n",
    "    if r.status_code != 200:\n",
    "        r.raise_for_status()\n",
    "\n",
    "    file_path = (Path.home() / \"1_modules/4_MultiLabel/train-impress/fn\").with_name(\n",
    "        f\"{name}.tif\"\n",
    "    )\n",
    "\n",
    "    with open(file_path, \"wb\") as out_file:\n",
    "        shutil.copyfileobj(r.raw, out_file)\n",
    "\n",
    "    print(\"Done: \", name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea7c0809-994b-46af-9621-c46014085c59",
   "metadata": {},
   "source": [
    "### Download points from GEE\n",
    "\n",
    "- We start from a point list in CSV (notebook 0).\n",
    "- Then, as I want to keep the processed points in GEE (to display purposes) I'm uploading them in the first step.\n",
    "- Then, I do the same process backwards to create the buffer and the bounds directly in GEE (why? because I don't want to deal with reprojections or use distances in degrees)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d451d8b4-4d99-496a-8f1f-ff3e181c204c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_patch(offset):\n",
    "    print(f\"working with {offset}\")\n",
    "    return (\n",
    "        ee.FeatureCollection(fc.toList(batch_size, offset))\n",
    "        .map(lambda center: center.buffer(70).bounds())\n",
    "        .getInfo()\n",
    "    )\n",
    "\n",
    "\n",
    "workers = 4\n",
    "with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:\n",
    "\n",
    "    fc = ee.FeatureCollection(\"projects/planet-afk/assets/kenya_impress\")\n",
    "    n_points = fc.size().getInfo()\n",
    "    batch_size = min(int(n_points / workers), 5000)\n",
    "\n",
    "    offsets = [0] + [x * batch_size for x in range(1, (n_points // batch_size) + 1)]\n",
    "\n",
    "    futures = [executor.submit(get_patch, offset) for offset in offsets]\n",
    "    batch_features = []\n",
    "    for future in concurrent.futures.as_completed(futures):\n",
    "        batch_features.append(future.result())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2bd8ef03-9398-4f64-8577-4f17b864188a",
   "metadata": {},
   "outputs": [],
   "source": [
    "features = [features for batch in batch_features for features in batch[\"features\"]]\n",
    "len(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23e0b641-f65a-4fbb-9e2b-7481fb687344",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check we're getting the same amount of elements\n",
    "n_points == len(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c79a44e-e38b-431d-b937-ca35bee2fd86",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "if __name__ == \"__main__\":\n",
    "\n",
    "    logging.basicConfig()\n",
    "    pool = multiprocessing.Pool(25)\n",
    "    pool.starmap(getResult, enumerate(features))\n",
    "    pool.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbfb315c-82db-4e26-92fd-0268e8c4144f",
   "metadata": {},
   "outputs": [],
   "source": [
    "(features[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe429fc5-cad5-4a32-bb96-72f53a52a93b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plots = ee.FeatureCollection(\"projects/planet-afk/assets/kenya_14100_to_8850\")\n",
    "\n",
    "# input_image = (ee.ImageCollection(\"projects/planet-nicfi/assets/basemaps/africa\")\n",
    "#          .filter(ee.Filter.date(\"2018-01-01\", \"2018-12-31\"))\n",
    "#          .mean()\n",
    "# )\n",
    "\n",
    "# input_image = ee.ImageCollection('projects/wri-datalab/TML').mosaic()\n",
    "\n",
    "# plots = table.map(lambda center: )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1c198f8-d81b-4c7d-8794-fc5ce715adf9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is an awful way to extract the features, but it works\n",
    "# I didn't want to create the geometries in pandas, I wanted to use GEE.\n",
    "f5000 = ee.FeatureCollection(plots.toList(5000)).getInfo()\n",
    "f5000_10000 = ee.FeatureCollection(plots.toList(5000, 5000)).getInfo()\n",
    "f5000[\"features\"] = f5000[\"features\"] + f5000_10000[\"features\"]\n",
    "features = f5000[\"features\"].copy()\n",
    "features[:5]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}