notebooks/python-demonstration.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Disclaimer\n",
    "\n",
    "Distribution authorized to U.S. Government agencies and their contractors. Other requests for this document shall be referred to the MIT Lincoln Laboratory Technology Office.\n",
    "\n",
    "This material is based upon work supported by the Under Secretary of Defense for Research and Engineering under Air Force Contract No. FA8702-15-D-0001. Any opinions, findings, conclusions or recommendations expressed in this material are those of the author(s) and do not necessarily reflect the views of the Under Secretary of Defense for Research and Engineering.\n",
    "\n",
    "© 2019 Massachusetts Institute of Technology.\n",
    "\n",
    "The software/firmware is provided to you on an As-Is basis\n",
    "\n",
    "Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS Part 252.227-7013 or 7014 (Feb 2014). Notwithstanding any copyright notice, U.S. Government rights in this work are defined by DFARS 252.227-7013 or DFARS 252.227-7014 as detailed above. Use of this work other than as specifically authorized by the U.S. Government may violate any copyrights that exist in this work.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tesse.env import Env\n",
    "from tesse.msgs import *\n",
    "\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "import defusedxml.ElementTree as ET\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = Env()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Demo Transform Message"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.send(Transform(-1, 0, 0)) # left\n",
    "time.sleep(1)\n",
    "env.send(Transform(1, 0, 0)) # right\n",
    "time.sleep(1)\n",
    "env.send(Transform(0, -1, 0)) # back\n",
    "time.sleep(1)\n",
    "env.send(Transform(0, 1, 0)) # forward\n",
    "time.sleep(1)\n",
    "env.send(Transform(0, 0, 45)) # turn right\n",
    "time.sleep(1)\n",
    "env.send(Transform(0, 0, -45)) # turn left"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Demo Force and Torque Message"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.send(AddForce(10, 0)) # Forward force\n",
    "time.sleep(1)\n",
    "env.send(AddForce(-10, 0)) # Backward force\n",
    "time.sleep(1)\n",
    "env.send(AddForce(0, 1)) # Torque right\n",
    "time.sleep(1)\n",
    "env.send(AddForce(0, -1)) # Torque left"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Demo Reposition Message"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.send(Reposition(0, .5, 2, 0, 0, 0, 0))\n",
    "time.sleep(1)\n",
    "env.send(Reposition(2, 2, 0, 0, 0, 0, 0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Demo getting images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot(response, title=None):\n",
    "    if len(response.images) == 0:\n",
    "        return\n",
    "    figsize = (len(response.images)*5, 5)\n",
    "    fig, axs = plt.subplots(1, len(response.images), figsize=figsize)\n",
    "    if len(response.images) > 1:\n",
    "        for ax, img in zip(axs, response.images):\n",
    "            ax.axis('off');\n",
    "            ax.imshow(img, cmap='gray');\n",
    "    else:\n",
    "        axs.axis('off');\n",
    "        axs.imshow(response.images[0], cmap='gray');\n",
    "        axs.set_title(title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "response = env.request(DataRequest())\n",
    "if response is not None:\n",
    "    print(response.metadata)\n",
    "    plot(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cameras=[\n",
    "    (Camera.RGB_LEFT, Compression.OFF, Channels.THREE),\n",
    "    (Camera.SEGMENTATION, Compression.OFF, Channels.THREE),\n",
    "]\n",
    "response = env.request(DataRequest(metadata=False, cameras=cameras))\n",
    "if response is not None:\n",
    "    plot(response)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Metadata request"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "response = env.request(MetadataRequest())\n",
    "if response is not None:\n",
    "    print(response.metadata)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Camera request"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "response = env.request(CameraInformationRequest())\n",
    "if response is not None:\n",
    "    print(response.metadata)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Demonstrate some camera manipulation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reset 3rd person view\n",
    "_ = env.request(SetCameraParametersRequest(Camera.THIRD_PERSON, 768, 1024, 90, .05, 10))\n",
    "_ = env.request(SetCameraPositionRequest(Camera.THIRD_PERSON, 0, 1.319, -2.258))\n",
    "response = env.request(DataRequest(cameras=[(Camera.THIRD_PERSON, Compression.OFF, Channels.THREE)]))\n",
    "if response is not None:\n",
    "    plot(response, 'First Parameters')\n",
    "\n",
    "# Change the parameters and get the camera\n",
    "_ = env.request(SetCameraParametersRequest(Camera.THIRD_PERSON, 768, 1024, 55, .3, 50))\n",
    "response = env.request(DataRequest(cameras=[(Camera.THIRD_PERSON, Compression.OFF, Channels.THREE)]))\n",
    "if response is not None:\n",
    "    plot(response, 'Second Parameters')\n",
    "\n",
    "\n",
    "# Change the position and get the camera\n",
    "_ = env.request(SetCameraPositionRequest(Camera.THIRD_PERSON, 0, 0.5, -2.5))\n",
    "response = env.request(DataRequest(cameras=[(Camera.THIRD_PERSON, Compression.OFF, Channels.THREE)]))\n",
    "if response is not None:\n",
    "    plot(response, 'New Position')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Demo respawning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(3):\n",
    "    env.send(Respawn())\n",
    "    response = env.request(DataRequest(cameras=[(Camera.THIRD_PERSON, Compression.OFF, Channels.THREE)]))\n",
    "    if response is not None:\n",
    "        plot(response)\n",
    "    time.sleep(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Demo changing scenes\n",
    "\n",
    "*NOTE: loading scenes for the first time may take a long time*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<current_scene>\n",
      "  <index>7</index>\n",
      "  <name>CorridorScene_07_V2</name>\n",
      "</current_scene>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "scene_index = 7\n",
    "\n",
    "response = env.request(SceneRequest(scene_index))\n",
    "if response is not None:\n",
    "    print(response.metadata)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Demo step mode\n",
    "\n",
    "This puts the game into step mode. The rate defines the update rate (Hz) for the game.\n",
    "\n",
    "Then, we step forward several times showing how Unit time progresses."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rate = 1\n",
    "env.send(SetFrameRate(rate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unity_time(env):\n",
    "    response = env.request(MetadataRequest())\n",
    "    root = ET.fromstring(response.metadata)\n",
    "    return float(root.find('time').text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "unity_start = unity_time(env)\n",
    "\n",
    "for _ in range(10):\n",
    "    env.send(StepWithForce())\n",
    "    print(\"Elapsed time is \", unity_time(env) - unity_start, \" seconds.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Exit step mode by setting rate to 0\n",
    "rate = 0\n",
    "env.send(SetFrameRate(rate))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Demo colliders\n",
    "\n",
    "Turn collisions off, then back on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.send(ColliderRequest(enable=0)) # turn collisions off"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.send(ColliderRequest(enable=1)) # turn collisions on"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Demo Manipulation of Objects\n",
    "This moves the agent then spawn 36 cubes around the agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = -6\n",
    "y = .5\n",
    "z = 0 #8.5\n",
    "radius = 1.5\n",
    "orientation = [ 0.4619398, 0.1913417, 0.4619398, 0.7325378 ]\n",
    "orientation = [0, 0, 0, 1]\n",
    "env.send(Reposition(x, y, z, 0, 0, 0, 0))\n",
    "time.sleep(.2)\n",
    "\n",
    "for angle in range(0, 360, 10):\n",
    "    response = env.request(SpawnObjectRequest(int((angle % 40)/10),\n",
    "                                ObjectSpawnMethod.USER,\n",
    "                                x + radius*np.cos(angle*np.pi/180), \n",
    "                                y, \n",
    "                                z + radius*np.sin(angle*np.pi/180),\n",
    "                                *orientation)\n",
    "            )\n",
    "    #time.sleep(.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(0, 20):\n",
    "    response = env.request(SpawnObjectRequest(int(i % 2),\n",
    "                                ObjectSpawnMethod.RANDOM))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(env.request(ObjectsRequest()).metadata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove some objects by id\n",
    "env.request(RemoveObjectsRequest([2,4,6,8,10,12,14,16,18,20]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tesse.msgs.DataResponse at 0x1fd2f3917f0>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove all objects\n",
    "env.request(RemoveObjectsRequest())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Demo Random Seed\n",
    "Sets the seed, then randomly respawns twice.\n",
    "\n",
    "Note that this may not be reproducible if a scene or some asset in the scene also uses the Unity Engine's random number generator."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 34512341\n",
    "\n",
    "for i in range(2):\n",
    "    print(\"Iteration \" + str(i) + \" results:\\n\")\n",
    "    env.send(SetRandomSeed(seed))\n",
    "    env.request(RemoveObjectsRequest())\n",
    "    time.sleep(.2)\n",
    "    env.send(Respawn())\n",
    "    time.sleep(1)\n",
    "    env.send(Respawn())\n",
    "    time.sleep(1)\n",
    "    response = env.request(SpawnObjectRequest(ObjectType.CUBE,\n",
    "                                ObjectSpawnMethod.RANDOM))\n",
    "    print(response.metadata)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}