Getdata.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Connect to IB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<IB connected to 127.0.0.1:7497 clientId=1>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from ib_insync import *\n",
    "util.startLoop()  \n",
    "\n",
    "ib = IB()\n",
    "ib.connect('127.0.0.1', 7497, clientId=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get tick by tick data for HSI trading  (RTH & ATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20200924 09:42:02\n",
      "20200924 10:08:50\n",
      "20200924 10:32:44\n",
      "20200924 11:09:33\n",
      "20200924 11:54:17\n",
      "20200924 13:54:06\n",
      "20200924 14:49:44\n",
      "20200924 15:36:08\n",
      "20200924 16:09:28\n",
      "20200924 16:29:59\n",
      "20200924 23:12:05\n",
      "20200925 09:20:04\n",
      "20200925 09:33:54\n",
      "20200925 09:45:27\n",
      "20200925 10:00:19\n",
      "20200925 10:18:08\n",
      "20200925 10:38:40\n",
      "20200925 10:58:21\n",
      "20200925 11:20:46\n",
      "20200925 11:44:12\n",
      "20200925 13:04:22\n",
      "20200925 13:30:05\n",
      "20200925 14:00:54\n",
      "20200925 14:20:47\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Error 1100, reqId -1: Connectivity between IB and Trader Workstation has been lost.\n",
      "Error 1102, reqId -1: Connectivity between IB and Trader Workstation has been restored - data maintained. All data farms are connected: usfarm.nj; hfarm; cashfarm; usfarm; hkhmds; ushmds; secdefnj.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-da2de01db8ae>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     15\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m     \u001b[1;32mwhile\u001b[0m \u001b[0mtiming\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;34mf'{END_DATE} 16:00:00'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 17\u001b[1;33m         \u001b[0mbars\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreqHistoricalTicks\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcontract\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstartDateTime\u001b[0m\u001b[1;33m=\u001b[0m \u001b[0mtiming\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mendDateTime\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m''\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnumberOfTicks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1000\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwhatToShow\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'TRADES'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0museRth\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     18\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mbars\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m             \u001b[0mtiming\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mbars\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtzinfo\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mtimedelta\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhours\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m8\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mseconds\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstrftime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'%Y%m%d %H:%M:%S'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\envs\\mlfinlab\\lib\\site-packages\\ib_insync\\ib.py\u001b[0m in \u001b[0;36mreqHistoricalTicks\u001b[1;34m(self, contract, startDateTime, endDateTime, numberOfTicks, whatToShow, useRth, ignoreSize, miscOptions)\u001b[0m\n\u001b[0;32m   1097\u001b[0m             \u001b[0mmiscOptions\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mUnknown\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1098\u001b[0m         \"\"\"\n\u001b[1;32m-> 1099\u001b[1;33m         return self._run(\n\u001b[0m\u001b[0;32m   1100\u001b[0m             self.reqHistoricalTicksAsync(\n\u001b[0;32m   1101\u001b[0m                 \u001b[0mcontract\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstartDateTime\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mendDateTime\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnumberOfTicks\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\envs\\mlfinlab\\lib\\site-packages\\ib_insync\\ib.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, *awaitables)\u001b[0m\n\u001b[0;32m    308\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    309\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_run\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0mawaitables\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mAwaitable\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 310\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mutil\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mawaitables\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mRequestTimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    311\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    312\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mwaitOnUpdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mfloat\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m->\u001b[0m \u001b[0mbool\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\envs\\mlfinlab\\lib\\site-packages\\ib_insync\\util.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(timeout, *awaitables)\u001b[0m\n\u001b[0;32m    320\u001b[0m         \u001b[0mglobalErrorEvent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconnect\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0monError\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    321\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 322\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mloop\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun_until_complete\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtask\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    323\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0masyncio\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCancelledError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    324\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mglobalErrorEvent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\envs\\mlfinlab\\lib\\site-packages\\nest_asyncio.py\u001b[0m in \u001b[0;36mrun_until_complete\u001b[1;34m(self, future)\u001b[0m\n\u001b[0;32m     90\u001b[0m                 \u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_log_destroy_pending\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     91\u001b[0m             \u001b[1;32mwhile\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdone\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 92\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_run_once\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     93\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stopping\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     94\u001b[0m                     \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\envs\\mlfinlab\\lib\\site-packages\\nest_asyncio.py\u001b[0m in \u001b[0;36m_run_once\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    117\u001b[0m             \u001b[1;32melse\u001b[0m \u001b[1;36m0.01\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_is_proactorloop\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    118\u001b[0m             else None)\n\u001b[1;32m--> 119\u001b[1;33m         \u001b[0mevent_list\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_selector\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mselect\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    120\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_process_events\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mevent_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    121\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\envs\\mlfinlab\\lib\\selectors.py\u001b[0m in \u001b[0;36mselect\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    321\u001b[0m         \u001b[0mready\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    322\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 323\u001b[1;33m             \u001b[0mr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mw\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_select\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_readers\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_writers\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    324\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mInterruptedError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    325\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mready\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\envs\\mlfinlab\\lib\\selectors.py\u001b[0m in \u001b[0;36m_select\u001b[1;34m(self, r, w, _, timeout)\u001b[0m\n\u001b[0;32m    312\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0msys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplatform\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'win32'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    313\u001b[0m         \u001b[1;32mdef\u001b[0m \u001b[0m_select\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mw\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 314\u001b[1;33m             \u001b[0mr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mw\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mselect\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mselect\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mw\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mw\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    315\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mw\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    316\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import time\n",
    "from datetime import timedelta, datetime\n",
    "\n",
    "# Constants\n",
    "list_of_dates = ['20201029', '20201127', '20201230', '20210128', '20210225']\n",
    "for END_DATE in list_of_dates:\n",
    "\n",
    "    # Initialization\n",
    "    contract = Future('HSI', END_DATE, 'HKFE', includeExpired=True)\n",
    "    start_date = (datetime.strptime(END_DATE, '%Y%m%d') - timedelta(days=35)).strftime('%Y%m%d') \n",
    "    df = pd.DataFrame()\n",
    "    temp = pd.DataFrame({'time': [0]})\n",
    "    timing = f'{start_date} 04:00:00'\n",
    "\n",
    "    while timing != f'{END_DATE} 16:00:00':\n",
    "        bars = ib.reqHistoricalTicks(contract, startDateTime= timing, endDateTime='', numberOfTicks=1000, whatToShow='TRADES', useRth=False)\n",
    "        if bars:\n",
    "            timing = (bars[-1].time.replace(tzinfo=None) + timedelta(hours=8, seconds=1)).strftime('%Y%m%d %H:%M:%S')\n",
    "            temp = util.df(bars)[['time','price','size']]\n",
    "\n",
    "            df = pd.concat([df, temp])\n",
    "\n",
    "            print(timing)\n",
    "\n",
    "        else:\n",
    "            timing = (datetime.strptime(timing, '%Y%m%d %H:%M:%S') + timedelta(seconds=1)).strftime('%Y%m%d %H:%M:%S')\n",
    "            \n",
    "    df.to_csv(f'{END_DATE}newhsitick.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'20210121'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import time\n",
    "from datetime import timedelta, datetime\n",
    "\n",
    "END_DATE = '20210225'\n",
    "(datetime.strptime(END_DATE, '%Y%m%d') - timedelta(days=35)).strftime('%Y%m%d')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save as csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('202102hsitick.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get 6 months HSI 5sec bar for first 1 hr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getdatesfuture(date):\n",
    "    # get list of dates \n",
    "    from datetime import timedelta, datetime\n",
    "    todate = datetime.strptime(date, '%Y%m%d')\n",
    "    delta = timedelta(days=35)    \n",
    "    list_of_dates = []\n",
    "\n",
    "    for i in range(delta.days + 1):\n",
    "        day = todate - delta + timedelta(days=i)\n",
    "        list_of_dates.append(day.strftime(\"%Y%m%d\"))\n",
    "\n",
    "    return list_of_dates\n",
    "\n",
    "\n",
    "def getdatafuture(date):\n",
    "    # get traded price of future\n",
    "    import pandas as pd\n",
    "\n",
    "    contract = Future('HSI', date, 'HKFE', includeExpired=True)\n",
    "\n",
    "    df = pd.DataFrame()\n",
    "\n",
    "    list_of_dates = getdatesfuture(date)\n",
    "\n",
    "    for i in list_of_dates:\n",
    "        bars = ib.reqHistoricalData(contract, endDateTime=f'{i} 09:45:00', durationStr='1800 S', barSizeSetting='1 secs', whatToShow='BID', useRTH=True)\n",
    "        dfbid = util.df(bars)\n",
    "\n",
    "        bars = ib.reqHistoricalData(contract, endDateTime=f'{i} 09:45:00', durationStr='1800 S', barSizeSetting='1 secs', whatToShow='ASK', useRTH=True)\n",
    "        dfask = util.df(bars)\n",
    "\n",
    "        merged = pd.merge(dfbid,dfask, left_index=True, right_index=True,suffixes=('bid','ask'))\n",
    "        temp = merged[['datebid', 'openbid', 'highbid', 'lowbid', 'openask', 'highask', 'lowask']]\n",
    "\n",
    "        bars = ib.reqHistoricalData(contract, endDateTime=f'{i} 09:45:00', durationStr='1800 S', barSizeSetting='1 secs', whatToShow='TRADES', useRTH=True)\n",
    "        dftrades = util.df(bars)\n",
    "\n",
    "        merged = pd.merge(temp,dftrades, left_index=True, right_index=True,suffixes=('','trades'))\n",
    "        temp = merged[['datebid', 'openbid', 'highbid', 'lowbid', 'openask', 'highask', 'lowask', 'open', 'high', 'low', 'close', 'volume', 'average', 'barCount']]\n",
    "\n",
    "        df = pd.concat([df, temp])\n",
    "        print(i)\n",
    "        \n",
    "        time.sleep(30)  \n",
    "        \n",
    "    return(df)\n",
    "\n",
    "#Get recent 2 years data\n",
    "import pandas as pd\n",
    "import time\n",
    "df = pd.DataFrame()\n",
    "\n",
    "for year in range(2020,2021):\n",
    "    for month in range(6,13):\n",
    "        for day in range(27,31):\n",
    "            try:\n",
    "                df = df.append(getdatafuture(f'{year}{month+1:02d}{day}'))\n",
    "            except:\n",
    "                pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get HSI 30secs bar inside and outside RTH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getdatesfuture(date):\n",
    "    # get list of dates \n",
    "    from datetime import timedelta, datetime\n",
    "    todate = datetime.strptime(date, '%Y%m%d')\n",
    "    delta = timedelta(days=35)    \n",
    "    list_of_dates = []\n",
    "\n",
    "    for i in range(delta.days + 1):\n",
    "        day = todate - delta + timedelta(days=i)\n",
    "        list_of_dates.append(day.strftime(\"%Y%m%d\"))\n",
    "\n",
    "    return list_of_dates\n",
    "\n",
    "\n",
    "def getdatafuture(date):\n",
    "    # get traded price of future\n",
    "    import pandas as pd\n",
    "\n",
    "    contract = Future('HSI', date, 'HKFE', includeExpired=True)\n",
    "\n",
    "    df = pd.DataFrame()\n",
    "\n",
    "    list_of_dates = getdatesfuture(date)\n",
    "\n",
    "    for i in list_of_dates:\n",
    "\n",
    "        bars = ib.reqHistoricalData(contract, endDateTime=f'{i} 16:30:00', durationStr='1 D', barSizeSetting='30 secs', whatToShow='TRADES', useRTH=False)\n",
    "        temp = util.df(bars)\n",
    "        temp = temp[['date', 'open', 'high', 'low', 'close', 'volume', 'average', 'barCount']]\n",
    "        \n",
    "        try:\n",
    "            check = df.iloc[-1].equals(temp.iloc[-1])\n",
    "        except:\n",
    "            check = False\n",
    "\n",
    "        if not check:\n",
    "            df = pd.concat([df, temp])\n",
    "            print(i)\n",
    "                \n",
    "    return(df)\n",
    "\n",
    "#Get recent 2 years data\n",
    "import pandas as pd\n",
    "import time\n",
    "df = pd.DataFrame()\n",
    "\n",
    "for year in range(2019,2021):\n",
    "    for month in range(1,13):\n",
    "        for day in range(25,31):\n",
    "            try:\n",
    "                df = df.append(getdatafuture(f'{year}{month+1:02d}{day}'))\n",
    "            except:\n",
    "                pass"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}