nvda_1min_dwnld.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import pandas as pd\n",
    "import json\n",
    "import mysql.connector\n",
    "from datetime import datetime, timedelta\n",
    "import time\n",
    "import nvda_config\n",
    "\n",
    "import twelvedata\n",
    "from twelvedata import TDClient\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set up login\n",
    "Cpassword = nvda_config.mysql_password\n",
    "td = TDClient(apikey=nvda_config.twelve_api_nvda) \n",
    "base_url = 'https://api.twelvedata.com'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Establish MySQL object\n",
    "mydb = mysql.connector.connect(\n",
    "    host=\"localhost\",   \n",
    "    user=\"root\",         \n",
    "    password=Cpassword,  \n",
    "    database=\"nvdav1\"   \n",
    ")\n",
    "\n",
    "mycursor = mydb.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'volume'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python312\\site-packages\\pandas\\core\\indexes\\base.py:3805\u001b[0m, in \u001b[0;36mIndex.get_loc\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   3804\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m-> 3805\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_loc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcasted_key\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   3806\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n",
      "File \u001b[1;32mindex.pyx:167\u001b[0m, in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32mindex.pyx:196\u001b[0m, in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32mpandas\\\\_libs\\\\hashtable_class_helper.pxi:7081\u001b[0m, in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32mpandas\\\\_libs\\\\hashtable_class_helper.pxi:7089\u001b[0m, in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'volume'",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[6], line 45\u001b[0m\n\u001b[0;32m     42\u001b[0m \u001b[38;5;66;03m# Check if volume is missing\u001b[39;00m\n\u001b[0;32m     43\u001b[0m volume_value \u001b[38;5;241m=\u001b[39m row[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvolume\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvolume\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m row \u001b[38;5;129;01mand\u001b[39;00m pd\u001b[38;5;241m.\u001b[39mnotna(row[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvolume\u001b[39m\u001b[38;5;124m\"\u001b[39m]) \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m  \u001b[38;5;66;03m# Set to None if missing\u001b[39;00m\n\u001b[1;32m---> 45\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m pd\u001b[38;5;241m.\u001b[39mnotna(\u001b[43mrow\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mvolume\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m):\n\u001b[0;32m     46\u001b[0m     time_series_data\u001b[38;5;241m.\u001b[39mappend(\n\u001b[0;32m     47\u001b[0m         (\n\u001b[0;32m     48\u001b[0m             symbol,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     55\u001b[0m         )\n\u001b[0;32m     56\u001b[0m     )\n\u001b[0;32m     57\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m pd\u001b[38;5;241m.\u001b[39mnotna(row[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmacd\u001b[39m\u001b[38;5;124m\"\u001b[39m]):  \u001b[38;5;66;03m# Check if MACD is not NaN\u001b[39;00m\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python312\\site-packages\\pandas\\core\\series.py:1121\u001b[0m, in \u001b[0;36mSeries.__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   1118\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_values[key]\n\u001b[0;32m   1120\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m key_is_scalar:\n\u001b[1;32m-> 1121\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_value\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1123\u001b[0m \u001b[38;5;66;03m# Convert generator to list before going through hashable part\u001b[39;00m\n\u001b[0;32m   1124\u001b[0m \u001b[38;5;66;03m# (We will iterate through the generator there to check for slices)\u001b[39;00m\n\u001b[0;32m   1125\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_iterator(key):\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python312\\site-packages\\pandas\\core\\series.py:1237\u001b[0m, in \u001b[0;36mSeries._get_value\u001b[1;34m(self, label, takeable)\u001b[0m\n\u001b[0;32m   1234\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_values[label]\n\u001b[0;32m   1236\u001b[0m \u001b[38;5;66;03m# Similar to Index.get_value, but we do not fall back to positional\u001b[39;00m\n\u001b[1;32m-> 1237\u001b[0m loc \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mindex\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_loc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlabel\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1239\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_integer(loc):\n\u001b[0;32m   1240\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_values[loc]\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python312\\site-packages\\pandas\\core\\indexes\\base.py:3812\u001b[0m, in \u001b[0;36mIndex.get_loc\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   3807\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(casted_key, \u001b[38;5;28mslice\u001b[39m) \u001b[38;5;129;01mor\u001b[39;00m (\n\u001b[0;32m   3808\u001b[0m         \u001b[38;5;28misinstance\u001b[39m(casted_key, abc\u001b[38;5;241m.\u001b[39mIterable)\n\u001b[0;32m   3809\u001b[0m         \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28many\u001b[39m(\u001b[38;5;28misinstance\u001b[39m(x, \u001b[38;5;28mslice\u001b[39m) \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m casted_key)\n\u001b[0;32m   3810\u001b[0m     ):\n\u001b[0;32m   3811\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m InvalidIndexError(key)\n\u001b[1;32m-> 3812\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(key) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01merr\u001b[39;00m\n\u001b[0;32m   3813\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m:\n\u001b[0;32m   3814\u001b[0m     \u001b[38;5;66;03m# If we have a listlike key, _check_indexing_error will raise\u001b[39;00m\n\u001b[0;32m   3815\u001b[0m     \u001b[38;5;66;03m#  InvalidIndexError. Otherwise we fall through and re-raise\u001b[39;00m\n\u001b[0;32m   3816\u001b[0m     \u001b[38;5;66;03m#  the TypeError.\u001b[39;00m\n\u001b[0;32m   3817\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_indexing_error(key)\n",
      "\u001b[1;31mKeyError\u001b[0m: 'volume'"
     ]
    }
   ],
   "source": [
    "# Date range for historical data (funky start date due to downloading first month as test batch)\n",
    "end_date = datetime.now()- timedelta(days=30)\n",
    "start_date = end_date - timedelta(days=395)  \n",
    "\n",
    "# Twelve Data API parameters\n",
    "symbol = \"NVDA\"\n",
    "interval = \"1min\"\n",
    "outputsize = 5000  \n",
    "\n",
    "# Fetch data in chunks\n",
    "current_date = start_date\n",
    "requests_made = 0\n",
    "\n",
    "while current_date < end_date:\n",
    "    if requests_made >= 2:  # Check if 4 requests have been made\n",
    "        time.sleep(60)  # Wait for 60 seconds (1 minute)\n",
    "        requests_made = 0  # Reset the request counter\n",
    "\n",
    "    chunk_end_date = current_date + timedelta(days=30)  # Fetch 30 days at a time\n",
    "\n",
    "    # Fetch time series data and indicators\n",
    "    time_series = td.time_series(\n",
    "        symbol=symbol,\n",
    "        interval=interval,\n",
    "        outputsize=outputsize,\n",
    "        start_date=current_date.strftime(\"%Y-%m-%d\"),\n",
    "        end_date=chunk_end_date.strftime(\"%Y-%m-%d\"),\n",
    "    ).with_macd().with_ema().with_rsi()\n",
    "    \n",
    "    # Convert to pandas Dataframes\n",
    "    time_series_df = time_series.as_pandas()\n",
    "    time_series_df = time_series_df.astype(str)\n",
    "\n",
    "    # Prepare for database insertion\n",
    "    time_series_data = []\n",
    "    indicator_data = []\n",
    "\n",
    "    # Process and align data\n",
    "    for index, row in time_series_df.iterrows():\n",
    "        # Extract the datetime string\n",
    "        datetime_str = row.name.strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "\n",
    "        time_series_data.append(\n",
    "            (\n",
    "                symbol,\n",
    "                datetime_str,\n",
    "                row[\"open\"],\n",
    "                row[\"high\"],\n",
    "                row[\"low\"],\n",
    "                row[\"close\"],\n",
    "                row[\"volume\"],\n",
    "            )\n",
    "        )\n",
    "        if pd.notna(row[\"macd\"]):  # Check if MACD is not NaN\n",
    "            indicator_data.append(\n",
    "                (\n",
    "                    symbol,\n",
    "                    datetime_str,\n",
    "                    row[\"macd\"],\n",
    "                    row[\"macd_signal\"],\n",
    "                    row[\"macd_hist\"],\n",
    "                    row[\"ema\"],\n",
    "                    row[\"rsi\"],\n",
    "                )\n",
    "            )\n",
    "\n",
    "\n",
    "    # Insert tick data into MySQL\n",
    "    sql_time_series = \"\"\"\n",
    "    INSERT INTO stock_ticks (symbol, datetime, open, high, low, close, volume)\n",
    "    VALUES (%s, %s, %s, %s, %s, %s, %s)\n",
    "    \"\"\"\n",
    "    mycursor.executemany(sql_time_series, time_series_data)\n",
    "\n",
    "    # Insert indicator data into MySQL\n",
    "    sql_indicators = \"\"\"\n",
    "    INSERT INTO technical_indicators (symbol, bydatetime, macd, bysignal, byhistogram, ema, rsi)\n",
    "    VALUES (%s, %s, %s, %s, %s, %s, %s)\n",
    "    \"\"\"\n",
    "    mycursor.executemany(sql_indicators, indicator_data)\n",
    "    current_date = chunk_end_date\n",
    "    requests_made += 1\n",
    " \n",
    "\n",
    "mydb.commit()\n",
    "print(mycursor.rowcount, \"records inserted.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}