diff --git a/.gitignore b/.gitignore index 346901b..9054f90 100644 --- a/.gitignore +++ b/.gitignore @@ -81,4 +81,6 @@ server/yarn.lock yarn.lock server/.yarn/ -packages/snap/snap.manifest.json \ No newline at end of file +packages/snap/snap.manifest.json + +__pycache__ \ No newline at end of file diff --git a/forecaster/.dockerignore b/forecaster/.dockerignore new file mode 100644 index 0000000..21abcf3 --- /dev/null +++ b/forecaster/.dockerignore @@ -0,0 +1,17 @@ +__pycache__/ +.git +.vscode +.dockerignore +.gitignore +.env +config +build +node_modules +docker-compose.dev.yaml +docker-compose.prod.yaml +docker-compose.yaml +Dockerfile +Dockerfile.dev +Dockerfile.prod +Makefile +README.md \ No newline at end of file diff --git a/forecaster/Dockerfile b/forecaster/Dockerfile new file mode 100644 index 0000000..0d3266d --- /dev/null +++ b/forecaster/Dockerfile @@ -0,0 +1,22 @@ +# Use the official Python base image +FROM pytorch/pytorch:latest + +# Set the working directory inside the container +WORKDIR /app + +# Copy the requirements file to the working directory +COPY requirements.txt . + +RUN echo $(python3 -m site --user-base) + +# Install the Python dependencies +RUN pip install -r requirements.txt + +# Copy the application code to the working directory +COPY . . + +# Expose the port on which the application will run +EXPOSE 8000 + +# Run the FastAPI application using uvicorn server +CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file diff --git a/forecaster/README.md b/forecaster/README.md new file mode 100644 index 0000000..1b36883 --- /dev/null +++ b/forecaster/README.md @@ -0,0 +1,18 @@ +# Python APT to USD rate forecasting engine +This is a simple python script that uses the BiLSTM model architecture to forecast the APT to USD rate for the next few minutes,given the data of last 16 minutes. + +## Start the server +```bash +cd bin/ +./deploy.sh up +``` +This command might take more than 10 minutes on first run as it will download the docker images and build the containers. +Server will run on localhost:9000 +View the documentation at localhost:9000/docs + +## Stop the server +```bash +cd bin/ +./deploy.sh down +``` + diff --git a/forecaster/apt.csv b/forecaster/apt.csv new file mode 100644 index 0000000..76243a8 --- /dev/null +++ b/forecaster/apt.csv @@ -0,0 +1,532 @@ +Minutes,Price +0,8.331683333 +1,8.333775 +2,8.3360875 +3,8.344625 +4,8.346854167 +5,8.342758333 +6,8.3458875 +7,8.347158333 +8,8.341604167 +9,8.356233333 +10,8.354408333 +11,8.357054167 +12,8.360329167 +13,8.3622625 +14,8.363383333 +15,8.363120833 +16,8.387416667 +17,8.396691667 +18,8.412808333 +19,8.41155 +20,8.4056875 +21,8.3965375 +22,8.386908333 +23,8.377266667 +24,8.367720833 +25,8.361858333 +26,8.359229167 +27,8.3565375 +28,8.368866667 +29,8.3641875 +30,8.363516667 +31,8.3560375 +32,8.3565125 +33,8.352016667 +34,8.340620833 +35,8.331858333 +36,8.338791667 +37,8.33665 +38,8.326025 +39,8.331991667 +40,8.3216 +41,8.325058333 +42,8.3224 +43,8.3259875 +44,8.32405 +45,8.326970833 +46,8.325725 +47,8.319775 +48,8.325266667 +49,8.3170875 +50,8.318041667 +51,8.321704167 +52,8.332308333 +53,8.345833333 +54,8.353933333 +55,8.356575 +56,8.360391667 +57,8.374766667 +58,8.383204167 +59,8.3805125 +60,8.378541667 +61,8.390020833 +62,8.400608333 +63,8.412541667 +64,8.4121 +65,8.412425 +66,8.406008333 +67,8.4002625 +68,8.397791667 +69,8.395225 +70,8.395654167 +71,8.396029167 +72,8.395158333 +73,8.3932875 +74,8.3995625 +75,8.395520833 +76,8.391995833 +77,8.393145833 +78,8.397370833 +79,8.404316667 +80,8.415654167 +81,8.421154167 +82,8.4212375 +83,8.4153875 +84,8.422845833 +85,8.428325 +86,8.4187 +87,8.401916667 +88,8.406554167 +89,8.400691667 +90,8.3997375 +91,8.395016667 +92,8.403675 +93,8.4129375 +94,8.415616667 +95,8.413566667 +96,8.414779167 +97,8.424033333 +98,8.421833333 +99,8.4248375 +100,8.441208333 +101,8.426908333 +102,8.430925 +103,8.4347625 +104,8.4147 +105,8.414933333 +106,8.413175 +107,8.417466667 +108,8.4249375 +109,8.432775 +110,8.430516667 +111,8.425166667 +112,8.4171875 +113,8.415658333 +114,8.418854167 +115,8.416216667 +116,8.416691667 +117,8.419504167 +118,8.41395 +119,8.418625 +120,8.418179167 +121,8.409520833 +122,8.404183333 +123,8.388904167 +124,8.400558333 +125,8.408383333 +126,8.412975 +127,8.404033333 +128,8.406358333 +129,8.402266667 +130,8.3994375 +131,8.402875 +132,8.4028125 +133,8.394233333 +134,8.394320833 +135,8.38835 +136,8.386895833 +137,8.386129167 +138,8.391754167 +139,8.390429167 +140,8.388133333 +141,8.386808333 +142,8.388433333 +143,8.386016667 +144,8.3922875 +145,8.3845 +146,8.390558333 +147,8.37705 +148,8.36475 +149,8.382379167 +150,8.405441667 +151,8.41465 +152,8.404241667 +153,8.398304167 +154,8.405558333 +155,8.4079875 +156,8.408058333 +157,8.404941667 +158,8.3997875 +159,8.404425 +160,8.405825 +161,8.392470833 +162,8.394991667 +163,8.402666667 +164,8.410679167 +165,8.426216667 +166,8.416775 +167,8.401995833 +168,8.4053125 +169,8.405304167 +170,8.404804167 +171,8.4057375 +172,8.411533333 +173,8.411725 +174,8.409691667 +175,8.403079167 +176,8.3952 +177,8.3953875 +178,8.388870833 +179,8.388870833 +180,8.383175 +181,8.399345833 +182,8.409116667 +183,8.410475 +184,8.414116667 +185,8.422545833 +186,8.431729167 +187,8.428775 +188,8.422045833 +189,8.419245833 +190,8.4209875 +191,8.424895833 +192,8.429725 +193,8.428670833 +194,8.423933333 +195,8.421979167 +196,8.421054167 +197,8.4222375 +198,8.425358333 +199,8.4280625 +200,8.4316 +201,8.4333 +202,8.43215 +203,8.436304167 +204,8.4355875 +205,8.44185 +206,8.448908333 +207,8.454433333 +208,8.4573375 +209,8.454066667 +210,8.428595833 +211,8.4225875 +212,8.410991667 +213,8.436966667 +214,8.437120833 +215,8.422408333 +216,8.426025 +217,8.424591667 +218,8.421933333 +219,8.422133333 +220,8.423220833 +221,8.4294125 +222,8.423091667 +223,8.422683333 +224,8.416933333 +225,8.409508333 +226,8.402679167 +227,8.393370833 +228,8.399891667 +229,8.399120833 +230,8.4012625 +231,8.4016125 +232,8.402995833 +233,8.404175 +234,8.409775 +235,8.41295 +236,8.4157125 +237,8.4442375 +238,8.446716667 +239,8.443170833 +240,8.4582125 +241,8.470604167 +242,8.466229167 +243,8.461095833 +244,8.453154167 +245,8.456133333 +246,8.457241667 +247,8.4530375 +248,8.450895833 +249,8.441733333 +250,8.435933333 +251,8.419904167 +252,8.414308333 +253,8.407079167 +254,8.37615 +255,8.383770833 +256,8.398495833 +257,8.387629167 +258,8.377 +259,8.38975 +260,8.409441667 +261,8.428408333 +262,8.445370833 +263,8.445433333 +264,8.4439125 +265,8.442958333 +266,8.445304167 +267,8.467145833 +268,8.459475 +269,8.455916667 +270,8.441045833 +271,8.450020833 +272,8.433183333 +273,8.439108333 +274,8.4016875 +275,8.417025 +276,8.417020833 +277,8.425045833 +278,8.4330875 +279,8.426158333 +280,8.406070833 +281,8.396366667 +282,8.415020833 +283,8.4033875 +284,8.401779167 +285,8.400795833 +286,8.4021375 +287,8.4040625 +288,8.4017625 +289,8.405475 +290,8.4006875 +291,8.396145833 +292,8.4014375 +293,8.412454167 +294,8.421945833 +295,8.405941667 +296,8.404233333 +297,8.408691667 +298,8.405854167 +299,8.4053875 +300,8.408883333 +301,8.414529167 +302,8.424570833 +303,8.438945833 +304,8.428929167 +305,8.4137625 +306,8.407804167 +307,8.403533333 +308,8.392341667 +309,8.400545833 +310,8.395458333 +311,8.386841667 +312,8.373275 +313,8.379383333 +314,8.3842625 +315,8.397420833 +316,8.403958333 +317,8.3831 +318,8.3928625 +319,8.398954167 +320,8.409595833 +321,8.407195833 +322,8.401683333 +323,8.404645833 +324,8.405283333 +325,8.417770833 +326,8.422708333 +327,8.429408333 +328,8.42865 +329,8.408591667 +330,8.403841667 +331,8.395329167 +332,8.401595833 +333,8.403695833 +334,8.3979625 +335,8.394333333 +336,8.393283333 +337,8.386491667 +338,8.3890625 +339,8.360720833 +340,8.373475 +341,8.386891667 +342,8.380783333 +343,8.35455 +344,8.3462875 +345,8.348045833 +346,8.356558333 +347,8.351766667 +348,8.345579167 +349,8.3422875 +350,8.347366667 +351,8.343558333 +352,8.342883333 +353,8.334775 +354,8.335129167 +355,8.319383333 +356,8.307770833 +357,8.303841667 +358,8.300816667 +359,8.293641667 +360,8.275720833 +361,8.2553125 +362,8.247020833 +363,8.250391667 +364,8.276833333 +365,8.279483333 +366,8.287875 +367,8.268616667 +368,8.2779125 +369,8.2725875 +370,8.311066667 +371,8.308566667 +372,8.296733333 +373,8.2918125 +374,8.3032875 +375,8.3004125 +376,8.308608333 +377,8.319933333 +378,8.31555 +379,8.313420833 +380,8.301341667 +381,8.2856625 +382,8.304520833 +383,8.320858333 +384,8.323833333 +385,8.316829167 +386,8.307345833 +387,8.2923 +388,8.291758333 +389,8.302083333 +390,8.303575 +391,8.312745833 +392,8.316045833 +393,8.3226625 +394,8.315616667 +395,8.317583333 +396,8.330425 +397,8.3305375 +398,8.318083333 +399,8.31365 +400,8.3187125 +401,8.321091667 +402,8.311795833 +403,8.314354167 +404,8.316408333 +405,8.320058333 +406,8.3181875 +407,8.321075 +408,8.313691667 +409,8.312391667 +410,8.2976 +411,8.3017 +412,8.3077625 +413,8.293258333 +414,8.292183333 +415,8.258266667 +416,8.267833333 +417,8.230275 +418,8.205279167 +419,8.218191667 +420,8.192425 +421,8.146279167 +422,7.856466667 +423,7.4951875 +424,7.613420833 +425,7.6529875 +426,7.682591667 +427,7.6917875 +428,7.748879167 +429,7.768908333 +430,7.782208333 +431,7.784183333 +432,7.7969875 +433,7.829479167 +434,7.838975 +435,7.825920833 +436,7.76995 +437,7.820233333 +438,7.834141667 +439,7.8402375 +440,7.855579167 +441,7.831358333 +442,7.807929167 +443,7.837195833 +444,7.841416667 +445,7.822279167 +446,7.816970833 +447,7.805545833 +448,7.796308333 +449,7.815183333 +450,7.8014375 +451,7.776420833 +452,7.756554167 +453,7.740920833 +454,7.763079167 +455,7.750066667 +456,7.736179167 +457,7.7311625 +458,7.7849375 +459,7.8110125 +460,7.796133333 +461,7.829658333 +462,7.837908333 +463,7.848891667 +464,7.870420833 +465,7.8666125 +466,7.850970833 +467,7.866054167 +468,7.862270833 +469,7.875383333 +470,7.86195 +471,7.859954167 +472,7.870383333 +473,7.842666667 +474,7.868316667 +475,7.863925 +476,7.871975 +477,7.876066667 +478,7.8813375 +479,7.864558333 +480,7.860345833 +481,7.876091667 +482,7.8728875 +483,7.866566667 +484,7.8628875 +485,7.8738 +486,7.874341667 +487,7.870241667 +488,7.856529167 +489,7.8581 +490,7.8536875 +491,7.8593125 +492,7.853404167 +493,7.8674125 +494,7.871070833 +495,7.8714375 +496,7.878475 +497,7.898358333 +498,7.897554167 +499,7.8821875 +500,7.8812375 +501,7.878995833 +502,7.896566667 +503,7.89495 +504,7.89255 +505,7.899654167 +506,7.9174 +507,7.924591667 +508,7.926766667 +509,7.927916667 +510,7.931616667 +511,7.928908333 +512,7.921645833 +513,7.918291667 +514,7.9192625 +515,7.914329167 +516,7.913491667 +517,7.9075125 +518,7.904775 +519,7.893825 +520,7.886429167 +521,7.892045833 +522,7.909529167 +523,7.906475 +524,7.905204167 +525,7.891854167 +526,7.892679167 +527,7.8667875 +528,7.868541667 +529,7.881320833 +530,7.867704167 diff --git a/forecaster/bin/deploy.sh b/forecaster/bin/deploy.sh new file mode 100755 index 0000000..0900c26 --- /dev/null +++ b/forecaster/bin/deploy.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [[ $1 = "up" || $1 = "down" ]]; then + cd .. + fileEnv="docker-compose.yaml" + downOrUp=$1 + echo "Running docker-compose -f docker-compose.yaml ${downOrUp}" + docker-compose -f docker-compose.yaml $downOrUp +else + echo "Usage: ./deploy.sh [up|down]" +fi \ No newline at end of file diff --git a/forecaster/docker-compose.yaml b/forecaster/docker-compose.yaml new file mode 100644 index 0000000..f6ab6fb --- /dev/null +++ b/forecaster/docker-compose.yaml @@ -0,0 +1,13 @@ +version: '3.1' +services: + app: + image: app-image + build: + context: ./ + dockerfile: Dockerfile + container_name: app-c + volumes: + - .:/app + ports: + - 9000:8000 + \ No newline at end of file diff --git a/forecaster/requirements.txt b/forecaster/requirements.txt new file mode 100644 index 0000000..99e21ba --- /dev/null +++ b/forecaster/requirements.txt @@ -0,0 +1,7 @@ +fastapi==0.104.1 +matplotlib==3.5.1 +numpy==1.22.4 +pandas==2.1.4 +pydantic==1.10.12 +scikit_learn==1.3.0 +uvicorn>=0.15.0,<0.16.0 diff --git a/forecaster/server.py b/forecaster/server.py new file mode 100644 index 0000000..1ff269c --- /dev/null +++ b/forecaster/server.py @@ -0,0 +1,241 @@ +from fastapi import FastAPI +from pydantic import BaseModel +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt +import torch +import torch.nn as nn +from copy import deepcopy as dc +from sklearn.preprocessing import MinMaxScaler +from torch.utils.data import Dataset +from torch.utils.data import DataLoader + +device = "cuda:0" if torch.cuda.is_available() else "cpu" + + +class PriceData(BaseModel): # price data of last 16 time stamps + price_0: float + price_1: float + price_2: float + price_3: float + price_4: float + price_5: float + price_6: float + price_7: float + price_8: float + price_9: float + price_10: float + price_11: float + price_12: float + price_13: float + price_14: float + price_15: float + + +class TimeSeriesDataset(Dataset): + def __init__(self, X, y): + self.X = X + self.y = y + + def __len__(self): + return len(self.X) + + def __getitem__(self, i): + return self.X[i], self.y[i] + + +class LSTM(nn.Module): + def __init__(self, input_size, hidden_size, hidden_size_2, num_stacked_layers): + super().__init__() + self.hidden_size = hidden_size + self.hidden_size_2 = hidden_size_2 + self.num_stacked_layers = num_stacked_layers + self.lstm = nn.LSTM( + input_size, hidden_size, num_stacked_layers, batch_first=True + ) + self.relu = nn.ReLU() + self.fc = nn.Linear(hidden_size_2, 1) + self.fc2 = nn.Linear(hidden_size, hidden_size_2) + + def forward(self, x): + batch_size = x.size(0) + h0 = torch.zeros(self.num_stacked_layers, batch_size, self.hidden_size).to( + device + ) + c0 = torch.zeros(self.num_stacked_layers, batch_size, self.hidden_size).to( + device + ) + out, _ = self.lstm(x, (h0, c0)) + out = self.relu(out[:, -1, :]) + out = self.fc2(out) + out = self.relu(out) + out = self.fc(out) + return out + + +# Create a bidirectional LSTM model class +class BiLSTM(nn.Module): + def __init__(self, input_size, hidden_size, hidden_size_2, num_layers): + super(BiLSTM, self).__init__() + self.hidden_size = hidden_size + self.hidden_size_2 = hidden_size_2 + self.num_layers = num_layers + self.lstm = nn.LSTM( + input_size, hidden_size, num_layers, batch_first=True, bidirectional=True + ) + self.fc = nn.Linear(hidden_size * 2, 1) + + def forward(self, x): + # h0 = torch.zeros(self.num_layers * 2, x.size(0), self.hidden_size).to(device) + # c0 = torch.zeros(self.num_layers * 2, x.size(0), self.hidden_size).to(device) + + out, _ = self.lstm(x) + out = self.fc(out[:, -1, :]) + + return out + + +def prepare_df_for_lstm(df, n_steps): + df = dc(df) + + df.set_index("Minutes", inplace=True) + + for i in range(1, n_steps + 1): + df[f"Price(t-{i})"] = df["Price"].shift(i) + df.dropna(inplace=True) + return df + + +def train_one_epoch(model, epoch, train_loader, loss_function, optimizer): + model.train(True) + print(f"Epoch: {epoch + 1}") + running_loss = 0.0 + + for batch_index, batch in enumerate(train_loader): + x_batch, y_batch = batch[0].to(device), batch[1].to(device) + output = model(x_batch) + loss = loss_function(output, y_batch) + running_loss += loss + optimizer.zero_grad() + loss.backward() + optimizer.step() + + if batch_index % 100 == 99: + avg_loss_batches = running_loss / 100 + print("Batch: {0}, loss: {1:.3f}".format(batch_index + 1, avg_loss_batches)) + running_loss = 0.0 + + +def validate_one_epoch(model, test_loader, loss_function): + model.train(False) + running_loss = 0.0 + for batch_index, batch in enumerate(test_loader): + x_batch, y_batch = batch[0].to(device), batch[1].to(device) + with torch.no_grad(): + output = model(x_batch) + loss = loss_function(output, y_batch) + running_loss += loss + + avg_loss_batches = running_loss / len(test_loader) + + print("Val loss: {0:.3f}".format(avg_loss_batches)) + print("*********************************************") + + +app = FastAPI() + + +@app.get("/") +async def root(): + return {"message": "Hello World"} + + +def train(): + data = pd.read_csv("apt.csv") + data = data[["Minutes", "Price"]] + lookback = 16 + shifted_df = prepare_df_for_lstm(data, lookback) + shifted_df_as_np = shifted_df.to_numpy() + scaler = MinMaxScaler(feature_range=(-1, 1)) + shifted_df_as_np = scaler.fit_transform(shifted_df_as_np) + X = shifted_df_as_np[:, 1:] # all the rows(first index), but first col onwards + y = shifted_df_as_np[:, 0] + X = dc(np.flip(X, axis=1)) + split_index = int(len(X) * 0.95) + X_train = X[:split_index] + X_test = X[split_index:] + y_train = y[:split_index] + y_test = y[split_index:] + # it is required for pytorch LSTMs to have an extra dimention at the end + X_train = X_train.reshape((-1, lookback, 1)) + X_test = X_test.reshape((-1, lookback, 1)) + y_train = y_train.reshape((-1, 1)) + y_test = y_test.reshape((-1, 1)) + X_train = torch.tensor(X_train).float() + X_test = torch.tensor(X_test).float() + y_train = torch.tensor(y_train).float() + y_test = torch.tensor(y_test).float() + train_dataset = TimeSeriesDataset(X_train, y_train) + test_dataset = TimeSeriesDataset(X_test, y_test) + batch_size = 9 + train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) + test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False) + for _, batch in enumerate(train_loader): + x_batch, y_batch = batch[0].to(device), batch[1].to(device) + print(x_batch.shape, y_batch.shape) + break + model = BiLSTM(1, 8, 4, 1) + model.to(device) + learning_rate = 0.001 + num_epochs = 100 + loss_function = nn.MSELoss() + optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) + for epoch in range(num_epochs): + train_one_epoch(model, epoch, train_loader, loss_function, optimizer) + validate_one_epoch(model, test_loader, loss_function) + return model + + +model = train() + + +@app.post("/predict/") +async def predict_price(price_data: PriceData): + device = "cuda:0" if torch.cuda.is_available() else "cpu" + model.to(device) + model.eval() + price_data = np.array( + [ + price_data.price_0, + price_data.price_1, + price_data.price_2, + price_data.price_3, + price_data.price_4, + price_data.price_5, + price_data.price_6, + price_data.price_7, + price_data.price_8, + price_data.price_9, + price_data.price_10, + price_data.price_11, + price_data.price_12, + price_data.price_13, + price_data.price_14, + price_data.price_15, + ] + ) + price_data = torch.tensor(price_data).float() + # normalize the data + scaler = MinMaxScaler(feature_range=(-1, 1)) + price_data = price_data.reshape((-1, 1)) + price_data_norm = scaler.fit_transform(price_data) + price_data = price_data_norm.reshape((-1, 16, 1)) + print(price_data.shape) + price_data = torch.tensor(price_data).float().to(device) + with torch.no_grad(): + output = model(price_data) + output = scaler.inverse_transform(output.cpu().numpy()) + prediction = { + "price": output.item(), + } + return prediction