From 379d082fe99c8fc7b1db5d6b3985688dd3d889f2 Mon Sep 17 00:00:00 2001 From: Julien Date: Mon, 24 Jul 2023 09:24:54 +0200 Subject: [PATCH] changed airflow version (python 3.9), dockerfile and docker compose --- fundamental_data/Dockerfile | 6 ++- .../dags/fetch_fundamental_dag.py | 39 +----------------- fundamental_data/docker-compose.yaml | 4 +- fundamental_data/fetch_fundamental_dag.py | 7 +--- fundamental_data/logs/scheduler/latest | 2 +- fundamental_data/requirements.txt | Bin 104 -> 45 bytes 6 files changed, 11 insertions(+), 47 deletions(-) diff --git a/fundamental_data/Dockerfile b/fundamental_data/Dockerfile index 8b02645..24a724b 100644 --- a/fundamental_data/Dockerfile +++ b/fundamental_data/Dockerfile @@ -1,3 +1,5 @@ -FROM apache/airflow:2.1.1 -COPY requirements.txt . +FROM apache/airflow:2.1.2-python3.9 +COPY fundamental_data/requirements.txt . +COPY src/domain/utils/fetch_data.py ./src/domain/utils/ +COPY src/domain/utils/new_earnings.py ./src/domain/utils/ RUN pip install -r requirements.txt \ No newline at end of file diff --git a/fundamental_data/dags/fetch_fundamental_dag.py b/fundamental_data/dags/fetch_fundamental_dag.py index ef08a0d..6223002 100644 --- a/fundamental_data/dags/fetch_fundamental_dag.py +++ b/fundamental_data/dags/fetch_fundamental_dag.py @@ -12,12 +12,7 @@ from selenium.webdriver.common.keys import Keys from selenium.webdriver.common.by import By from selenium.webdriver.support.select import Select -import io -from pathlib import Path -current_path = Path(os.getcwd()) -project_path = current_path.parent.parent.as_posix() -sys.path.append(project_path) -from src.domain.utils.fetch_data import fetch_fundamental +from src.domain.utils.fetch_data import fetch_fundamental#, fetch_stock fetch_data_dag = DAG( @@ -33,42 +28,12 @@ schedule_interval="0 6 * * 1-5", tags=['tradingbot'], default_args={ - 'start_date': datetime(2023,7,19) + 'start_date': datetime(2023,7,21) } ) tickers = ["AAPL", "TSLA"] -def fetch_fundamental(ticker, historical_days=35): - # Get close data to compute peRatio and DividendsYield - fundamental_data = fetch_stock(ticker, historical_days=historical_days)["close"] - # Get stock dividents & earnings - stock_earnings, stock_dividends = new_earnings.get_earn_and_dividends(ticker, inference=True) - # Combining data with earnings & dividends info - fundamental_data = pd.concat([fundamental_data, stock_earnings], axis=1, join="inner") - fundamental_data = fundamental_data.join(stock_dividends, how = 'left') - # Get stock sector and industry from yahoo finance - stock_metadata = yf.Ticker(ticker).info - # They are not always there - try: - fundamental_data['sector'] = stock_metadata['sector'] - except: - fundamental_data['sector'] = "Industrials" #to check - try: - fundamental_data['industry'] = stock_metadata['industry'] - except: - fundamental_data['industry'] = float("nan") - # # Get VIX Volatility index and join - # vix_df = VIX.get_vix(historical_days=historical_days) - # fundamental_data = fundamental_data.join(vix_df, how = 'left') - # # Get 10Y_bond index and combine data with US Bonds - # us_bond = US_bond_yfinance.get_bonds(historical_days = 35) - # fundamental_data = fundamental_data.join(us_bond, how = 'left') - # Sorting values by date - fundamental_data.sort_values(by = 'date', axis = 0, ascending = True, inplace = True) - - return fundamental_data - def fetch_all_fundamental(): X_fundamental = pd.DataFrame(columns=["date", "stock"]).set_index("date") for ticker in tickers: diff --git a/fundamental_data/docker-compose.yaml b/fundamental_data/docker-compose.yaml index 1d8ca67..c505f76 100644 --- a/fundamental_data/docker-compose.yaml +++ b/fundamental_data/docker-compose.yaml @@ -43,7 +43,9 @@ --- version: "3" x-airflow-common: &airflow-common - build: . + build: + context: ../ + dockerfile: fundamental_data/Dockerfile environment: &airflow-common-env AIRFLOW__CORE__EXECUTOR: CeleryExecutor AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow diff --git a/fundamental_data/fetch_fundamental_dag.py b/fundamental_data/fetch_fundamental_dag.py index ef08a0d..cc7bd34 100644 --- a/fundamental_data/fetch_fundamental_dag.py +++ b/fundamental_data/fetch_fundamental_dag.py @@ -12,12 +12,7 @@ from selenium.webdriver.common.keys import Keys from selenium.webdriver.common.by import By from selenium.webdriver.support.select import Select -import io -from pathlib import Path -current_path = Path(os.getcwd()) -project_path = current_path.parent.parent.as_posix() -sys.path.append(project_path) -from src.domain.utils.fetch_data import fetch_fundamental +from fetch_data import fetch_fundamental fetch_data_dag = DAG( diff --git a/fundamental_data/logs/scheduler/latest b/fundamental_data/logs/scheduler/latest index 6880c93..f3fe96a 120000 --- a/fundamental_data/logs/scheduler/latest +++ b/fundamental_data/logs/scheduler/latest @@ -1 +1 @@ -/opt/airflow/logs/scheduler/2023-07-23 \ No newline at end of file +/opt/airflow/logs/scheduler/2023-07-24 \ No newline at end of file diff --git a/fundamental_data/requirements.txt b/fundamental_data/requirements.txt index c6d4e7da7cbd0543ef4476fc7388bd829c2589f3..c7f370799234749a63550d60ba04cfd40adb3aa7 100644 GIT binary patch literal 45 zcmXR;%}LG6EY0QRDo;&HDatHMEz-?R%u7s9E#l>>Ov}tm%u7zSwKdQ)(las!0DCJB AO#lD@ literal 104 zcmYL>OA3H66vVO?Jc@^^RivmD{N7%Tg18u%%nR>27g`3BNoNJ