merge from develop. Better import of split prices

robcarver17 · Oct 23, 2024 · aa9d479 · aa9d479
1 parent 7a9cfe4
commit aa9d479
Showing 1 changed file with 58 additions and 11 deletions.
diff --git a/sysinit/futures/contract_prices_from_split_freq_csv_to_db.py b/sysinit/futures/contract_prices_from_split_freq_csv_to_db.py
@@ -2,9 +2,9 @@
 from syscore.dateutils import MIXED_FREQ, HOURLY_FREQ, DAILY_PRICE_FREQ
 from syscore.pandas.frequency import merge_data_with_different_freq
 from sysdata.csv.csv_futures_contract_prices import csvFuturesContractPriceData
-from sysproduction.data.prices import diagPrices
 from sysobjects.contracts import futuresContract
 from sysobjects.futures_per_contract_prices import futuresContractPrices
+from sysproduction.data.prices import diagPrices
 
 diag_prices = diagPrices()
 db_prices = diag_prices.db_futures_contract_price_data
@@ -13,6 +13,7 @@
 def init_db_with_split_freq_csv_prices(
     datapath: str,
     csv_config=arg_not_supplied,
+    ignore_duplication: bool = True,  # if True, we overwrite existing prices
 ):
     csv_prices = csvFuturesContractPriceData(datapath)
     input(
@@ -24,14 +25,18 @@ def init_db_with_split_freq_csv_prices(
     instrument_codes.sort()
     for instrument_code in instrument_codes:
         init_db_with_split_freq_csv_prices_for_code(
-            instrument_code, datapath, csv_config=csv_config
+            instrument_code,
+            datapath,
+            csv_config=csv_config,
+            ignore_duplication=ignore_duplication,
         )
 
 
 def init_db_with_split_freq_csv_prices_for_code(
     instrument_code: str,
     datapath: str,
     csv_config=arg_not_supplied,
+    ignore_duplication: bool = True,
 ):
     same_length = []
     too_short = []
@@ -64,13 +69,19 @@ def init_db_with_split_freq_csv_prices_for_code(
         print(f"Contract object is {str(contract)}")
 
         hourly = hourly_dict[contract_date_str]
-        write_prices_for_contract_at_frequency(contract, hourly, HOURLY_FREQ)
+        write_prices_for_contract_at_frequency(
+            contract, hourly, HOURLY_FREQ, ignore_duplication=ignore_duplication
+        )
 
         daily = daily_dict[contract_date_str]
-        write_prices_for_contract_at_frequency(contract, daily, DAILY_PRICE_FREQ)
+        write_prices_for_contract_at_frequency(
+            contract, daily, DAILY_PRICE_FREQ, ignore_duplication=ignore_duplication
+        )
 
         merged = futuresContractPrices(merge_data_with_different_freq([hourly, daily]))
-        write_prices_for_contract_at_frequency(contract, merged, MIXED_FREQ)
+        write_prices_for_contract_at_frequency(
+            contract, merged, MIXED_FREQ, ignore_duplication=ignore_duplication
+        )
 
         if len(hourly) == len(daily):
             same_length.append(contract_date_str)
@@ -83,8 +94,25 @@ def init_db_with_split_freq_csv_prices_for_code(
         print(f"Contract object is {str(contract)}")
 
         daily = daily_dict[contract_date_str]
-        write_prices_for_contract_at_frequency(contract, daily, DAILY_PRICE_FREQ)
-        write_prices_for_contract_at_frequency(contract, daily, MIXED_FREQ)
+        write_prices_for_contract_at_frequency(
+            contract, daily, DAILY_PRICE_FREQ, ignore_duplication=ignore_duplication
+        )
+
+        # if we already have hourly data in the db, get it and merge with daily
+        if db_prices.has_price_data_for_contract_at_frequency(
+            contract, frequency=HOURLY_FREQ
+        ):
+            hourly = db_prices.get_prices_at_frequency_for_contract_object(
+                contract, frequency=HOURLY_FREQ
+            )
+            merged = futuresContractPrices(
+                merge_data_with_different_freq([hourly, daily])
+            )
+        else:
+            merged = daily
+        write_prices_for_contract_at_frequency(
+            contract, merged, MIXED_FREQ, ignore_duplication=ignore_duplication
+        )
 
         if len(daily) < 65:
             too_short.append(contract_date_str)
@@ -97,21 +125,40 @@ def init_db_with_split_freq_csv_prices_for_code(
         print(f"Contract object is {str(contract)}")
 
         hourly = hourly_dict[contract_date_str]
-        write_prices_for_contract_at_frequency(contract, hourly, HOURLY_FREQ)
-        write_prices_for_contract_at_frequency(contract, hourly, MIXED_FREQ)
+        write_prices_for_contract_at_frequency(
+            contract, hourly, HOURLY_FREQ, ignore_duplication=ignore_duplication
+        )
+
+        # if we already have daily data in the db, get it and merge with hourly
+        if db_prices.has_price_data_for_contract_at_frequency(
+            contract, frequency=DAILY_PRICE_FREQ
+        ):
+            daily = db_prices.get_prices_at_frequency_for_contract_object(
+                contract, frequency=DAILY_PRICE_FREQ
+            )
+            merged = futuresContractPrices(
+                merge_data_with_different_freq([hourly, daily])
+            )
+        else:
+            merged = hourly
+        write_prices_for_contract_at_frequency(
+            contract, merged, MIXED_FREQ, ignore_duplication=ignore_duplication
+        )
 
     print(f"These contracts have the same length for daily and hourly: {same_length}")
     print(f"These daily contracts are short: {too_short}")
 
 
-def write_prices_for_contract_at_frequency(contract, prices, frequency):
+def write_prices_for_contract_at_frequency(
+    contract, prices, frequency, ignore_duplication=False
+):
     print(f"{frequency} .csv prices are \n{str(prices)}")
     print("Writing to db")
     db_prices.write_prices_at_frequency_for_contract_object(
         contract,
         prices,
-        ignore_duplication=True,
         frequency=frequency,
+        ignore_duplication=ignore_duplication,
     )
     print("Reading back prices from db to check")
     written_prices = db_prices.get_prices_at_frequency_for_contract_object(