Merge remote-tracking branch 'robcarver17/develop' into missing-data-6

.github/workflows/slow-test.yml → .github/workflows/slow-test-develop.yml

@@ -17,6 +17,8 @@ jobs:
     steps:
 
     - uses: actions/checkout@v3
+      with:
+        ref: "develop"
 
     - name: Set up Python
       uses: actions/setup-python@v4

.github/workflows/slow-test-master.yml

@@ -0,0 +1,40 @@
+name: Slow test
+
+on:
+  schedule:
+    - cron: '30 1 * * 0'
+
+  workflow_dispatch:
+
+jobs:
+  build:
+
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: [ 3.8, 3.7 ]
+
+    steps:
+
+    - uses: actions/checkout@v3
+      with:
+        ref: "master"
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install pip setuptools wheel
+        pip install pytest
+        pip install -r requirements.txt
+
+    - name: Test with pytest
+      run: |
+        pytest --runslow  --disable-warnings
+
+    #- name: Coverage report
+    #  run: |
+    #    coverage html

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Release notes
 
+## Version 1.62
+
+- Added order simulator as an optimal replacement for vectorised p&l calculation; prequisite for limit order simulation
+- Replace pst logging with python logging
+- ignore daily expiries for certain EUREX contracts
+- Allow fixed instrument and forecast weights to be specificed as a hierarchy
+
 ## Version 1.61
 
 - Replaced log to database with log to file

README.md

@@ -7,10 +7,10 @@ Rob Carver
 [https://qoppac.blogspot.com/p/pysystemtrade.html](https://qoppac.blogspot.com/p/pysystemtrade.html)
 
 
-Version 1.61
+Version 1.62
 
 
-2023-03-24
+2023-06-09
 
 
 

docs/backtesting.md

@@ -4399,7 +4399,7 @@ Other methods exist to access logging and caching.
 | `positionSize.get_block_value` | Standard | `instrument_code` | D | Get value of a 1% move in the price |
 | `positionSize.get_instrument_currency_vol` | Standard | `instrument_code` |D | Get daily volatility in the currency of the instrument |
 | `positionSize.get_instrument_value_vol` | Standard | `instrument_code` |D | Get daily volatility in the currency of the trading account |
-| `positionSize.get_volatility_scalar` | Standard | `instrument_code` | D |Get ratio of target volatility vs volatility of instrument in instrument's own currency |
+| `positionSize.get_average_position_at_subsystem_level` | Standard | `instrument_code` | D |Get ratio of target volatility vs volatility of instrument in instrument's own currency |
 | `positionSize.get_subsystem_position`| Standard | `instrument_code` | D, O |Get position if we put our entire trading capital into one instrument |
 
 

examples/introduction/simplesystem.py

@@ -150,7 +150,7 @@
 print(my_system.positionSize.get_block_value("SOFR").tail(5))
 print(my_system.positionSize.get_underlying_price("SOFR"))
 print(my_system.positionSize.get_instrument_value_vol("SOFR").tail(5))
-print(my_system.positionSize.get_volatility_scalar("SOFR").tail(5))
+print(my_system.positionSize.get_average_position_at_subsystem_level("SOFR").tail(5))
 print(my_system.positionSize.get_vol_target_dict())
 print(my_system.positionSize.get_subsystem_position("SOFR").tail(5))
 

syscore/genutils.py

@@ -132,6 +132,10 @@ def str_of_int(x: int) -> str:
         return ""
 
 
+def same_sign(x, y):
+    return sign(x) == sign(y)
+
+
 def sign(x: Union[int, float]) -> float:
     """
     >>> sign(3)

sysdata/production/historic_orders.py

@@ -19,7 +19,7 @@
 from sysexecution.orders.named_order_objects import missing_order
 
 from sysdata.base_data import baseData
-from sysobjects.fills import listOfFills, fill_from_order
+from sysobjects.fills import ListOfFills, fill_from_order
 from sysexecution.orders.base_orders import Order
 from sysexecution.orders.broker_orders import single_fill_from_broker_order
 from sysexecution.order_stacks.order_stack import missingOrder
@@ -71,7 +71,7 @@ def get_list_of_order_ids_in_date_range(
 class strategyHistoricOrdersData(genericOrdersData):
     def get_fills_history_for_instrument_strategy(
         self, instrument_strategy: instrumentStrategy
-    ) -> listOfFills:
+    ) -> ListOfFills:
         """
 
         :param instrument_code:  str
@@ -82,7 +82,7 @@ def get_fills_history_for_instrument_strategy(
             instrument_strategy
         )
         order_list_as_fills = [fill_from_order(order) for order in order_list]
-        list_of_fills = listOfFills(order_list_as_fills)
+        list_of_fills = ListOfFills(order_list_as_fills)
 
         return list_of_fills
 
@@ -115,7 +115,7 @@ class contractHistoricOrdersData(genericOrdersData):
 class brokerHistoricOrdersData(contractHistoricOrdersData):
     def get_fills_history_for_contract(
         self, futures_contract: futuresContract
-    ) -> listOfFills:
+    ) -> ListOfFills:
         """
 
         :param instrument_code:  str
@@ -133,7 +133,7 @@ def get_fills_history_for_contract(
             for orderid in list_of_order_ids
         ]
         list_of_fills = [fill for fill in list_of_fills if fill is not missing_order]
-        list_of_fills = listOfFills(list_of_fills)
+        list_of_fills = ListOfFills(list_of_fills)
 
         return list_of_fills
 

sysexecution/orders/named_order_objects.py

@@ -1,5 +1,6 @@
 from syscore.constants import named_object
 
+
 missing_order = named_object("missing order")
 locked_order = named_object("locked order")
 duplicate_order = named_object("duplicate order")

sysobjects/fills.py

@@ -1,23 +1,47 @@
+from typing import Union
 import datetime
-from collections import namedtuple
+from dataclasses import dataclass
 
 import pandas as pd
+import numpy as np
 
-from syscore.constants import named_object
-from sysexecution.orders.named_order_objects import missing_order
-from sysobjects.orders import SimpleOrder, ListOfSimpleOrders
+from sysexecution.orders.named_order_objects import missing_order, named_object
 
-from sysexecution.orders.list_of_orders import listOfOrders
 from sysexecution.orders.base_orders import Order
 
-Fill = namedtuple("Fill", ["date", "qty", "price"])
 
-NOT_FILLED = named_object("not filled")
+@dataclass
+class Fill:
+    date: datetime.datetime
+    qty: int
+    price: float
+    price_requires_slippage_adjustment: bool = False
+
+    @classmethod
+    def zero_fill(cls, date):
+        return cls(date=date, qty=0, price=np.nan)
+
+    @property
+    def is_unfilled(self) -> bool:
+        return self.qty == 0
+
+
+def is_empty_fill(fill: Union[named_object, Fill]) -> bool:
+    if fill is missing_order:
+        return True
+    if fill.is_unfilled:
+        return True
+
+    return False
 
 
-class listOfFills(list):
+def empty_fill(date: datetime.datetime) -> Fill:
+    return Fill.zero_fill(date)
+
+
+class ListOfFills(list):
     def __init__(self, list_of_fills):
-        list_of_fills = [fill for fill in list_of_fills if fill is not missing_order]
+        list_of_fills = [fill for fill in list_of_fills if not is_empty_fill(fill)]
         super().__init__(list_of_fills)
 
     def _as_dict_of_lists(self) -> dict:
@@ -47,7 +71,7 @@ def from_position_series_and_prices(cls, positions: pd.Series, price: pd.Series)
 
 def _list_of_fills_from_position_series_and_prices(
     positions: pd.Series, price: pd.Series
-) -> listOfFills:
+) -> ListOfFills:
 
     (
         trades_without_zeros,
@@ -59,11 +83,11 @@ def _list_of_fills_from_position_series_and_prices(
     dates_as_list = list(prices_aligned_to_trades.index)
 
     list_of_fills_as_list = [
-        Fill(date, qty, price)
+        Fill(date, qty, price, price_requires_slippage_adjustment=True)
         for date, qty, price in zip(dates_as_list, trades_as_list, prices_as_list)
     ]
 
-    list_of_fills = listOfFills(list_of_fills_as_list)
+    list_of_fills = ListOfFills(list_of_fills_as_list)
 
     return list_of_fills
 
@@ -101,59 +125,3 @@ def fill_from_order(order: Order) -> Fill:
         return missing_order
 
     return Fill(fill_datetime, fill_qty, fill_price)
-
-
-def fill_from_simple_order(
-    simple_order: SimpleOrder,
-    market_price: float,
-    fill_datetime: datetime.datetime,
-    slippage: float = 0,
-) -> Fill:
-    if simple_order.is_zero_order:
-        return NOT_FILLED
-
-    elif simple_order.is_market_order:
-        fill = fill_from_simple_market_order(
-            simple_order,
-            market_price=market_price,
-            slippage=slippage,
-            fill_datetime=fill_datetime,
-        )
-    else:
-        ## limit order
-        fill = fill_from_simple_limit_order(
-            simple_order, market_price=market_price, fill_datetime=fill_datetime
-        )
-
-    return fill
-
-
-def fill_from_simple_limit_order(
-    simple_order: SimpleOrder, market_price: float, fill_datetime: datetime.datetime
-) -> Fill:
-
-    limit_price = simple_order.limit_price
-    if simple_order.quantity > 0:
-        if limit_price > market_price:
-            return Fill(fill_datetime, simple_order.quantity, limit_price)
-
-    if simple_order.quantity < 0:
-        if limit_price < market_price:
-            return Fill(fill_datetime, simple_order.quantity, limit_price)
-
-    return NOT_FILLED
-
-
-def fill_from_simple_market_order(
-    simple_order: SimpleOrder,
-    market_price: float,
-    fill_datetime: datetime.datetime,
-    slippage: float = 0,
-) -> Fill:
-
-    if simple_order.quantity > 0:
-        fill_price_with_slippage = market_price + slippage
-    else:
-        fill_price_with_slippage = market_price - slippage
-
-    return Fill(fill_datetime, simple_order.quantity, fill_price_with_slippage)

sysobjects/instruments.py

@@ -328,13 +328,20 @@ def calculate_cost_percentage_terms(
         return cost_in_percentage_terms
 
     def calculate_cost_instrument_currency(
-        self, blocks_traded: float, block_price_multiplier: float, price: float
+        self,
+        blocks_traded: float,
+        block_price_multiplier: float,
+        price: float,
+        include_slippage: bool = True,
     ) -> float:
 
         value_per_block = price * block_price_multiplier
-        slippage = self.calculate_slippage_instrument_currency(
-            blocks_traded, block_price_multiplier=block_price_multiplier
-        )
+        if include_slippage:
+            slippage = self.calculate_slippage_instrument_currency(
+                blocks_traded, block_price_multiplier=block_price_multiplier
+            )
+        else:
+            slippage = 0
 
         commission = self.calculate_total_commission(
             blocks_traded, value_per_block=value_per_block

sysproduction/data/orders.py

@@ -19,7 +19,7 @@
 )
 from sysdata.data_blob import dataBlob
 
-from sysobjects.fills import listOfFills
+from sysobjects.fills import ListOfFills
 from sysexecution.order_stacks.broker_order_stack import brokerOrderStackData
 from sysexecution.order_stacks.contract_order_stack import contractOrderStackData
 from sysexecution.order_stacks.instrument_order_stack import instrumentOrderStackData
@@ -160,7 +160,7 @@ def get_historic_broker_order_from_order_id(self, order_id: int) -> brokerOrder:
 
     def get_fills_history_for_contract(
         self, futures_contract: futuresContract
-    ) -> listOfFills:
+    ) -> ListOfFills:
         ## We get this from broker fills, as they have leg by leg information
         list_of_fills = (
             self.db_broker_historic_orders_data.get_fills_history_for_contract(
@@ -172,7 +172,7 @@ def get_fills_history_for_contract(
 
     def get_fills_history_for_instrument_strategy(
         self, instrument_strategy: instrumentStrategy
-    ) -> listOfFills:
+    ) -> ListOfFills:
         list_of_fills = self.db_strategy_historic_orders_data.get_fills_history_for_instrument_strategy(
             instrument_strategy
         )

sysproduction/strategy_code/report_system_classic.py

@@ -304,7 +304,13 @@ def get_forecast_matrix_over_code(
 )
 
 get_vol_scalar = configForMethod(
-    "positionSize", "get_volatility_scalar", "Vol Scalar", False, True, None, False
+    "positionSize",
+    "get_average_position_at_subsystem_level",
+    "Vol Scalar",
+    False,
+    True,
+    None,
+    False,
 )
 
 get_subsystem_position = configForMethod(