blacked

examples/production/example_of_custom_run_system.py

@@ -66,8 +66,8 @@ def production_carry_trend_dynamic_system(
 from systems.forecasting import Rules
 from systems.basesystem import System
 from systems.forecast_combine import ForecastCombine
-from private.systems.carrytrend.forecastScaleCap import volAttenForecastScaleCap
-from private.systems.carrytrend.rawdata import myFuturesRawData
+from private.systems.arch.carrytrend.forecastScaleCap import volAttenForecastScaleCap
+from private.systems.arch.carrytrend import myFuturesRawData
 from systems.positionsizing import PositionSizing
 from systems.portfolio import Portfolios
 from systems.provided.dynamic_small_system_optimise.optimised_positions_stage import (

syscore/exceptions.py

@@ -14,9 +14,11 @@ class missingContract(Exception):
 class missingData(Exception):
     pass
 
+
 class missingFile(Exception):
     pass
 
+
 class existingData(Exception):
     pass
 

syscore/pandas/frequency.py

@@ -226,7 +226,7 @@ def reindex_last_monthly_include_first_date(df: pd.DataFrame) -> pd.DataFrame:
 def infer_frequency(df_or_ts: Union[pd.DataFrame, pd.Series]) -> Frequency:
     inferred = pd.infer_freq(df_or_ts.index)
     if inferred is None:
-        return _infer_frequency_approx(df_or_ts)
+        return infer_frequency_approx(df_or_ts)
     if inferred == "B":
         return BUSINESS_DAY_FREQ
     if inferred == "H":
@@ -239,9 +239,8 @@ def infer_frequency(df_or_ts: Union[pd.DataFrame, pd.Series]) -> Frequency:
 BUSINESS_CALENDAR_FRACTION = CALENDAR_DAYS_IN_YEAR / BUSINESS_DAYS_IN_YEAR
 
 
-def _infer_frequency_approx(df_or_ts: Union[pd.DataFrame, pd.Series]) -> Frequency:
-    avg_time_delta = abs(np.diff(df_or_ts.index)).mean()
-    avg_time_delta_in_days = avg_time_delta / np.timedelta64(1, "D")
+def infer_frequency_approx(df_or_ts: Union[pd.DataFrame, pd.Series]) -> Frequency:
+    avg_time_delta_in_days = average_time_delta_for_time_series(df_or_ts)
 
     if _probably_daily_freq(avg_time_delta_in_days):
         return BUSINESS_DAY_FREQ
@@ -260,3 +259,12 @@ def _probably_hourly_freq(avg_time_delta_in_days: float) -> bool:
     return (avg_time_delta_in_days < UPPER_BOUND_HOUR_FRACTION_OF_A_DAY) & (
         avg_time_delta_in_days >= LOWER_BOUND_HOUR_FRACTION_OF_A_DAY
     )
+
+
+def average_time_delta_for_time_series(
+    df_or_ts: Union[pd.DataFrame, pd.Series]
+) -> float:
+    avg_time_delta = abs(np.diff(df_or_ts.index)).mean()
+    avg_time_delta_in_days = avg_time_delta / np.timedelta64(1, "D")
+
+    return avg_time_delta_in_days

syscore/pandas/pdutils.py

@@ -293,8 +293,9 @@ def make_df_from_list_of_named_tuple(
     if make_index:
         if field_name_for_index is arg_not_supplied:
             field_name_for_index = elements[0]
-        pdf.index = pdf[field_name_for_index]
+        pdf_index = list(pdf[field_name_for_index])
         pdf = pdf.drop(labels=field_name_for_index, axis=1)
+        pdf.index = pdf_index
 
     return pdf
 

systems/accounts/order_simulator/account_curve_order_simulator.py

@@ -69,6 +69,7 @@ def _pandl_calculator_for_subsystem_with_cash_costs(
             roundpositions=roundpositions,
             vol_normalise_currency_costs=vol_normalise_currency_costs,
             rolls_per_year=rolls_per_year,
+            passed_diagnostic_df=order_simulator.diagnostic_df(),
         )
 
         return pandl_calculator

systems/accounts/order_simulator/pandl_order_simulator.py

@@ -42,27 +42,41 @@ def diagnostic_df(self) -> pd.DataFrame:
         return self.cache.get(self._diagnostic_df)
 
     def _diagnostic_df(self) -> pd.DataFrame:
+        other_df = self._other_diagnostics()
+        orders_and_fills_df = self._orders_and_fills_df()
+        diagnostic_df = pd.concat([other_df, orders_and_fills_df], axis=1)
+
+        return diagnostic_df
+
+    def _other_diagnostics(self) -> pd.DataFrame:
         position_series = self.positions()
         position_df = pd.DataFrame(position_series)
 
         optimal_positions_series = self.optimal_positions_series()
         optimal_position_df = pd.DataFrame(optimal_positions_series)
 
+        df_positions = pd.concat([optimal_position_df, position_df], axis=1)
+        df_positions.columns = [
+            "optimal_position",
+            "position",
+        ]
+
+        return df_positions
+
+    def _orders_and_fills_df(self) -> pd.DataFrame:
         list_of_fills = self.list_of_fills()
         fills_df = list_of_fills.as_pd_df()
         list_of_orders = self.list_of_orders()
         orders_df = list_of_orders.as_pd_df()
-        df = pd.concat([optimal_position_df, orders_df, fills_df, position_df], axis=1)
-        df.columns = [
-            "optimal_position",
+        orders_and_fills_df = pd.concat([orders_df, fills_df], axis=1)
+        orders_and_fills_df.columns = [
             "order_qty",
             "limit_price",
             "fill_qty",
             "fill_price",
-            "position",
         ]
 
-        return df
+        return orders_and_fills_df
 
     def prices(self) -> pd.Series:
         return self.series_data.price_series

systems/accounts/order_simulator/simple_orders.py

@@ -84,7 +84,9 @@ def __init__(self, list_of_orders: List[SimpleOrderWithDate]):
 
     def as_pd_df(self):
         return make_df_from_list_of_named_tuple(
-            _SimpleOrderWithDateAsTuple, self._as_list_of_named_tuples()
+            _SimpleOrderWithDateAsTuple,
+            self._as_list_of_named_tuples(),
+            field_name_for_index="submit_date",
         )
 
     def _as_list_of_named_tuples(self) -> list:

systems/accounts/pandl_calculators/pandl_calculation.py

@@ -16,17 +16,30 @@ def __init__(
         value_per_point: float = 1.0,
         roundpositions=False,
         delayfill=False,
+        passed_diagnostic_df: pd.DataFrame = arg_not_supplied,
     ):
 
         self._price = price
         self._positions = positions
         self._fx = fx
         self._capital = capital
         self._value_per_point = value_per_point
-
+        self._passed_diagnostic_df = passed_diagnostic_df
         self._delayfill = delayfill
         self._roundpositions = roundpositions
 
+    def calculations_and_diagnostic_df(self) -> pd.DataFrame:
+        diagnostic_df = self.passed_diagnostic_df
+        calculations = self.calculations_df()
+        calculations_and_diagnostic_df = pd.concat(
+            [diagnostic_df, calculations], axis=1
+        )  ## no ffill
+
+        return calculations_and_diagnostic_df
+
+    def calculations_df(self) -> pd.Series:
+        raise NotImplemented("Not implemented")
+
     def weight(self, weight: pd.Series):
 
         weighted_capital = apply_weighting(weight, self.capital)
@@ -162,6 +175,18 @@ def roundpositions(self) -> bool:
     def value_per_point(self) -> float:
         return self._value_per_point
 
+    @property
+    def passed_diagnostic_df(self) -> pd.DataFrame:
+        diagnostic_df = self._passed_diagnostic_df
+        if diagnostic_df is arg_not_supplied:
+            return self._default_diagnostic_df
+        return diagnostic_df
+
+    @property
+    def _default_diagnostic_df(self) -> pd.DataFrame:
+        diagnostic_df = pd.DataFrame(dict(price=self.price))
+        return diagnostic_df
+
     @property
     def fx(self) -> pd.Series:
         fx = self._fx

systems/accounts/pandl_calculators/pandl_cash_costs.py

@@ -37,6 +37,17 @@ def __init__(
         self._rolls_per_year = rolls_per_year
         self._multiply_roll_costs_by = multiply_roll_costs_by
 
+    def calculations_df(self):
+        #### TEMPORARY
+        costs = self.costs_from_trading_in_instrument_currency_as_series()
+        pandl = self.pandl_in_instrument_currency()
+        net = self.net_pandl_in_instrument_currency()
+
+        calculations_df = pd.concat([pandl, costs, net], axis=1)
+        calculations_df.index = ["gross", "costs", "net"]
+
+        return calculations_df
+
     def costs_pandl_in_points(self) -> pd.Series:
         ## We work backwards since the cost calculator returns a currency cost
         costs_pandl_in_instrument_currency = self.costs_pandl_in_instrument_currency()