Skip to content

Commit

Permalink
Merge pull request #248 from enarjord/v5.1.0_user_stream
Browse files Browse the repository at this point in the history 
V5.1.0 user stream
  • Loading branch information
@enarjord
enarjord authored Oct 10, 2021
2 parents da0f209 + f5f3b7c commit 9603e07
Show file tree
Hide file tree
Showing 27 changed files with 2,735 additions and 3,530 deletions.
24 changes: 8 additions & 16 deletions backtest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,46 +10,39 @@
import pandas as pd

from downloader import Downloader
from njit_funcs import njit_backtest_vanilla, njit_backtest_scalp, round_
from njit_funcs import njit_backtest, round_
from plotting import dump_plots
from procedures import prepare_backtest_config, make_get_filepath, load_live_config, add_argparse_args
from pure_funcs import create_xk, denumpyize, ts_to_date, analyze_fills, spotify_config, determine_config_type
from pure_funcs import create_xk, denumpyize, ts_to_date, analyze_fills, spotify_config


def backtest(config: dict, data: np.ndarray, do_print=False) -> (list, bool):
xk = create_xk(config)
config_type = determine_config_type(config)
if config_type == 'vanilla':
return njit_backtest_vanilla(data, config['starting_balance'], config['latency_simulation_ms'],
config['maker_fee'], **xk)
elif config_type == 'scalp':
return njit_backtest_scalp(data, config['starting_balance'], config['latency_simulation_ms'],
config['maker_fee'], **xk)
else:
raise Exception('unknown config type')
return njit_backtest(data, config['starting_balance'], config['latency_simulation_ms'],
config['maker_fee'], **xk)


def plot_wrap(config, data):
print('n_days', round_(config['n_days'], 0.1))
print('starting_balance', config['starting_balance'])
print('backtesting...')
sts = time()
fills, info, stats = backtest(config, data, do_print=True)
fills, stats = backtest(config, data, do_print=True)
print(f'{time() - sts:.2f} seconds elapsed')
if not fills:
print('no fills')
return
fdf, result = analyze_fills(fills, {**config, **{'lowest_eqbal_ratio': info[1], 'closest_bkr': info[2]}},
data[0][0], data[-1][0])
fdf, sdf, result = analyze_fills(fills, stats, config)
config['result'] = result
config['plots_dirpath'] = make_get_filepath(os.path.join(
config['plots_dirpath'], f"{ts_to_date(time())[:19].replace(':', '')}", '')
)
fdf.to_csv(config['plots_dirpath'] + "fills.csv")
sdf.to_csv(config['plots_dirpath'] + "stats.csv")
df = pd.DataFrame({**{'timestamp': data[:, 0], 'qty': data[:, 1], 'price': data[:, 2]},
**{}})
print('dumping plots...')
dump_plots(config, fdf, df)
dump_plots(config, fdf, sdf, df)


async def main():
Expand All @@ -60,7 +53,6 @@ async def main():

config = await prepare_backtest_config(args)
live_config = load_live_config(args.live_config_path)
config_type = config['config_type'] = determine_config_type(live_config)
config.update(live_config)
if 'spot' in config['market_type']:
live_config = spotify_config(live_config)
Expand Down
182 changes: 182 additions & 0 deletions balance_overview.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from procedures import create_binance_bot, create_binance_bot_spot, create_bybit_bot, load_exchange_key_secret\n",
"from pure_funcs import get_template_live_config, ts_to_date, get_daily_from_income\n",
"from njit_funcs import round_dynamic\n",
"from time import time\n",
"import pandas as pd\n",
"import matplotlib.pyplot as plt"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"plt.rcParams['figure.figsize'] = [21, 13]\n",
"pd.set_option('precision', 10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"config = get_template_live_config()\n",
"config['market_type'] = 'futures'\n",
"config['user'] = 'user_name'\n",
"config['exchange'], _, _ = load_exchange_key_secret(config['user'])\n",
"\n",
"n_days = 30\n",
"start_time = (time() - 60 * 60 * 24 * n_days) * 1000\n",
"end_time = time() * 1000\n",
"\n",
"symbols = ['XRPUSDT', 'XLMUSDT', 'ADAUSDT', 'EOSUSDT', 'BNBUSDT', 'LINKUSDT', 'COMPUSDT', 'FILUSDT']\n",
"\n",
"incomes = []\n",
"# binance fetches income for all symbols; bybit only one symbol at a time\n",
"for symbol in (symbols if config['exchange'] == 'bybit' else symbols[:1]):\n",
" config['symbol'] = symbol\n",
" try:\n",
" await bot.session.close()\n",
" except:\n",
" pass\n",
" if config['exchange'] == 'binance':\n",
" if config['market_type'] == 'spot':\n",
" bot = await create_binance_bot_spot(config)\n",
" else:\n",
" bot = await create_binance_bot(config)\n",
" elif config['exchange'] == 'bybit':\n",
" bot = await create_bybit_bot(config)\n",
"\n",
" await bot.update_position()\n",
" balance = bot.position['wallet_balance']\n",
"\n",
" income = await bot.get_all_income(start_time=start_time)\n",
" incomes += income\n",
"idf, bdf = get_daily_from_income(sorted(incomes, key=lambda x: x['timestamp']), bot.position['wallet_balance'], start_time=start_time, end_time=end_time)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(f'current balance {round_dynamic(balance, 5)}')\n",
"print(f'abs sum {round_dynamic(idf.income.sum(), 4)} {idf.iloc[-1][\"token\"]}')\n",
"print(f'abs adg {round_dynamic(idf.income.sum() / n_days, 4)} {idf.iloc[-1][\"token\"]}')\n",
"print(f'pct sum {((balance + idf.income.sum()) / balance) - 1:.5f}')\n",
"print(f'pct adg {((balance + idf.income.sum()) / balance) ** (1 / n_days) - 1:.5f}')\n",
"\n",
"idf.income.cumsum().plot()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"idf.tail(20)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"bdf"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"bdf.gain.plot()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# check income last x days\n",
"x = 3\n",
"since = (time() - 60 * 60 * 24 * x) * 1000\n",
"idf[idf.timestamp > since].groupby('symbol').income.sum().sort_values()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
53 changes: 33 additions & 20 deletions batch_optimize.py
View file
Original file line number Diff line number Diff line change
@@ -1,31 +1,44 @@
import subprocess

import os
import shutil
from procedures import make_get_filepath

def main():
kwargs_list = [
{'optimize_config_path': 'configs/optimize/scalp.hjson',
'symbol': 'XMRUSDT',
'starting_balance': 1000.0},
{'optimize_config_path': 'configs/optimize/scalp.hjson',
'symbol': 'BTCUSD_PERP',
'starting_balance': 0.1},
{'optimize_config_path': 'configs/optimize/vanilla.hjson',
'symbol': 'ADABTC',
'starting_balance': 0.1,
'market_type': 'spot'},
{'optimize_config_path': 'configs/optimize/scalp.hjson',
'user': 'ebybit',
'symbol': 'EOSUSD',
'starting_balance': 100.0},
]
tokens = [
'BTS', 'LTC', 'STORJ', 'BAT', 'DASH', 'SOL', 'AVAX', 'LUNA', 'DYDX', 'COMP',
'FIL', 'LINK', 'MATIC', 'LIT', 'NEO', 'OMG', 'XRP', 'HBAR', 'MANA', 'IOTA',
'ADA', 'QTUM', 'SXP', 'XEM', 'EOS', 'XMR', 'ETC', 'XLM', 'MKR', 'BNB',
'AAVE', 'ALGO', 'TRX', 'ZEC','XTZ', 'BCH']
start_from = 'BTS'
symbols = tokens[tokens.index(start_from):] + tokens[:tokens.index(start_from)]

quote = 'USDT'
cfgs_dir = make_get_filepath('cfgs_batch_optimize/')
exchange = 'binance'

symbols = [e + quote for e in symbols]
kwargs_list = [{
'start': cfgs_dir,
'symbol': symbol,
#'starting_balance': 10000.0,
#'end_date': '2021-09-20T15:00',
#'start_date': '2021-03-01',
} for symbol in symbols]
for kwargs in kwargs_list:
formatted = f"python3 optimize.py {kwargs['optimize_config_path']}"
for key in [k for k in kwargs if k != 'optimize_config_path']:
formatted = f"python3 optimize.py "
for key in kwargs:
formatted += f' --{key} {kwargs[key]}'
print(formatted)
subprocess.run([formatted], shell=True)
try:
d = f'backtests/{exchange}/{kwargs["symbol"]}/plots/'
ds = sorted([f for f in os.listdir(d) if '20' in f])
for d1 in ds:
print(f'copying resulting config to {cfgs_dir}', d + d1)
shutil.copy(d + d1 + '/live_config.json', f'{cfgs_dir}{kwargs["symbol"]}_{d1}.json')
except Exception as e:
print('error', kwargs['symbol'], e)


if __name__ == '__main__':
main()
main()
Loading

0 comments on commit 9603e07

Please sign in to comment.