diff --git a/docker-compose.payments.yml b/docker-compose.payments.yml index c49e419ce..10ad494be 100644 --- a/docker-compose.payments.yml +++ b/docker-compose.payments.yml @@ -3,8 +3,8 @@ services: image: redis:7.0 restart: always #ports: - # - '6379:6379' - command: redis-server --save 20 1 --loglevel verbose --requirepass testpass --user redis + # - '6380:6380' + command: redis-server --save 20 1 --loglevel verbose --requirepass testpass --user redis --port 6380 #volumes: # - redis-cache:/data @@ -17,7 +17,7 @@ services: localstack: container_name: localstack image: localstack/localstack - stop_grace_period: 1s + stop_grace_period: "1s" ports: - "127.0.0.1:4566:4566" # LocalStack Gateway - "127.0.0.1:4510-4559:4510-4559" # external services port range @@ -31,10 +31,10 @@ services: worker: image: bat-go-local - command: /build/bat-go serve payments worker depends_on: - redis - localstack + command: /build/bat-go serve payments worker environment: - NITRO_ENCLAVE_MOCKING=1 - REDIS_ADDR=redis:6379 @@ -46,9 +46,12 @@ services: service: image: bat-go-local - command: /build/bat-go serve nitro inside-enclave --egress-address none --log-address none --upstream-url http://0.0.0.0:8080 + depends_on: + - redis + - localstack volumes: - ./payments-test-secretes.json:/etc/bat-test-secretes.json:ro + command: /build/bat-go serve nitro inside-enclave --egress-address none --log-address none --upstream-url http://0.0.0.0:8080 environment: - NITRO_ENCLAVE_MOCKING=1 - DEBUG=1 @@ -58,7 +61,7 @@ services: - QLDB_LEDGER_ARN=arn:aws:qldb:us-west-2:239563960694:ledger/testing-igor-payment - QLDB_LEDGER_NAME=testing-igor-payment - QLDB_ROLE_ARN=arn:aws:iam::239563960694:role/settlements-dev-sso-qldb-20240628161845683100000001 - - AWS_REGION=${AWS_REGION-} + - AWS_REGION=${AWS_REGION-us-west-2} - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID-} - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY-} - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN-} @@ -69,6 +72,30 @@ services: - "127.0.0.1:18080:18080" - "127.0.0.1:18443:18443" + shell: + image: bat-go-local + depends_on: + - redis + - localstack + # The shell uses localhost + network_mode: "service:redis" + command: ipython3 --profile-dir=ipython-profile + working_dir: /build/src/tools/payments + stdin_open: true + tty: true + environment: + - NITRO_ENCLAVE_MOCKING=1 + - DEBUG=1 + - REDIS_ADDR=redis:6379 + - REDIS_USERNAME=default + - REDIS_PASSWORD=testpass + - AWS_REGION=${AWS_REGION-us-west-2} + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID-} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY-} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN-} + - AWS_CONTAINER_CREDENTIALS_FULL_URI=${AWS_CONTAINER_CREDENTIALS_FULL_URI-} + - AWS_CONTAINER_AUTHORIZATION_TOKEN=${AWS_CONTAINER_AUTHORIZATION_TOKEN-} + volumes: redis-cache: driver: local diff --git a/local-dev/local.dockerfile b/local-dev/local.dockerfile index cfba5c274..76f7299fb 100644 --- a/local-dev/local.dockerfile +++ b/local-dev/local.dockerfile @@ -6,7 +6,9 @@ ARG GOLANG_VERSION=1.22.4 RUN apt-get update \ && apt-get install -y -qq \ tmux curl man less \ - python3 git make + python3 ipython3 python-is-python3 \ + git make \ + redis-tools # Install Go RUN set -x && curl -L -o /var/tmp/go.tgz \ @@ -44,9 +46,20 @@ COPY --link --from=sources --chown=user:user /build/mod-files/ /build/src/ RUN cd /build/src/main && go mod download -x +# Download all modules for the payment commands. To make things simpler -execdir +# is used as it cd to the directory and properly fails if the command fails, but +# then the argument should be ignored and a shell insirection is used for that. +RUN find /build/src/tools/payments/cmd -type f -name go.mod \ + -print -execdir /bin/sh -c 'go mod download -x' '{}' + + COPY --link --from=sources --chown=user:user /build/repo/ /build/src/ RUN cd /build/src/main \ - && CGO_ENABLED=0 GOOS=linux go build \ - -o /build/bat-go main.go + && go build -o /build/bat-go main.go + +RUN rm -rf /build/src/tools/payments/dist \ + && mkdir -p /build/src/tools/payments/dist \ + && find /build/src/tools/payments/cmd -type f -name go.mod \ + -print -execdir /bin/sh -c \ + 'go build -o "/build/src/tools/payments/dist/${PWD##*/}" main.go' '{}' + diff --git a/tools/payments/ipython-profile/ipython_config.py b/tools/payments/ipython-profile/ipython_config.py index ede5ddf4a..749f617d4 100644 --- a/tools/payments/ipython-profile/ipython_config.py +++ b/tools/payments/ipython-profile/ipython_config.py @@ -5,13 +5,13 @@ # InteractiveShellApp(Configurable) configuration #------------------------------------------------------------------------------ ## A Mixin for applications that start InteractiveShell instances. -# +# # Provides configurables for loading extensions and executing files # as part of configuring a Shell environment. -# +# # The following methods should be called by the :meth:`initialize` method # of the subclass: -# +# # - :meth:`init_path` # - :meth:`init_shell` (to be implemented by the subclass) # - :meth:`init_gui_pylab` @@ -41,9 +41,9 @@ "%alias report dist/report -e $$environment -p $$payout_id -ru $$redis_username -rp $$redis_password -pcr2 $$pcr2 $$payout_report", "%alias info dist/info -e $$environment -pcr2 $$pcr2 $$payout_report", "%alias await_prepare touch $$prepare_log && dist/prepare -cg cli2 -e $$environment -p $$payout_id -ru $$redis_username -rp $$redis_password -pcr2 $$pcr2 $$payout_report", -"%alias redis redis-cli -p 6380 --user $$redis_username --pass $$redis_password", -"%alias flush_redis redis-cli -p 6380 --user $$redis_username --pass $$redis_password del \*$$payout_id\*", -"%alias list_payouts redis-cli -p 6380 --user $$redis_username --pass $$redis_password keys \*", +"%alias redis redis-cli -p 6380 --user $$redis_username", +"%alias flush_redis redis-cli -p 6380 --user $$redis_username del \*$$payout_id\*", +"%alias list_payouts redis-cli -p 6380 --user $$redis_username keys \*", "%alias worker_logs AWS_VAULT= kubectl --context $$cluster --namespace payment-$$environment logs deployments/worker", "%alias enclave_logs AWS_VAULT= kubectl --context $$cluster --namespace payment-$$environment logs deployments/web -c nitro-utils", "%alias pods AWS_VAULT= kubectl --context $$cluster --namespace payment-$$environment get pods", @@ -58,9 +58,9 @@ # c.InteractiveShellApp.extensions = [] ## Dotted module name(s) of one or more IPython extensions to load. -# +# # For specifying extra extensions to load on the command-line. -# +# # .. versionadded:: 7.10 # Default: [] # c.InteractiveShellApp.extra_extensions = [] @@ -105,7 +105,7 @@ ## If true, IPython will populate the user namespace with numpy, pylab, etc. # and an ``import *`` is done from numpy and pylab, when using pylab mode. -# +# # When False, pylab mode should not import any names into the user # namespace. # Default: True @@ -134,32 +134,32 @@ # c.Application.log_level = 30 ## Configure additional log handlers. -# +# # The default stderr logs handler is configured by the log_level, log_datefmt # and log_format settings. -# +# # This configuration can be used to configure additional handlers (e.g. to # output the log to a file) or for finer control over the default handlers. -# +# # If provided this should be a logging configuration dictionary, for more # information see: # https://docs.python.org/3/library/logging.config.html#logging-config- # dictschema -# +# # This dictionary is merged with the base logging configuration which defines # the following: -# +# # * A logging formatter intended for interactive use called # ``console``. # * A logging handler that writes to stderr called # ``console`` which uses the formatter ``console``. # * A logger with the name of this application set to ``DEBUG`` # level. -# +# # This example adds a new handler that writes to a file: -# +# # .. code-block:: python -# +# # c.Application.logging_config = { # "handlers": { # "file": { @@ -206,7 +206,7 @@ # c.BaseIPythonApplication.copy_config_files = False ## Path to an extra config file to load. -# +# # If specified, load this config file in addition to any other IPython # config. # Default: '' @@ -231,7 +231,7 @@ # See also: Application.log_level # c.BaseIPythonApplication.log_level = 30 -## +## # See also: Application.logging_config # c.BaseIPythonApplication.logging_config = {} @@ -295,7 +295,7 @@ # See also: BaseIPythonApplication.extra_config_file # c.TerminalIPythonApp.extra_config_file = '' -## +## # See also: InteractiveShellApp.extra_extensions # c.TerminalIPythonApp.extra_extensions = [] @@ -328,7 +328,7 @@ # Default: 'IPython.terminal.interactiveshell.TerminalInteractiveShell' # c.TerminalIPythonApp.interactive_shell_class = 'IPython.terminal.interactiveshell.TerminalInteractiveShell' -## +## # See also: BaseIPythonApplication.ipython_dir # c.TerminalIPythonApp.ipython_dir = '' @@ -344,7 +344,7 @@ # See also: Application.log_level # c.TerminalIPythonApp.log_level = 30 -## +## # See also: Application.logging_config # c.TerminalIPythonApp.logging_config = {} @@ -554,11 +554,11 @@ #------------------------------------------------------------------------------ # TerminalInteractiveShell(InteractiveShell) configuration #------------------------------------------------------------------------------ -## +## # See also: InteractiveShell.ast_node_interactivity # c.TerminalInteractiveShell.ast_node_interactivity = 'last_expr' -## +## # See also: InteractiveShell.ast_transformers # c.TerminalInteractiveShell.ast_transformers = [] @@ -567,11 +567,11 @@ # Default: False # c.TerminalInteractiveShell.auto_match = False -## +## # See also: InteractiveShell.autoawait # c.TerminalInteractiveShell.autoawait = True -## +## # See also: InteractiveShell.autocall # c.TerminalInteractiveShell.autocall = 0 @@ -579,11 +579,11 @@ # Default: None # c.TerminalInteractiveShell.autoformatter = None -## +## # See also: InteractiveShell.autoindent # c.TerminalInteractiveShell.autoindent = True -## +## # See also: InteractiveShell.automagic # c.TerminalInteractiveShell.automagic = True @@ -602,11 +602,11 @@ # See also: InteractiveShell.banner2 # c.TerminalInteractiveShell.banner2 = '' -## +## # See also: InteractiveShell.cache_size # c.TerminalInteractiveShell.cache_size = 1000 -## +## # See also: InteractiveShell.color_info # c.TerminalInteractiveShell.color_info = True @@ -658,7 +658,7 @@ # Default: True # c.TerminalInteractiveShell.enable_history_search = True -## +## # See also: InteractiveShell.enable_html_pager # c.TerminalInteractiveShell.enable_html_pager = False @@ -690,7 +690,7 @@ # See also: InteractiveShell.history_length # c.TerminalInteractiveShell.history_length = 10000 -## +## # See also: InteractiveShell.history_load_length # c.TerminalInteractiveShell.history_load_length = 1000 @@ -701,15 +701,15 @@ # See also: InteractiveShell.ipython_dir # c.TerminalInteractiveShell.ipython_dir = '' -## +## # See also: InteractiveShell.logappend # c.TerminalInteractiveShell.logappend = '' -## +## # See also: InteractiveShell.logfile # c.TerminalInteractiveShell.logfile = '' -## +## # See also: InteractiveShell.logstart # c.TerminalInteractiveShell.logstart = False @@ -734,7 +734,7 @@ # See also: InteractiveShell.object_info_string_level # c.TerminalInteractiveShell.object_info_string_level = 0 -## +## # See also: InteractiveShell.pdb # c.TerminalInteractiveShell.pdb = False @@ -770,20 +770,20 @@ # c.TerminalInteractiveShell.separate_out2 = '' ## Add, disable or modifying shortcuts. -# +# # Each entry on the list should be a dictionary with ``command`` key # identifying the target function executed by the shortcut and at least # one of the following: -# +# # - ``match_keys``: list of keys used to match an existing shortcut, # - ``match_filter``: shortcut filter used to match an existing shortcut, # - ``new_keys``: list of keys to set, # - ``new_filter``: a new shortcut filter to set -# +# # The filters have to be composed of pre-defined verbs and joined by one # of the following conjunctions: ``&`` (and), ``|`` (or), ``~`` (not). # The pre-defined verbs are: -# +# # - `always` # - `never` # - `has_line_below` @@ -825,16 +825,16 @@ # - `navigable_suggestions` # - `cursor_in_leading_ws` # - `pass_through` -# +# # To disable a shortcut set ``new_keys`` to an empty list. # To add a shortcut add key ``create`` with value ``True``. -# +# # When modifying/disabling shortcuts, ``match_keys``/``match_filter`` can # be omitted if the provided specification uniquely identifies a shortcut # to be modified/disabled. When modifying a shortcut ``new_filter`` or # ``new_keys`` can be omitted which will result in reuse of the existing # filter/keys. -# +# # Only shortcuts defined in IPython (and not default prompt-toolkit # shortcuts) can be modified or disabled. The full list of shortcuts, # command identifiers and filters is available under @@ -847,10 +847,10 @@ # c.TerminalInteractiveShell.show_rewritten_input = True ## Use `raw_input` for the REPL, without completion and prompt colors. -# +# # Useful when controlling IPython as a subprocess, and piping STDIN/OUT/ERR. Known usage are: # IPython own testing machinery, and emacs inferior-shell integration through elpy. -# +# # This mode default to `True` if the `IPY_TEST_SIMPLE_PROMPT` # environment variable is set, or the current terminal is not a tty. # Default: False @@ -863,7 +863,7 @@ # Default: 6 # c.TerminalInteractiveShell.space_for_menu = 6 -## +## # See also: InteractiveShell.sphinxify_docstring # c.TerminalInteractiveShell.sphinxify_docstring = False @@ -884,7 +884,7 @@ ## Use 24bit colors instead of 256 colors in prompt highlighting. # If your terminal supports true color, the following command should # print ``TRUECOLOR`` in orange:: -# +# # printf "\x1b[38;2;255;100;0mTRUECOLOR\x1b[0m\n" # Default: False # c.TerminalInteractiveShell.true_color = False @@ -910,19 +910,19 @@ # HistoryAccessor(HistoryAccessorBase) configuration #------------------------------------------------------------------------------ ## Access the history database without adding to it. -# +# # This is intended for use by standalone history tools. IPython shells use # HistoryManager, below, which is a subclass of this. ## Options for configuring the SQLite connection -# +# # These options are passed as keyword args to sqlite3.connect # when establishing database connections. # Default: {} # c.HistoryAccessor.connection_options = {} ## enable the SQLite history -# +# # set enabled=False to disable the SQLite history, # in which case there will be no stored history, no SQLite connection, # and no background saving thread. This may be necessary in some @@ -931,17 +931,17 @@ # c.HistoryAccessor.enabled = True ## Path to file to use for SQLite history database. -# +# # By default, IPython will put the history database in the IPython # profile directory. If you would rather share one history among # profiles, you can set this value in each, so that they are consistent. -# +# # Due to an issue with fcntl, SQLite is known to misbehave on some NFS # mounts. If you see IPython hanging, try setting this to something on a # local disk, e.g:: -# +# # ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite -# +# # you can also use the specific value `:memory:` (including the colon # at both end but not the back ticks), to avoid creating an history file. # Default: traitlets.Undefined @@ -983,24 +983,24 @@ # c.MagicsManager.auto_magic = True ## Mapping from magic names to modules to load. -# +# # This can be used in IPython/IPykernel configuration to declare lazy magics # that will only be imported/registered on first use. -# +# # For example:: -# +# # c.MagicsManager.lazy_magics = { # "my_magic": "slow.to.import", # "my_other_magic": "also.slow", # } -# +# # On first invocation of `%my_magic`, `%%my_magic`, `%%my_other_magic` or # `%%my_other_magic`, the corresponding module will be loaded as an ipython # extensions as if you had previously done `%load_ext ipython`. -# +# # Magics names should be without percent(s) as magics can be both cell and line # magics. -# +# # Lazy loading happen relatively late in execution process, and complex # extensions that manipulate Python/IPython internal state or global state might # not support lazy loading. @@ -1011,10 +1011,10 @@ # ProfileDir(LoggingConfigurable) configuration #------------------------------------------------------------------------------ ## An object to manage the profile directory and its resources. -# +# # The profile directory is used by all IPython applications, to manage # configuration, logging and security. -# +# # This object knows how to find, create and manage these directories. This # should be used by any code that wants to handle profiles. @@ -1027,25 +1027,25 @@ # BaseFormatter(Configurable) configuration #------------------------------------------------------------------------------ ## A base formatter class that is configurable. -# +# # This formatter should usually be used as the base class of all formatters. # It is a traited :class:`Configurable` class and includes an extensible # API for users to determine how their objects are formatted. The following # logic is used to find a function to format an given object. -# +# # 1. The object is introspected to see if it has a method with the name # :attr:`print_method`. If is does, that object is passed to that method # for formatting. # 2. If no print method is found, three internal dictionaries are consulted # to find print method: :attr:`singleton_printers`, :attr:`type_printers` # and :attr:`deferred_printers`. -# +# # Users should use these dictionaries to register functions that will be # used to compute the format data for their objects (if those objects don't # have the special print methods). The easiest way of using these # dictionaries is through the :meth:`for_type` and :meth:`for_type_by_name` # methods. -# +# # If no function/callable is found to compute the format data, ``None`` is # returned and this format type is not used. @@ -1065,12 +1065,12 @@ # PlainTextFormatter(BaseFormatter) configuration #------------------------------------------------------------------------------ ## The default pretty-printer. -# +# # This uses :mod:`IPython.lib.pretty` to compute the format data of # the object. If the object cannot be pretty printed, :func:`repr` is used. # See the documentation of :mod:`IPython.lib.pretty` for details on # how to write pretty printers. Here is a simple example:: -# +# # def dtype_pprinter(obj, p, cycle): # if cycle: # return p.text('dtype(...)') @@ -1093,7 +1093,7 @@ # c.PlainTextFormatter.float_precision = '' ## Truncate large collections (lists, dicts, tuples, sets) to this size. -# +# # Set to 0 to disable truncation. # Default: 1000 # c.PlainTextFormatter.max_seq_length = 1000 @@ -1120,7 +1120,7 @@ # Completer(Configurable) configuration #------------------------------------------------------------------------------ ## Enable auto-closing dictionary keys. -# +# # When enabled string keys will be suffixed with a final quote (matching the # opening quote), tuple keys will also receive a separating comma if needed, and # keys which are final will receive a closing bracket (``]``). @@ -1139,15 +1139,15 @@ # c.Completer.debug = False ## Policy for code evaluation under completion. -# +# # Successive options allow to enable more eager evaluation for better # completion suggestions, including for nested dictionaries, nested lists, # or even results of function calls. # Setting ``unsafe`` or higher can lead to evaluation of arbitrary user # code on :kbd:`Tab` with potentially unwanted or dangerous side effects. -# +# # Allowed values are: -# +# # - ``forbidden``: no evaluation of code is permitted, # - ``minimal``: evaluation of literals and access to built-in namespace; # no item/attribute evaluationm no access to locals/globals, @@ -1164,13 +1164,13 @@ # c.Completer.evaluation = 'limited' ## Activate greedy completion. -# +# # .. deprecated:: 8.8 # Use :std:configtrait:`Completer.evaluation` and :std:configtrait:`Completer.auto_close_dict_keys` instead. -# +# # When enabled in IPython 8.8 or newer, changes configuration as # follows: -# +# # - ``Completer.evaluation = 'unsafe'`` # - ``Completer.auto_close_dict_keys = True`` # Default: False @@ -1192,7 +1192,7 @@ #------------------------------------------------------------------------------ ## Extension of the completer class with IPython-specific features -## +## # See also: Completer.auto_close_dict_keys # c.IPCompleter.auto_close_dict_keys = False @@ -1208,7 +1208,7 @@ # c.IPCompleter.debug = False ## List of matchers to disable. -# +# # The list should contain matcher identifiers (see # :any:`completion_matcher`). # Default: [] @@ -1228,35 +1228,35 @@ # c.IPCompleter.jedi_compute_type_timeout = 400 ## DEPRECATED as of version 5.0. -# +# # Instruct the completer to use __all__ for the completion -# +# # Specifically, when completing on ``object.``. -# +# # When True: only those names in obj.__all__ will be included. -# +# # When False [default]: the __all__ attribute is ignored # Default: False # c.IPCompleter.limit_to__all__ = False ## Whether to merge completion results into a single list -# +# # If False, only the completion results from the first non-empty # completer will be returned. -# +# # As of version 8.6.0, setting the value to ``False`` is an alias for: # ``IPCompleter.suppress_competing_matchers = True.``. # Default: True # c.IPCompleter.merge_completions = True ## Instruct the completer to omit private method names -# +# # Specifically, when completing on ``object.``. -# +# # When 2 [default]: all names that start with '_' will be excluded. -# +# # When 1: all 'magic' names (``__foo__``) will be excluded. -# +# # When 0: nothing will be excluded. # Choices: any of [0, 1, 2] # Default: 2 @@ -1271,19 +1271,19 @@ # c.IPCompleter.profiler_output_dir = '.completion_profiles' ## Whether to suppress completions from other *Matchers*. -# +# # When set to ``None`` (default) the matchers will attempt to auto-detect # whether suppression of other matchers is desirable. For example, at the # beginning of a line followed by `%` we expect a magic completion to be the # only applicable option, and after ``my_dict['`` we usually expect a completion # with an existing dictionary key. -# +# # If you want to disable this heuristic and see completions from all matchers, # set ``IPCompleter.suppress_competing_matchers = False``. To disable the # heuristic for specific matchers provide a dictionary mapping: # ``IPCompleter.suppress_competing_matchers = {'IPCompleter.dict_key_matcher': # False}``. -# +# # Set ``IPCompleter.suppress_competing_matchers = True`` to limit completions to # the set of matchers with the highest priority; this is equivalent to # ``IPCompleter.merge_completions`` and can be beneficial for performance, but @@ -1301,22 +1301,22 @@ # ScriptMagics(Magics) configuration #------------------------------------------------------------------------------ ## Magics for talking to scripts -# +# # This defines a base `%%script` cell magic for running a cell # with a program in a subprocess, and registers a few top-level # magics that call %%script with common interpreters. ## Extra script cell magics to define -# +# # This generates simple wrappers of `%%script foo` as `%%foo`. -# +# # If you want to add script magics that aren't on your path, # specify them in script_paths # Default: [] # c.ScriptMagics.script_magics = [] ## Dict mapping short 'ruby' names to full paths, such as '/opt/secret/bin/ruby' -# +# # Only necessary for items in script_magics where the default path will not # find the right interpreter. # Default: {} @@ -1335,7 +1335,7 @@ # StoreMagics(Magics) configuration #------------------------------------------------------------------------------ ## Lightweight persistence for python variables. -# +# # Provides the %store magic. ## If True, any %store-d variables will be automatically restored diff --git a/tools/payments/ipython-profile/startup/00-setup.py b/tools/payments/ipython-profile/startup/00-setup.py index 0b9657520..163ea1ff5 100644 --- a/tools/payments/ipython-profile/startup/00-setup.py +++ b/tools/payments/ipython-profile/startup/00-setup.py @@ -36,6 +36,7 @@ redis_username = os.getenv("REDIS_USERNAME") redis_password = os.getenv("REDIS_PASSWORD") operator_key = "~/.ssh/settlements" +os.putenv("REDISCLI_AUTH", redis_password) jobs = bg.BackgroundJobManager() @@ -90,7 +91,7 @@ def _get_web_env(): return env def _set_secrets_s3_bucket(): - global secrets_s3_bucket + global secrets_s3_bucket env = _get_web_env() secrets_s3_bucket = env["ENCLAVE_CONFIG_BUCKET_NAME"]