Merge pull request #5 from cedadev/development

Pull development into main ahead of tagged release

.github/workflows/testing.yml

@@ -0,0 +1,41 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: auto-testing
+
+on:
+  push:
+    branches: [ "main", "development" ]
+  pull_request:
+    branches: [ "main", "development" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10"]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        python -m pip install -e .
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest
+      run: |
+        pytest

.gitignore

@@ -0,0 +1,5 @@
+**/__pycache__/
+.pytest_cache/
+*egg-info
+.venv/
+tests/test_dir*

docs/user_guide/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/user_guide/source/conf.py

@@ -0,0 +1,28 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = 'Near-Line Data Store client'
+copyright = '2023, Neil Massey and Jack Leland'
+author = 'Neil Massey and Jack Leland'
+release = '0.1.0'
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = ['sphinx_click']
+
+templates_path = ['_templates']
+exclude_patterns = []
+
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = 'alabaster'
+html_static_path = ['_static']

nlds_client/clientlib/config.py

@@ -1,5 +1,7 @@
 import json
 import os.path
+
+from click import option
 from nlds_client.clientlib.nlds_client_setup import CONFIG_FILE_LOCATION
 from nlds_client.clientlib.exceptions import ConfigError
 
@@ -16,7 +18,7 @@ def validate_config_file(json_config):
 
     for key in ["url", "api"]:
         try:
-            value = server_section[key]
+            _ = server_section[key]
         except KeyError:
             raise ConfigError(
                 f"The config file at {CONFIG_FILE_LOCATION} does not contain "
@@ -38,21 +40,73 @@ def validate_config_file(json_config):
                 "oauth_scopes",
                 "oauth_token_file_location"]:
         try:
-            value = auth_section[key]
+            _ = auth_section[key]
         except KeyError:
             raise ConfigError(
                 f"The config file at {CONFIG_FILE_LOCATION} does not contain "
                 f"{key} in ['authentication'] section."
             )
 
+    # user section
+    try:
+        user_section = json_config["user"]
+    except KeyError:
+        raise ConfigError(
+            f"The config file at {CONFIG_FILE_LOCATION} does not contain an "
+             "['user'] section."
+        )
+
+    for key in ["default_user",
+                "default_group"]:
+        try:
+            _ = user_section[key]
+        except KeyError:
+            raise ConfigError(
+                f"The config file at {CONFIG_FILE_LOCATION} does not contain "
+                f"{key} in ['user'] section."
+            )               
+
+    # object_storage section
+    try:
+        os_section = json_config["object_storage"]
+    except KeyError:
+        raise ConfigError(
+            f"The config file at {CONFIG_FILE_LOCATION} does not contain an "
+             "['object_storage'] section."
+        )
+
+    for key in ["access_key",       # "tenancy" is optional and will default
+                "secret_key"]:      # on the server
+        try:
+            _ = os_section[key]
+        except KeyError:
+            raise ConfigError(
+                f"The config file at {CONFIG_FILE_LOCATION} does not contain "
+                f"{key} in ['object_storage'] section."
+            )    
 
 def load_config():
     """Config file for the client contains:
+        server : {
+            url : <server>,
+            api : <version>
+        },
+        user : {
+            default_user : <user>,
+            default_group : <group>
+        },
         authentication : {
-            oauth_client_id : <client id>,
-            oauth_client_secret : <client secret>,
-            oauth_token_url : <token url>,
-            oauth_token_introspect_url : <token introspect url>
+            oauth_client_id : <client_id>,
+            oauth_client_secret : <client_secret>,
+            oauth_token_url : <token_url>,
+            oauth_scopes : <scopes>,
+            oauth_token_file_location : <token_location>
+        },
+        object_storage : {
+            tenancy : <os_tenancy> (optional),
+            access_key : <os_access_key>,
+            secret_key : <os_secret_key>
+
         }
     """
     # Location of config file is {CONFIG_FILE_LOCATION}.  Open it, checking
@@ -98,3 +152,49 @@ def get_group(config, group):
         "default_group" in config["user"]):
         group = config["user"]["default_group"]
     return group
+
+
+_DEFAULT_OPTIONS = {
+    "verify_certificates": True,
+    "resolve_filenames": True
+}
+def get_option(config, option_name, section_name='option'):
+    """Get an option from either the config or the DEFAULT_OPTIONS dict."""
+    if (section_name in config and
+        # Get value from config if option section and option present
+        option_name in config[section_name]):
+        option_value = config[section_name][option_name]
+    elif option_name in _DEFAULT_OPTIONS:
+        # Otherwise get the default value 
+        option_value = _DEFAULT_OPTIONS[option_name]
+    else:
+        # Silently fail if option not specified in _DEFAULT_OPTIONS
+        option_value = None
+    return option_value
+
+
+def get_tenancy(config):
+    """Get the object storage tenancy from the config file.  This is optional
+    so could be None."""
+    tenancy = None
+    if ("object_storage" in config and
+        "tenancy" in config["object_storage"]):
+        tenancy = config["object_storage"]["tenancy"]
+    return tenancy
+
+
+def get_access_key(config):
+    """Get the object storage access key from the config file. """
+    access_key = ""
+    if ("object_storage" in config and
+        "access_key" in config["object_storage"]):
+        access_key = config["object_storage"]["access_key"]
+    return access_key
+
+def get_secret_key(config):
+    """Get the object storage secret key from the config file. """
+    secret_key = ""
+    if ("object_storage" in config and
+        "secret_key" in config["object_storage"]):
+        secret_key = config["object_storage"]["secret_key"]
+    return secret_key

nlds_client/clientlib/exceptions.py

@@ -23,3 +23,6 @@ class UsageError(Exception):
 
 class ConfigError(Exception):
     pass
+
+class StateError(Exception):
+    pass