Allowing setting config vars from environment (#409)

* allowing setting config vars from environment * refactor to use a single settings.py, allow a settings.yaml * consolidate secrets into same settings the user can now define variables directly in settings.py, via an external secrets.py or settings.yaml, or in the environment with SREGISTRY_ as a prefix. * ensure linting is done for new release * ensure SECRET_KEY also honored in environment Signed-off-by: vsoch <[email protected]>
singularityhub · Oct 13, 2022 · dfff8fd · dfff8fd
1 parent 8a9fb8b
commit dfff8fd
Show file tree

Hide file tree

Showing 43 changed files with 1,087 additions and 675 deletions.
diff --git a/.gitignore b/.gitignore
@@ -21,6 +21,7 @@ celerybeat-schedule.db
 *.mp4
 OLD
 secrets.py
+settings.yaml
 local_settings.py
 cronjob
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,7 +12,9 @@ represented by the pull requests that fixed them. Critical items to know are:
 
 
 ## [master](https://github.com/singularityhub/sregistry/tree/master) (master)
- - update python base to 3.9, minio server to use new credentials
+ - consolidate config into one file with environment (2.0.0)
+   - This is an API breaking change, as the settings are completely refactored
+ - update python base to 3.9, minio server to use new credentials (1.1.4)
    - docker-compose updated to use docker compose
  - add: auto set "verify" attribute of s3 and s3_external obj in minio.py for SSL use (1.1.39) 
  - fix issues with psycopg2-binary and saml auth (1.1.38)

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.1.4
+2.0.0
diff --git a/docs/_docs/install/settings.md b/docs/_docs/install/settings.md
diff --git a/requirements.txt b/requirements.txt
@@ -1,5 +1,6 @@
 coreapi==2.3.3
 cython
+django==2.2.28
 django-chosen
 django-crispy-forms
 django-datatables-view
@@ -12,25 +13,26 @@ django-guardian
 django-hstore==1.3.5
 django-notifications-hq
 django-ratelimit==2.0.0
+djangorestframework==3.11.2
 django-rest-swagger
 django-rq
 django-taggit
 django-taggit-templatetags
 django-user-agents
-django==2.2.28
-djangorestframework==3.11.2
 google
 google-api-python-client
 h5py
 ipython
 markdown
+minio==5.0.8
 numexpr
 oauth2client==3.0
 Pillow
 psycopg2-binary~=2.8.6
 pygments
 python3-saml
 python-social-auth
+pyyaml
 requests
 requests-oauthlib
 requests-toolbelt
@@ -41,4 +43,3 @@ social-auth-app-django
 social-auth-core[saml]
 sregistry[all-basic]>=0.2.19
 uwsgi
-minio==5.0.8
diff --git a/shub/apps/api/urls/registry.py b/shub/apps/api/urls/registry.py
@@ -18,7 +18,7 @@
 ################################################################################
 
 
-class Registry(object):
+class Registry:
     def __init__(self, **kwargs):
         self.name = settings.REGISTRY_NAME
         self.id = settings.REGISTRY_URI

diff --git a/shub/apps/api/utils.py b/shub/apps/api/utils.py
@@ -30,7 +30,7 @@ def _parse_header(auth):
 
     header, content = auth.split(" ")
     content = content.split(",")
-    values = dict()
+    values = {}
     for entry in content:
         key, val = re.split("=", entry, 1)
         values[key] = val

diff --git a/shub/apps/library/views/helpers.py b/shub/apps/library/views/helpers.py
@@ -72,7 +72,6 @@ def get_collection_downloads(collection):
     downloads = 0
     for container in collection.containers.all():
         downloads += get_container_downloads(container)
-    return
 
 
 def get_container_downloads(container):
@@ -190,9 +189,9 @@ def generate_collection_tags(collection):
     latest. Technically, collections should be namespaced
     consistently, and we assume this.
     """
-    unique_tags = set(
-        [c.tag for c in collection.containers.all() if not re.search(tag_regex, c.tag)]
-    )
+    unique_tags = {
+        c.tag for c in collection.containers.all() if not re.search(tag_regex, c.tag)
+    }
     tags = {}
 
     for tag in unique_tags:
@@ -267,9 +266,6 @@ def generate_container_metadata(container):
 
     data = {
         "deleted": False,  # 2019-03-15T19:02:24.015Z
-        "createdAt": container.add_date.strftime(
-            formatString
-        ),  # No idea what their format is...
         "createdBy": str(container.collection.owners.first().id),
         "createdAt": container.add_date.strftime(
             formatString

diff --git a/shub/apps/library/views/images.py b/shub/apps/library/views/images.py
@@ -39,7 +39,6 @@
     generate_collection_tags,
     generate_collections_list,
     generate_container_metadata,
-    get_collection,
     get_container,
     get_token,
     validate_token,

diff --git a/shub/apps/library/views/minio.py b/shub/apps/library/views/minio.py
@@ -10,7 +10,6 @@
 
 import hashlib
 import hmac
-import os
 from datetime import datetime
 
 from boto3 import Session

diff --git a/shub/apps/logs/mixins.py b/shub/apps/logs/mixins.py
@@ -14,7 +14,7 @@
 from shub.apps.logs.utils import clean_data
 
 
-class BaseLoggingMixin(object):
+class BaseLoggingMixin:
     logging_methods = "__all__"
 
     """Mixin to log requests"""

diff --git a/shub/apps/main/models/shared.py b/shub/apps/main/models/shared.py
@@ -137,9 +137,9 @@ def labels(self):
 
     def container_names(self):
         """return distinct container names"""
-        return list(
+        return [
             [x[0] for x in self.containers.values_list("name").distinct() if len(x) > 0]
-        )
+        ]
 
     # Permissions
 

diff --git a/shub/apps/main/views/collections.py b/shub/apps/main/views/collections.py
@@ -116,8 +116,7 @@ def new_collection(request):
             return redirect("collection_details", cid=collection.id)
 
         # Just new collection form, not a post
-        else:
-            return render(request, "collections/new_collection.html")
+        return render(request, "collections/new_collection.html")
 
     # If user makes it down here, does not have permission
     messages.info(request, "You don't have permission to perform this action.")

diff --git a/shub/apps/main/views/compare.py b/shub/apps/main/views/compare.py
@@ -40,7 +40,7 @@ def generate_size_data(collections):
     flare.analytics.graph.ShortestPaths,5914,9
     flare.analytics.graph.SpanningTree,3416,10
     """
-    data = dict()
+    data = {}
     for collection in collections:
 
         collection_name = collection.name

diff --git a/shub/apps/main/views/download.py b/shub/apps/main/views/download.py
@@ -132,8 +132,5 @@ def _download_container(container, request):
                 return redirect(signed_url)
             return HttpResponseForbidden()
 
-        # There is no container
-        raise Http404
-
-    else:
-        raise Http404
+    # There is no container
+    raise Http404
diff --git a/shub/apps/main/views/labels.py b/shub/apps/main/views/labels.py
@@ -25,7 +25,7 @@
 
 def get_label(key=None, value=None):
 
-    keyargs = dict()
+    keyargs = {}
     if key is not None:
         keyargs["key"] = key
     if value is not None:

diff --git a/shub/apps/main/views/tags.py b/shub/apps/main/views/tags.py
@@ -25,7 +25,7 @@
 
 def get_tag(name=None, tid=None):
 
-    keyargs = dict()
+    keyargs = {}
     if name is not None:
         keyargs["name"] = name
     if tid is not None:

diff --git a/shub/apps/users/management/commands/add_admin.py b/shub/apps/users/management/commands/add_admin.py
@@ -39,6 +39,5 @@ def handle(self, *args, **options):
 
         if user.is_staff is True:  # and user.manager is True:
             raise CommandError("This user can already manage and build.")
-        else:
-            user = User.objects.add_staff(user)
-            bot.debug("%s can now manage and build." % (user.username))
+        user = User.objects.add_staff(user)
+        bot.debug("%s can now manage and build." % user.username)
diff --git a/shub/apps/users/management/commands/add_superuser.py b/shub/apps/users/management/commands/add_superuser.py
@@ -39,6 +39,5 @@ def handle(self, *args, **options):
 
         if user.is_superuser is True:
             raise CommandError("This user is already a superuser.")
-        else:
-            user = User.objects.add_superuser(user)
-            bot.debug("%s is now a superuser." % (user.username))
+        user = User.objects.add_superuser(user)
+        bot.debug("%s is now a superuser." % (user.username))
diff --git a/shub/apps/users/management/commands/remove_admin.py b/shub/apps/users/management/commands/remove_admin.py
@@ -39,7 +39,6 @@ def handle(self, *args, **options):
 
         if user.is_staff is False:  # and user.manager is False:
             raise CommandError("This user already can't manage and build.")
-        else:
-            user.is_staff = False
-            bot.debug("%s can no longer manage and build." % (user.username))
-            user.save()
+        user.is_staff = False
+        bot.debug("%s can no longer manage and build." % (user.username))
+        user.save()
diff --git a/shub/apps/users/management/commands/remove_superuser.py b/shub/apps/users/management/commands/remove_superuser.py
@@ -39,7 +39,6 @@ def handle(self, *args, **options):
 
         if user.is_superuser is False:
             raise CommandError("This user already is not a superuser.")
-        else:
-            user.is_superuser = False
-            bot.debug("%s is no longer a superuser." % (user.username))
-            user.save()
+        user.is_superuser = False
+        bot.debug("%s is no longer a superuser." % (user.username))
+        user.save()
diff --git a/shub/apps/users/views/auth.py b/shub/apps/users/views/auth.py
@@ -35,8 +35,7 @@ def validate_credentials(user, context=None):
     user: the user to check, should have social_auth
     context: an optional context object to append to
     """
-    if context is None:
-        context = dict()
+    context = context or {}
 
     # Right now we have github for repos and google for storage
     credentials = [