Merge pull request #265 from fasrc/cp_slurm

Slurm plugin

Dockerfile

@@ -12,6 +12,7 @@ RUN apt-get update \
     && apt-get install -y --no-install-recommends \
     && apt-get install -y redis redis-server \
     && apt-get install -y libsasl2-dev libldap2-dev libssl-dev \
+    && apt-get install -y sssd sssd-tools supervisor \
     && rm -rf /var/lib/apt/lists/*
 RUN mkdir ~/.ssh && echo "Host git*\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
 
@@ -50,6 +51,10 @@ ENV PYTHONPATH /usr/src/app:/usr/src/app/ifxreport
 
 RUN mkdir -p /usr/src/app/media/reports
 
+RUN printf "deb http://ftp.us.debian.org/debian buster main" > /etc/apt/sources.list.d/backports.list && \
+    apt-get update && apt-get install libreadline7 && \
+    rm -rf /var/lib/apt/lists/*
+
 EXPOSE 80
 EXPOSE 25
 

coldfront/config/logging.py

@@ -66,6 +66,10 @@
         'coldfront.core.project': {
             'handlers': ['key-events'],
             'level': 'INFO',
+        },
+        'coldfront.core.allocation': {
+            'handlers': ['key-events'],
+            'level': 'INFO',
         }
     },
 }

coldfront/config/plugins/slurm.py

@@ -1,5 +1,6 @@
 from coldfront.config.base import INSTALLED_APPS
 from coldfront.config.env import ENV
+from coldfront.config.logging import LOGGING
 
 INSTALLED_APPS += [
     'coldfront.plugins.slurm',
@@ -9,3 +10,23 @@
 SLURM_NOOP = ENV.bool('SLURM_NOOP', False)
 SLURM_IGNORE_USERS = ENV.list('SLURM_IGNORE_USERS', default=['root'])
 SLURM_IGNORE_ACCOUNTS = ENV.list('SLURM_IGNORE_ACCOUNTS', default=[])
+
+
+LOGGING['formatters']['slurm'] = {
+            "()": "django.utils.log.ServerFormatter",
+            "format": "[{server_time}] {levelname} {message}",
+            "style": "{",
+        }
+
+LOGGING['handlers']['slurm'] = {
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': 'logs/slurm.log',
+            'when': 'D',
+            'backupCount': 30, # how many backup files to keep
+            'formatter': 'slurm',
+        }
+
+LOGGING['loggers']['slurm'] = {
+            'handlers': ['slurm'],
+            'level': 'INFO',
+        }

coldfront/config/plugins/xdmod.py

@@ -1,5 +1,6 @@
 from coldfront.config.base import INSTALLED_APPS
 from coldfront.config.env import ENV
+from coldfront.config.logging import LOGGING
 
 #------------------------------------------------------------------------------
 # Enable XDMoD support
@@ -8,4 +9,26 @@
     'coldfront.plugins.xdmod',
 ]
 
+XDMOD_USER = ENV.str('XDMOD_USER', default='')
+XDMOD_PASS = ENV.str('XDMOD_PASS', default='')
 XDMOD_API_URL = ENV.str('XDMOD_API_URL')
+
+
+LOGGING['formatters']['xdmod'] = {
+            "()": "django.utils.log.ServerFormatter",
+            "format": "[{server_time}] {levelname} {message}",
+            "style": "{",
+        }
+
+LOGGING['handlers']['xdmod'] = {
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': 'logs/xdmod.log',
+            'when': 'D',
+            'backupCount': 30, # how many backup files to keep
+            'formatter': 'xdmod',
+        }
+
+LOGGING['loggers']['xdmod'] = {
+            'handlers': ['xdmod'],
+            'level': 'INFO',
+        }

coldfront/config/urls.py

@@ -22,6 +22,7 @@
     path('allocation-summary', portal_views.allocation_summary, name='allocation-summary'),
     path('allocation-by-fos', portal_views.allocation_by_fos, name='allocation-by-fos'),
     path('user/', include('coldfront.core.user.urls')),
+    # path('opt_out/', portal_views.opt_out, name='opt-out'),
     # path('__debug__/', include('debug_toolbar.urls')),
     path('project/', include('coldfront.core.project.urls')),
     path('allocation/', include('coldfront.core.allocation.urls')),
@@ -52,4 +53,4 @@
     urlpatterns.append(path('', include('coldfront.plugins.fasrc_monitoring.urls')))
 
 if 'django_prometheus' in settings.INSTALLED_APPS:
-    urlpatterns.append(path('metrics', include('coldfront.plugins.fasrc_monitoring.urls')))
+    urlpatterns.append(path('', include('django_prometheus.urls')))

coldfront/core/allocation/admin.py

@@ -16,6 +16,8 @@
                                               AllocationUser,
                                               AllocationUserNote,
                                               AllocationUserStatusChoice,
+                                              AllocationUserAttributeType,
+                                              AllocationUserAttribute,
                                               AttributeType)
 
 
@@ -276,6 +278,52 @@ def set_removed(self, request, queryset):
     ]
 
 
+@admin.register(AllocationUserAttributeType)
+class AllocationUserAttributeTypeAdmin(admin.ModelAdmin):
+    list_display = ('pk', 'name', 'attribute_type', 'is_changeable', 'is_private')
+
+
+@admin.register(AllocationUserAttribute)
+class AllocationUserAttributeAdmin(SimpleHistoryAdmin):
+    readonly_fields_change = (
+        'allocationuser', 'allocationuser_attribute_type', 'created', 'modified')
+    fields_change = ('allocationuser',
+                     'allocationuser_attribute_type', 'value', 'created', 'modified',)
+    list_display = (
+        'pk', 'allocationuser', 'project', 'resource',
+        'allocationuser_attribute_type', 'value', 'created', 'modified',
+    )
+    list_filter = (UsageValueFilter, 'allocationuser_attribute_type',)
+    search_fields = (
+        'allocationuser__user__first_name',
+        'allocationuser__user__last_name',
+        'allocationuser__user__username',
+    )
+
+    def resource(self, obj):
+        return obj.allocationuser.allocation.get_parent_resource
+
+    def allocationuser_status(self, obj):
+        return obj.allocationuser.status
+
+    def project(self, obj):
+        return textwrap.shorten(obj.allocationuser.allocation.project.title, width=50)
+
+    def project_title(self, obj):
+        return obj.allocationuser.allocation.project.title
+
+    def get_fields(self, request, obj):
+        if obj is None:
+            return super().get_fields(request)
+        return self.fields_change
+
+    def get_readonly_fields(self, request, obj):
+        if obj is None:
+            # We are adding an object
+            return super().get_readonly_fields(request)
+        return self.readonly_fields_change
+
+
 class ValueFilter(admin.SimpleListFilter):
     title = _('value')
     parameter_name = 'value'

coldfront/core/allocation/management/commands/add_allocation_defaults.py

@@ -4,6 +4,7 @@
                                               AllocationAttributeType,
                                               AllocationStatusChoice,
                                               AllocationChangeStatusChoice,
+                                              AllocationUserAttributeType,
                                               AllocationUserStatusChoice)
 
 
@@ -30,6 +31,20 @@ def handle(self, *args, **options):
         for choice in ('Active', 'Error', 'Removed', ):
             AllocationUserStatusChoice.objects.get_or_create(name=choice)
 
+        for name, attribute_type, is_private in (
+            ('FairShare', 'Float', False),
+            ('NormShares', 'Float', False),
+            ('EffectvUsage', 'Float', False),
+            ('RawUsage', 'Int', False),
+        ):
+            AllocationUserAttributeType.objects.update_or_create(
+                name=name,
+                defaults={
+                    'attribute_type': AttributeType.objects.get(name=attribute_type),
+                    'is_private': is_private,
+                }
+            )
+
         for name, attribute_type, has_usage, is_private in (
             # FASRC defaults
             ('Storage Quota (TB)', 'Float', True, False),
@@ -45,10 +60,14 @@ def handle(self, *args, **options):
             ('High Security', 'Yes/No', False, False),
             ('DUA', 'Yes/No', False, False),
             ('External Sharing', 'Yes/No', False, False),
+            ('FairShare', 'Float', False, False),
+            ('NormShares', 'Float', False, False),
+            ('EffectvUsage', 'Float', False, False),
+            ('RawUsage', 'Int', False, False),
             # UBCCR defaults
-            # ('Cloud Account Name', 'Text', False, False),
+            ('Cloud Account Name', 'Text', False, False),
             # ('CLOUD_USAGE_NOTIFICATION', 'Yes/No', False, True),
-            # ('Core Usage (Hours)', 'Int', True, False),
+            ('Core Usage (Hours)', 'Float', True, False),
             # ('Accelerator Usage (Hours)', 'Int', True, False),
             # ('Cloud Storage Quota (TB)', 'Float', True, False),
             # ('EXPIRE NOTIFICATION', 'Yes/No', False, True),
@@ -60,13 +79,13 @@ def handle(self, *args, **options):
             # ('Paid Storage Support (Hours)', 'Float', True, True),
             # ('Purchase Order Number', 'Int', False, True),
             # ('send_expiry_email_on_date', 'Date', False, True),
-            # ('slurm_account_name', 'Text', False, False),
+            ('slurm_account_name', 'Text', False, False),
             # ('slurm_specs', 'Attribute Expanded Text', False, True),
             # ('slurm_specs_attriblist', 'Text', False, True),
             # ('slurm_user_specs', 'Attribute Expanded Text', False, True),
             # ('slurm_user_specs_attriblist', 'Text', False, True),
             # ('Storage Quota (GB)', 'Int', False, False),
-            # ('Storage_Group_Name', 'Text', False, False),
+            ('Storage_Group_Name', 'Text', False, False),
             # ('SupportersQOS', 'Yes/No', False, False),
             # ('SupportersQOSExpireDate', 'Date', False, False),
         ):