Removed the

.env.example

@@ -23,6 +23,10 @@ CONTACT_EMAIL='PhysioNet Contact <[email protected]>'
 SERVER_EMAIL='PhysioNet System <[email protected]>'
 ERROR_EMAIL='[email protected]'
 
+# Contact address for project editors. This address may be viewable by authors.
+# Optionally, add "PROJECT-SLUG" to include the project slug.
+PROJECT_EDITOR_EMAIL='[email protected]'
+
 # Admins
 ADMINS_NAME=PhysioNet Technical
 ADMINS_MAIL=[email protected]
@@ -60,6 +64,18 @@ AWS_VALUE=secret
 AWS_VALUE2=secret
 AWS_CLOUD_FORMATION=url
 
+# AWS credentials (Access Key and Secret Key): Configure AWS credentials in the AWS CLI profile using the 'aws configure' command.
+AWS_PROFILE=
+# AWS account ID
+AWS_ACCOUNT_ID=
+# Path to the file containing credentials for AWS
+# (https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#shared-credentials-file)
+AWS_SHARED_CREDENTIALS_FILE=
+# The default bucket name to store projects with an 'OPEN' access policy.
+S3_OPEN_ACCESS_BUCKET=
+# The default bucket name to store logs and metrics related to project usage.
+S3_SERVER_ACCESS_LOG_BUCKET=
+
 # Datacite
 # Used to assign the DOIs
 # Changing the password can be done at the settings tab in DataCite website
@@ -98,8 +114,6 @@ GCS_SIGNED_URL_LIFETIME_IN_MINUTES=1440
 # GCP Research Environments
 ENABLE_CLOUD_RESEARCH_ENVIRONMENTS="False"
 CLOUD_RESEARCH_ENVIRONMENTS_API_URL="https://example.api"
-CLOUD_RESEARCH_ENVIRONMENTS_API_JWT_SERVICE_ACCOUNT_PATH="environment/api/tests/service_account.json"
-CLOUD_RESEARCH_ENVIRONMENTS_API_JWT_AUDIENCE="https://example.com"
 
 # Site-specific content
 SITE_NAME="DataShare"
@@ -160,6 +174,10 @@ GRADIENT_85 = 'rgba(42, 47, 52, 0.85)'
 # maximum number of emails that can be associated to a user model
 MAX_EMAILS_PER_USER = 10
 
+# maximum number of active projects that can be created by a submitting author at any time.
+# if MAX_SUBMITTABLE_PROJECTS is reached, the user must wait for a project to be archived or published before starting another.
+MAX_SUBMITTABLE_PROJECTS = 10
+
 # Max training report size in bytes
 MAX_TRAINING_REPORT_UPLOAD_SIZE = 1048576
 ENABLE_LIGHTWAVE=True
@@ -187,7 +205,7 @@ MIN_WORDS_RESEARCH_SUMMARY_CREDENTIALING = 20
 # CITISOAPService API
 # This is the WebServices username and password to access the CITI SOAP Service to obtain users training report details
 # The account can be created at https://webservices.citiprogram.org/login/CreateAccount.aspx
-# The SOAP Service Access can be tested at https://webservices.citiprogram.org/Client/CITISOAPClient_Simple.aspx 
+# The SOAP Service Access can be tested at https://webservices.citiprogram.org/Client/CITISOAPClient_Simple.aspx
 CITI_USERNAME=
 CITI_PASSWORD=
 CITI_SOAP_URL="https://webservices.citiprogram.org/SOAP/CITISOAPService.asmx"

.github/workflows/physionet-build-test.yml

@@ -14,7 +14,7 @@ jobs:
   test:
     name: Test
     runs-on: ubuntu-latest
-    container: debian:11
+    container: ${{ matrix.container }}
     env:
       DJANGO_SETTINGS_MODULE: physionet.settings.settings
       TEST_GCS_INTEGRATION: false
@@ -26,6 +26,7 @@ jobs:
       fail-fast: false
       matrix:
         pip3: ['poetry', 'requirements.txt']
+        container: ['debian:11', 'debian:12']
     steps:
 
       - name: Update packages
@@ -96,7 +97,7 @@ jobs:
 
       - name: Install and setup lightwave
         run: |
-          wget https://github.com/bemoody/lightwave/archive/0.71.tar.gz -O lightwave.tar.gz
+          wget https://github.com/bemoody/lightwave/archive/0.72.tar.gz -O lightwave.tar.gz
           tar -xf lightwave.tar.gz
           (cd lightwave-* && make CGIDIR=/usr/local/bin sandboxed-server)
 

.github/workflows/physionet-upgrade-test.yml

@@ -12,7 +12,10 @@ jobs:
   testupgrade:
     name: Upgrade Test
     runs-on: ubuntu-latest
-    container: debian:11
+    container: ${{ matrix.container }}
+    strategy:
+      matrix:
+        container: ['debian:11', 'debian:12']
     steps:
       - name: Install dependencies
         run: |
@@ -53,7 +56,7 @@ jobs:
 
       - name: Install lightwave
         run: |
-          wget https://github.com/bemoody/lightwave/archive/0.71.tar.gz \
+          wget https://github.com/bemoody/lightwave/archive/0.72.tar.gz \
                -O lightwave.tar.gz
           tar -xf lightwave.tar.gz
           (cd lightwave-* && make CGIDIR=/usr/local/bin sandboxed-server)

Dockerfile

@@ -11,7 +11,7 @@ RUN wget https://github.com/bemoody/wfdb/archive/10.7.0.tar.gz -O wfdb.tar.gz \
     && ldconfig \
     && rm -rf wfdb*
 
-RUN wget https://github.com/bemoody/lightwave/archive/0.71.tar.gz -O lightwave.tar.gz \
+RUN wget https://github.com/bemoody/lightwave/archive/0.72.tar.gz -O lightwave.tar.gz \
     && tar -xf lightwave.tar.gz \
     && (cd lightwave-* && make sandboxed-lightwave && mkdir -p /usr/local/bin && install -m 4755 sandboxed-lightwave /usr/local/bin) \
     && rm -rf lightwave*

README.md

@@ -41,6 +41,12 @@ The code should dynamically reload in development, however, if there are any iss
 
 Docker-compose uses volumes to persist the database contents and data directories (media and static files). To clean up the created containers, networks and volumes stop `docker-compose up` and run `docker-compose down -v`. Do not run `docker-compose down -v` if you want to retain current database contents.
 
+## Background tasks
+
+Background tasks are managed by [Django Q2](https://django-q2.readthedocs.io/en/master/), "a native Django task queue, scheduler and worker application using Python multiprocessing".
+
+If you would like to run background tasks on your development server, you will need to start the task manager with `python manage.py qcluster`
+
 ## Using a debugger with Docker
 
 To access a debug prompt raised using `breakpoint()`:

demo-files/aws_credentials

@@ -0,0 +1,3 @@
+[default]
+aws_access_key_id = AKIAZZZZZZZZZZZZZZZZ
+aws_secret_access_key = xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

deploy/README.md

@@ -100,16 +100,16 @@ cd /physionet
 # Copy over the .env file into /physionet/physionet-build
 scp <somewhere>/.env /physionet/physionet-build/
 # The software folder should be owned by the dedicated user. The socket file directory should be accessible by nginx.
-chown -R pn.pn /physionet
-chown pn.www-data /physionet/deploy
+chown -R pn:pn /physionet
+chown pn:www-data /physionet/deploy
 chmod g+w /physionet/deploy
 # Make the static and media roots
 mkdir /data
 mkdir /data/pn-static
 mkdir /data/pn-static/published-projects
 mkdir /data/pn-media
 mkdir /data/pn-media/{active-projects,archived-projects,credential-applications,published-projects,users}
-chown -R pn.pn /data/{pn-media,pn-static}
+chown -R pn:pn /data/{pn-media,pn-static}
 ```
 
 The directory structure for the site's software and files will be:

deploy/common/etc/sudoers.d/physionet

@@ -16,3 +16,7 @@ pn ALL=(root:root) NOPASSWD: \
 # Restart django-background-tasks.service (if it is running)
 pn ALL=(root:root) NOPASSWD: \
     /bin/systemctl try-restart django-background-tasks.service
+
+# Restart django-q2-tasks.service (if it is running)
+pn ALL=(root:root) NOPASSWD: \
+    /bin/systemctl try-restart django-q2-tasks.service

deploy/post-receive

@@ -112,6 +112,7 @@ else
 fi
 
 # Restart the django-background-tasks server
+# Background tasks will soon be migrated to Django Q2
 (
     cd $working_dir/physionet-django
     if cmp -s OLD-TARGETS LATE-TARGETS; then
@@ -130,6 +131,25 @@ fi
     fi
 )
 
+# Restart the Django Q2 server
+(
+    cd $working_dir/physionet-django
+    if cmp -s OLD-TARGETS LATE-TARGETS; then
+        # assume that if there are no migrations, no need to restart
+        # background tasks (and don't even bother showing a message)
+        :
+    elif [ -n "$no_bgtasks" ]; then
+        echo "- SKIPPING restarting django-q2-tasks due to"
+        echo "  --push-option=no-bgtasks"
+    elif [ -n "$no_install" ]; then
+        echo "- SKIPPING restarting django-q2-tasks due to"
+        echo "  --push-option=no-install"
+    else
+        echo "* Restarting django-q2-tasks..."
+        sudo systemctl try-restart django-q2-tasks.service
+    fi
+)
+
 # Run late migrations (as well as early migrations, if they were
 # skipped before)
 (

deploy/production/etc/rsyslog.d/django_q.conf

@@ -0,0 +1,3 @@
+# Filter to get all background tasks and send them to a custom location
+:programname, isequal, "django-q2-tasks" -/data/log/background_tasks/django_q_tasks.log
+& stop

deploy/production/etc/systemd/system/django-q2-tasks.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Command that runs Django Q2 tasks
+After=emperor.uwsgi.service
+
+[Service]
+Environment=DJANGO_SETTINGS_MODULE=physionet.settings.production
+ExecStart=/physionet/python-env/physionet/bin/python /physionet/physionet-build/physionet-django/manage.py qcluster
+StandardError=syslog
+SyslogIdentifier=django-q2-tasks
+Restart=always
+KillSignal=SIGINT
+Type=simple
+NotifyAccess=all
+User=www-data
+Group=www-data
+
+[Install]
+WantedBy=multi-user.target

deploy/staging/etc/rsyslog.d/django_q.conf

@@ -0,0 +1,3 @@
+# Filter to get all background tasks and send them to a custom location
+:programname, isequal, "django-q2-tasks" -/data/log/background_tasks/django_q_tasks.log
+& stop

deploy/staging/etc/systemd/system/django-q2-tasks.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Command that runs Django Q2 tasks
+After=emperor.uwsgi.service
+
+[Service]
+Environment=DJANGO_SETTINGS_MODULE=physionet.settings.staging
+ExecStart=/physionet/python-env/physionet/bin/python /physionet/physionet-build/physionet-django/manage.py qcluster
+StandardError=syslog
+SyslogIdentifier=django-q2-tasks
+Restart=always
+KillSignal=SIGINT
+Type=simple
+NotifyAccess=all
+User=www-data
+Group=www-data
+
+[Install]
+WantedBy=multi-user.target

deploy/test-server/install-pn-test-server

@@ -90,7 +90,7 @@ printf '%s\n%s\n' "$DBPASSWORD" "$DBPASSWORD" |
 su postgres -c 'createdb physionet -O physionet'
 
 mkdir -p /physionet /data/pn-static /data/pn-media
-chown pn.pn /physionet /data/pn-static /data/pn-media
+chown pn:pn /physionet /data/pn-static /data/pn-media
 su pn -c '
    set -e
    umask 0002
@@ -153,17 +153,17 @@ su pn -c '
    yes | python3 manage.py loaddemo
 '
 
-chown www-data.www-data -R /physionet/deploy
-chown www-data.www-data -R /data/pn-media
-chown www-data.www-data -R /data/pn-static/published-projects
+chown www-data:www-data -R /physionet/deploy
+chown www-data:www-data -R /data/pn-media
+chown www-data:www-data -R /data/pn-static/published-projects
 
 rm /etc/nginx/sites-enabled/default
 ln -s ../sites-available/physionet_nginx.conf \
    /etc/nginx/sites-enabled/physionet_nginx.conf
 
 mkdir /data/log /data/log/nginx /data/log/uwsgi /data/log/pn
-chown www-data.root /data/log/uwsgi
-chown pn.root /data/log/pn
+chown www-data:root /data/log/uwsgi
+chown pn:root /data/log/pn
 
 (
     cd /physionet/physionet-build/deploy/common
@@ -234,9 +234,14 @@ systemctl daemon-reload
 systemctl enable emperor.uwsgi
 systemctl restart emperor.uwsgi
 systemctl restart nginx
+
+# django-background-tasks will be replaced with Django Q2 soon
 systemctl enable django-background-tasks
 systemctl restart django-background-tasks
 
+systemctl enable django-q2-tasks
+systemctl restart django-q2-tasks
+
 su pn -c '
    cd /physionet/physionet-build.git
    echo

physionet-django/.coveragerc

@@ -0,0 +1,5 @@
+[run]
+plugins = django_coverage_plugin
+
+[django_coverage_plugin]
+template_extensions = html, txt, json, xml