[CR][ENG-5997] merge develop into b-and-i branch (CenterForOpenScienc…

…e#10691)

Bring `feature/b-and-i-24-14` up to date with `develop` (and the upgrade to python v3.12)

* Some small conflict resolutions
* One change to make a `.format()` string an f-string

This may merit a retesting, but I have reviewed the changes in the branch and they all seem pretty compatible with the py3.12 updates.

No documentation changes needed.

Devs will need to update their envs for the new py3.12 base.  That'll involve pulling images, rebuilding requirements, and maybe something else?

https://openscience.atlassian.net/browse/ENG-5997

README-docker-compose.md

@@ -402,14 +402,12 @@ resetting docker. To back up your database, follow the following sequence of com
 ```
 2. Delete a persistent storage volume:
   **WARNING: All postgres data will be destroyed.**
-```bash
-docker compose stop -t 0 postgres
-docker compose rm postgres
-docker volume rm osfio_postgres_data_vol
-```
+  - `$ docker-compose stop -t 0 postgres`
+  - `$ docker-compose rm postgres`
+  - `$ docker volume rm osfio_postgres_data_vol`
 3. Starting a new postgres container.
 ```bash
-docker compose up -d postgres
+docker-compose up -d postgres
 ```
 4.  Restoring the database from the dump file into the new postgres container.
 ```bash

addons/boa/requirements.txt

@@ -0,0 +1,5 @@
+# Requirements for the boa add-on
+boa-api==0.1.14
+
+# Requirements for running asyncio in celery, using 3.4.1 for Python 3.6 compatibility
+asgiref==3.7.2

addons/box/requirements.txt

@@ -0,0 +1 @@
+boxsdk==3.9.2

addons/dataverse/requirements.txt

@@ -0,0 +1,3 @@
+# Allow for optional timeout parameter.
+# https://github.com/IQSS/dataverse-client-python/pull/27
+git+https://github.com/CenterForOpenScience/dataverse-client-python.git@2b3827578048e6df3818f82381c7ea9a2395e526 # branch is feature/dv-client-updates

addons/dropbox/requirements.txt

@@ -0,0 +1 @@
+dropbox==11.36.2

addons/github/requirements.txt

@@ -0,0 +1,3 @@
+cachecontrol==0.14.0
+github3.py==4.0.1
+uritemplate==4.1.1

addons/gitlab/requirements.txt

@@ -0,0 +1 @@
+python-gitlab==4.4.0

addons/mendeley/requirements.txt

@@ -0,0 +1,2 @@
+# up-to-date with mendeley's master + add folder support and future dep updates
+git+https://github.com/CenterForOpenScience/mendeley-python-sdk.git@be8a811fa6c3b105d9f5c656cabb6b1ba855ed5b # branch is feature/osf-dep-updates

addons/owncloud/requirements.txt

@@ -0,0 +1,2 @@
+# Requirements for the owncloud add-on
+pyocclient==0.6.0

addons/s3/requirements.txt

@@ -0,0 +1 @@
+boto3==1.34.60

addons/twofactor/requirements.txt

@@ -0,0 +1 @@
+pyotp==2.9.0

addons/wiki/requirements.txt

@@ -0,0 +1 @@
+pymongo==4.6.3

addons/zotero/requirements.txt

@@ -0,0 +1 @@
+Pyzotero==1.5.18

api/base/utils.py

@@ -1,5 +1,5 @@
 from furl import furl
-from urllib.parse import urlunsplit, urlsplit, parse_qs, urlencode
+from urllib.parse import urlunsplit, urlsplit, parse_qs, urlencode, quote
 from packaging.version import Version
 from hashids import Hashids
 
@@ -236,7 +236,7 @@ def waterbutler_api_url_for(node_id, provider, path='/', _internal=False, base_u
     # NOTE: furl encoding to be verified later
     url = furl(website_settings.WATERBUTLER_INTERNAL_URL if _internal else (base_url or website_settings.WATERBUTLER_URL))
     segments = ['v1', 'resources', node_id, 'providers', provider] + path.split('/')[1:]
-    url.add(path=segments)
+    url.add(path=[quote(x) for x in segments])
     url.args.update(kwargs)
     return url.url
 

api/users/serializers.py

@@ -19,21 +19,16 @@
     JSONAPIListField,
     ShowIfCurrentUser,
 )
-
-from api.base.utils import default_node_list_queryset
-from osf.models import Registration, Node
-from api.base.utils import absolute_reverse, get_user_auth, is_deprecated, hashids
-from osf.models import Email
+from api.base.utils import absolute_reverse, default_node_list_queryset, get_user_auth, is_deprecated, hashids
+from api.base.versioning import get_kebab_snake_case_field
+from api.nodes.serializers import NodeSerializer, RegionRelationshipField
+from framework.auth.views import send_confirm_email_async
 from osf.exceptions import ValidationValueError, ValidationError, BlockedEmailError
-from osf.models import OSFUser, Preprint
-from osf.utils.requests import string_type_request_headers
-from website.settings import MAILCHIMP_GENERAL_LIST, OSF_HELP_LIST, CONFIRM_REGISTRATIONS_BY_EMAIL
+from osf.models import Email, Node, OSFUser, Preprint, Registration
 from osf.models.provider import AbstractProviderGroupObjectPermission
+from osf.utils.requests import string_type_request_headers
 from website.profile.views import update_osf_help_mails_subscription, update_mailchimp_subscription
-from api.nodes.serializers import NodeSerializer, RegionRelationshipField
-from api.base.schemas.utils import validate_user_json, from_json
-from framework.auth.views import send_confirm_email_async
-from api.base.versioning import get_kebab_snake_case_field
+from website.settings import MAILCHIMP_GENERAL_LIST, OSF_HELP_LIST, CONFIRM_REGISTRATIONS_BY_EMAIL
 
 
 class SocialField(ser.DictField):

api_tests/files/views/test_file_detail.py

@@ -644,7 +644,7 @@ def file(self, root_node, user):
 
     @pytest.fixture()
     def file_url(self, file):
-        return '/{}files/{}/'.format(API_BASE, file._id)
+        return f'/{API_BASE}files/{file._id}/'
 
     def test_listing(self, app, user, file):
         file.create_version(user, {

api_tests/users/views/test_user_settings.py

@@ -579,7 +579,7 @@ def test_updating_verified_for_merge(self, app, user_one, user_two, payload):
 
     @mock.patch('api.users.views.send_confirm_email_async')
     def test_resend_confirmation_email(self, mock_send_confirm_email_async, app, user_one, unconfirmed_url, confirmed_url):
-        url = '{}?resend_confirmation=True'.format(unconfirmed_url)
+        url = f'{unconfirmed_url}?resend_confirmation=True'
         res = app.get(url, auth=user_one.auth)
         assert res.status_code == 202
         assert mock_send_confirm_email_async.called

conftest.py

@@ -124,7 +124,7 @@ def _test_speedups_disable(request, settings, _test_speedups):
 
 @pytest.fixture(scope='session')
 def setup_connections():
-    connections.create_connection(hosts=[website_settings.ELASTIC6_URI])
+    connections.create_connection(hosts=['http://localhost:9201'])
 
 
 @pytest.fixture(scope='function')

docker-compose.yml

@@ -358,16 +358,15 @@ services:
       - /bin/bash
       - -c
       - python -m venv /tmp/venv
-        && /tmp/venv/bin/pip install poetry==1.8.3 &&
+        && /tmp/venv/bin/pip install poetry==1.8.0 &&
         /tmp/venv/bin/poetry install --no-root --without release --compile --sync &&
         rm -rf /python3.12/* &&
         cp -Rf -p /usr/local/lib/python3.12 /
     restart: 'no'
     environment:
       DJANGO_SETTINGS_MODULE: api.base.settings
     volumes:
-      - ./pyproject.toml:/code/pyproject.toml
-      - ./poetry.lock:/code/poetry.lock
+      - ./:/code:cached
       - osf_requirements_3_12_vol:/python3.12
 
   assets:

framework/auth/views.py

@@ -22,6 +22,7 @@
 from framework.celery_tasks.handlers import enqueue_task
 from framework.exceptions import HTTPError
 from framework.flask import redirect  # VOL-aware redirect
+from framework.postcommit_tasks.handlers import enqueue_postcommit_task
 from framework.sessions.utils import remove_sessions_for_user
 from framework.sessions import get_session
 from framework.utils import throttle_period_expired
@@ -36,8 +37,6 @@
 from website.util.metrics import CampaignClaimedTags, CampaignSourceTags
 from website.ember_osf_web.decorators import ember_flag_is_active
 from osf import features
-from framework.postcommit_tasks.handlers import enqueue_postcommit_task
-# from osf.models import PreprintProvider
 
 
 @block_bing_preview

osf/management/commands/daily_reporters_go.py

@@ -23,10 +23,17 @@ def daily_reporters_go(report_date=None, reporter_filter=None, **kwargs):
     for _reporter_key, _reporter_class in AllDailyReporters.__members__.items():
         if reporter_filter and (reporter_filter.lower() not in _reporter_class.__name__.lower()):
             continue
-        daily_reporter_go.apply_async(kwargs={
-            'reporter_key': _reporter_key,
-            'report_date': report_date.isoformat(),
-        })
+        try:
+            reporter_class().run_and_record_for_date(
+                report_date=report_date,
+                also_send_to_keen=also_send_to_keen,
+            )
+        except Exception as e:
+            errors[reporter_class.__name__] = repr(e)
+            logger.exception(e)
+            sentry.log_exception(e)
+            # continue with the next reporter
+    return errors
 
 
 @celery_app.task(

osf/management/commands/monthly_reporters_go.py

@@ -34,18 +34,16 @@ def monthly_reporters_go(report_year=None, report_month=None):
             'yearmonth': str(report_yearmonth),
         })
 
-
-@celery_app.task(
-    name='management.commands.monthly_reporter_go',
-    autoretry_for=(OperationalError,),
-    max_retries=5,
-    retry_backoff=True,
-    bind=True,
-)
-def monthly_reporter_go(task, reporter_key: str, yearmonth: str):
-    _reporter_class = AllMonthlyReporters[reporter_key].value
-    _parsed_yearmonth = YearMonth.from_str(yearmonth)
-    _reporter_class().run_and_record_for_month(_parsed_yearmonth)
+    errors = {}
+    for reporter_class in MONTHLY_REPORTERS:
+        try:
+            reporter_class().run_and_record_for_month(report_yearmonth)
+        except Exception as e:
+            errors[reporter_class.__name__] = str(e)
+            logger.exception(e)
+            sentry.log_exception(e)
+            # continue with the next reporter
+    return errors
 
 
 class Command(BaseCommand):