Merge branch 'main' into view-recordings

.github/workflows/build.yml

@@ -363,33 +363,15 @@ jobs:
 
     - name: Destroy Build VM + resources
       if: always()
-      shell: pwsh
       run: |
-        echo "Destroying VM..."        
+        echo "Terminate VM..."        
         az vm delete -g ghaDatatracker -n tmpGhaBuildVM-${{ github.run_number }} --yes --force-deletion true
-
-        # $resourceOrderRemovalOrder = [ordered]@{
-        #     "Microsoft.Compute/virtualMachines" = 0
-        #     "Microsoft.Compute/disks" = 1
-        #     "Microsoft.Network/networkInterfaces" = 2
-        #     "Microsoft.Network/publicIpAddresses" = 3
-        #     "Microsoft.Network/networkSecurityGroups" = 4
-        #     "Microsoft.Network/virtualNetworks" = 5
-        # }
-        # echo "Fetching remaining resources..."
-        # $resources = az resource list --resource-group ghaDatatracker | ConvertFrom-Json
-
-        # $orderedResources = $resources 
-        #     | Sort-Object @{
-        #         Expression = {$resourceOrderRemovalOrder[$_.type]}
-        #         Descending = $False
-        #     }
-
-        # echo "Deleting remaining resources..."
-        # $orderedResources | ForEach-Object {
-        #     az resource delete --resource-group ghaDatatracker --ids $_.id --verbose
-        # }
-
+        echo "Delete Public IP..."
+        az resource delete -g ghaDatatracker -n tmpGhaBuildVM-${{ github.run_number }}PublicIP --resource-type "Microsoft.Network/publicIPAddresses"
+        echo "Delete Network Security Group..."
+        az resource delete -g ghaDatatracker -n tmpGhaBuildVM-${{ github.run_number }}NSG --resource-type "Microsoft.Network/networkSecurityGroups"
+        echo "Delete Virtual Network..."
+        az resource delete -g ghaDatatracker -n tmpGhaBuildVM-${{ github.run_number }}VNET --resource-type "Microsoft.Network/virtualNetworks"
         echo "Logout from Azure..."
         az logout
         

.github/workflows/tests.yml

@@ -11,12 +11,12 @@ on:
       skipSelenium:
         description: 'Skip Selenium Tests'
         default: false
-        required: true
+        required: false
         type: boolean
       targetBaseVersion:
         description: 'Target Base Image Version'
         default: latest
-        required: true
+        required: false
         type: string
 
 jobs:

dev/build/gunicorn.conf.py

@@ -47,3 +47,18 @@
         }
     }
 }
+
+def pre_request(worker, req):
+    client_ip = "-"
+    cf_ray = "-"
+    for (header, value) in req.headers:
+        header = header.lower()
+        if header == "cf-connecting-ip":
+            client_ip = value
+        elif header == "cf-ray":
+            cf_ray = value
+    if req.query:
+        path = f"{req.path}?{req.query}"
+    else:
+        path = req.path
+    worker.log.info(f"gunicorn starting to process {req.method} {path} (client_ip={client_ip}, cf_ray={cf_ray})")

dev/tests/debug.sh

@@ -3,7 +3,7 @@
 # This script recreate the same environment used during tests on GitHub Actions
 # and drops you into a terminal at the point where the actual tests would be run.
 #
-# Refer to https://github.com/ietf-tools/datatracker/blob/main/.github/workflows/build.yml#L141-L155
+# Refer to https://github.com/ietf-tools/datatracker/blob/main/.github/workflows/tests.yml#L47-L66
 # for the commands to run next.
 #
 # Simply type "exit" + ENTER to exit and shutdown this test environment.

ietf/meeting/tasks.py

@@ -0,0 +1,12 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+#
+# Celery task definitions
+#
+from celery import shared_task
+
+from .views import generate_agenda_data
+
+
+@shared_task
+def agenda_data_refresh():
+    generate_agenda_data(force_refresh=True)

ietf/meeting/tests_js.py

@@ -249,7 +249,9 @@ def test_edit_meeting_schedule(self):
         self.assertTrue(s1_element.is_displayed())  # should still be displayed
         self.assertIn('hidden-parent', s1_element.get_attribute('class'),
                       'Session should be hidden when parent disabled')
-        s1_element.click()  # try to select
+
+        self.scroll_and_click((By.CSS_SELECTOR, '#session{}'.format(s1.pk)))
+
         self.assertNotIn('selected', s1_element.get_attribute('class'),
                          'Session should not be selectable when parent disabled')
 
@@ -299,9 +301,9 @@ def test_edit_meeting_schedule(self):
                         'Session s1 should have moved to second meeting day')
 
         # swap timeslot column - put session in a differently-timed timeslot
-        self.driver.find_element(By.CSS_SELECTOR,
+        self.scroll_and_click((By.CSS_SELECTOR,
             '.day .swap-timeslot-col[data-timeslot-pk="{}"]'.format(slot1b.pk)
-        ).click()  # open modal on the second timeslot for room1
+        ))  # open modal on the second timeslot for room1
         self.assertTrue(self.driver.find_element(By.CSS_SELECTOR, "#swap-timeslot-col-modal").is_displayed())
         self.driver.find_element(By.CSS_SELECTOR,
             '#swap-timeslot-col-modal input[name="target_timeslot"][value="{}"]'.format(slot4.pk)
@@ -1373,13 +1375,8 @@ def test_upcoming_materials_modal(self):
         self.assertFalse(modal_div.is_displayed())
 
         # Click the 'materials' button
-        open_modal_button = self.wait.until(
-            expected_conditions.element_to_be_clickable(
-                (By.CSS_SELECTOR, '[data-bs-target="#modal-%s"]' % slug)
-            ),
-            'Modal open button not found or not clickable',
-        )
-        open_modal_button.click()
+        open_modal_button_locator = (By.CSS_SELECTOR, '[data-bs-target="#modal-%s"]' % slug)
+        self.scroll_and_click(open_modal_button_locator)
         self.wait.until(
             expected_conditions.visibility_of(modal_div),
             'Modal did not become visible after clicking open button',

ietf/meeting/tests_views.py

@@ -26,6 +26,7 @@
 from django.urls import reverse as urlreverse
 from django.conf import settings
 from django.contrib.auth.models import User
+from django.core.serializers.json import DjangoJSONEncoder
 from django.test import Client, override_settings
 from django.db.models import F, Max
 from django.http import QueryDict, FileResponse
@@ -49,7 +50,7 @@
 from ietf.meeting.utils import add_event_info_to_session_qs, participants_for_meeting
 from ietf.meeting.utils import create_recording, get_next_sequence, bluesheet_data
 from ietf.meeting.views import session_draft_list, parse_agenda_filter_params, sessions_post_save, agenda_extract_schedule
-from ietf.meeting.views import get_summary_by_area, get_summary_by_type, get_summary_by_purpose
+from ietf.meeting.views import get_summary_by_area, get_summary_by_type, get_summary_by_purpose, generate_agenda_data
 from ietf.name.models import SessionStatusName, ImportantDateName, RoleName, ProceedingsMaterialTypeName
 from ietf.utils.decorators import skip_coverage
 from ietf.utils.mail import outbox, empty_outbox, get_payload_text
@@ -125,8 +126,12 @@ def tearDown(self):
         settings.MEETINGHOST_LOGO_PATH = self.saved_meetinghost_logo_path
         super().tearDown()
 
-    def write_materials_file(self, meeting, doc, content, charset="utf-8"):
-        path = os.path.join(self.materials_dir, "%s/%s/%s" % (meeting.number, doc.type_id, doc.uploaded_filename))
+    def write_materials_file(self, meeting, doc, content, charset="utf-8", with_ext=None):
+        if with_ext is None:
+            filename = doc.uploaded_filename
+        else:
+            filename = Path(doc.uploaded_filename).with_suffix(with_ext)
+        path = os.path.join(self.materials_dir, "%s/%s/%s" % (meeting.number, doc.type_id, filename))
 
         dirname = os.path.dirname(path)
         if not os.path.exists(dirname):
@@ -241,30 +246,34 @@ def test_meeting_agenda(self):
 
         # Agenda API tests
         # -> Meeting data
-        r = self.client.get(urlreverse("ietf.meeting.views.api_get_agenda_data", kwargs=dict(num=meeting.number)))
-        self.assertEqual(r.status_code, 200)  
-        rjson = json.loads(r.content.decode("utf8"))
-        self.assertJSONEqual(
-            r.content.decode("utf8"),
+        # First, check that the generation function does the right thing
+        generated_data = generate_agenda_data(meeting.number)
+        self.assertEqual(
+            generated_data,
             {
                 "meeting": {
                     "number": meeting.number,
                     "city": meeting.city,
                     "startDate": meeting.date.isoformat(),
                     "endDate": meeting.end_date().isoformat(),
-                    "updated": rjson.get("meeting").get("updated"), # Just expect the value to exist
+                    "updated": generated_data.get("meeting").get("updated"),  # Just expect the value to exist
                     "timezone": meeting.time_zone,
                     "infoNote": meeting.agenda_info_note,
                     "warningNote": meeting.agenda_warning_note
                 },
-                "categories": rjson.get("categories"), # Just expect the value to exist
+                "categories": generated_data.get("categories"),  # Just expect the value to exist
                 "isCurrentMeeting": True,
-                "usesNotes": False, # make_meeting_test_data sets number=72
-                "schedule": rjson.get("schedule"), # Just expect the value to exist
+                "usesNotes": False,  # make_meeting_test_data sets number=72
+                "schedule": generated_data.get("schedule"),  # Just expect the value to exist
                 "floors": []
             }
         )
-        # -> Session Materials
+        with patch("ietf.meeting.views.generate_agenda_data", return_value=generated_data):
+            r = self.client.get(urlreverse("ietf.meeting.views.api_get_agenda_data", kwargs=dict(num=meeting.number)))
+        self.assertEqual(r.status_code, 200)  
+        # json.dumps using the DjangoJSONEncoder to handle timestamps consistently
+        self.assertJSONEqual(r.content.decode("utf8"), json.dumps(generated_data, cls=DjangoJSONEncoder))
+        # -> Session MaterialM
         r = self.client.get(urlreverse("ietf.meeting.views.api_get_session_materials", kwargs=dict(session_id=session.id)))
         self.assertEqual(r.status_code, 200)  
         rjson = json.loads(r.content.decode("utf8"))
@@ -787,7 +796,56 @@ def test_materials_has_edit_links(self):
         )
         self.assertEqual(len(q(f'a[href^="{edit_url}#session"]')), 1, f'Link to session_details page for {acro}')
 
+    def test_materials_document_extension_choice(self):
+        def _url(**kwargs):
+            return urlreverse("ietf.meeting.views.materials_document", kwargs=kwargs)
+
+        presentation = SessionPresentationFactory(
+            document__rev="00",
+            document__name="slides-whatever",
+            document__uploaded_filename="slides-whatever-00.txt",
+            document__type_id="slides",
+            document__states=(("reuse_policy", "single"),)
+        )
+        session = presentation.session
+        meeting = session.meeting
+        # This is not a realistic set of files to exist, but is useful for testing. Normally,
+        # we'd have _either_ txt, pdf, or pptx + pdf.
+        self.write_materials_file(meeting, presentation.document, "Hi I'm a txt", with_ext=".txt")
+        self.write_materials_file(meeting, presentation.document, "Hi I'm a pptx", with_ext=".pptx")
 
+        # with no rev, prefers the uploaded_filename
+        r = self.client.get(_url(document="slides-whatever", num=meeting.number))  # no rev
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.content.decode(), "Hi I'm a txt")
+
+        # with a rev, prefers pptx because it comes first alphabetically
+        r = self.client.get(_url(document="slides-whatever-00", num=meeting.number))
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.content.decode(), "Hi I'm a pptx")
+
+        # now create a pdf
+        self.write_materials_file(meeting, presentation.document, "Hi I'm a pdf", with_ext=".pdf")
+
+        # with no rev, still prefers uploaded_filename
+        r = self.client.get(_url(document="slides-whatever", num=meeting.number))  # no rev
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.content.decode(), "Hi I'm a txt")
+
+        # pdf should be preferred with a rev
+        r = self.client.get(_url(document="slides-whatever-00", num=meeting.number))
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.content.decode(), "Hi I'm a pdf")
+
+        # and explicit extensions should, of course, be respected
+        for ext in ["pdf", "pptx", "txt"]:
+            r = self.client.get(_url(document="slides-whatever-00", num=meeting.number, ext=f".{ext}"))
+            self.assertEqual(r.status_code, 200)
+            self.assertEqual(r.content.decode(), f"Hi I'm a {ext}")
+
+        # and 404 should come up if the ext is not found
+        r = self.client.get(_url(document="slides-whatever-00", num=meeting.number, ext=".docx"))
+        self.assertEqual(r.status_code, 404)
 
     def test_materials_editable_groups(self):
         meeting = make_meeting_test_data()

ietf/meeting/urls.py

@@ -83,7 +83,7 @@ def get_redirect_url(self, *args, **kwargs):
     url(r'^week-view(?:.html)?/?$', AgendaRedirectView.as_view(pattern_name='agenda', permanent=True)),
     url(r'^materials(?:.html)?/?$', views.materials),
     url(r'^request_minutes/?$', views.request_minutes),
-    url(r'^materials/%(document)s((?P<ext>\.[a-z0-9]+)|/)?$' % settings.URL_REGEXPS, views.materials_document),
+    url(r'^materials/%(document)s(?P<ext>\.[a-z0-9]+)?/?$' % settings.URL_REGEXPS, views.materials_document),
     url(r'^session/?$', views.materials_editable_groups),
     url(r'^proceedings(?:.html)?/?$', views.proceedings),
     url(r'^proceedings(?:.html)?/finalize/?$', views.finalize_proceedings),