diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8f88c7c22e..3887661cac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -335,7 +335,7 @@ jobs: steps: - name: Notify on Slack (Success) if: ${{ !contains(join(needs.*.result, ','), 'failure') }} - uses: slackapi/slack-github-action@v1.26.0 + uses: slackapi/slack-github-action@v1.27.0 with: channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} payload: | @@ -358,7 +358,7 @@ jobs: SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }} - name: Notify on Slack (Failure) if: ${{ contains(join(needs.*.result, ','), 'failure') }} - uses: slackapi/slack-github-action@v1.26.0 + uses: slackapi/slack-github-action@v1.27.0 with: channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} payload: | diff --git a/.github/workflows/tests-az.yml b/.github/workflows/tests-az.yml index bfd1fdd584..3f828430a9 100644 --- a/.github/workflows/tests-az.yml +++ b/.github/workflows/tests-az.yml @@ -38,7 +38,7 @@ jobs: ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts - name: Remote SSH into VM - uses: appleboy/ssh-action@029f5b4aeeeb58fdfe1410a5d17f967dacf36262 + uses: appleboy/ssh-action@25ce8cbbcb08177468c7ff7ec5cbfa236f9341e1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 616ffdcc63..64db03db09 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -59,7 +59,7 @@ jobs: path: geckodriver.log - name: Upload Coverage Results to Codecov - uses: codecov/codecov-action@v4.5.0 + uses: codecov/codecov-action@v4.6.0 with: files: coverage.xml diff --git a/dev/coverage-action/package-lock.json b/dev/coverage-action/package-lock.json index 8c9b97e026..f23a9b66c7 100644 --- a/dev/coverage-action/package-lock.json +++ b/dev/coverage-action/package-lock.json @@ -9,19 +9,27 @@ "version": "1.0.0", "license": "BSD-3-Clause", "dependencies": { - "@actions/core": "1.10.1", + "@actions/core": "1.11.1", "@actions/github": "6.0.0", "lodash": "4.17.21", - "luxon": "3.4.4" + "luxon": "3.5.0" } }, "node_modules/@actions/core": { - "version": "1.10.1", - "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.1.tgz", - "integrity": "sha512-3lBR9EDAY+iYIpTnTIXmWcNbX3T2kCkAEQGIQx4NVQ0575nk2k3GRZDTPQG+vVtS2izSLmINlxXf0uLtnrTP+g==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.11.1.tgz", + "integrity": "sha512-hXJCSrkwfA46Vd9Z3q4cpEpHB1rL5NG04+/rbqW9d3+CSvtB1tYe8UTpAlixa1vj0m/ULglfEK2UKxMGxCxv5A==", "dependencies": { - "@actions/http-client": "^2.0.1", - "uuid": "^8.3.2" + "@actions/exec": "^1.1.1", + "@actions/http-client": "^2.0.1" + } + }, + "node_modules/@actions/exec": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.1.1.tgz", + "integrity": "sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==", + "dependencies": { + "@actions/io": "^1.0.1" } }, "node_modules/@actions/github": { @@ -44,6 +52,11 @@ "undici": "^5.25.4" } }, + "node_modules/@actions/io": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.1.3.tgz", + "integrity": "sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q==" + }, "node_modules/@fastify/busboy": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.0.0.tgz", @@ -196,9 +209,9 @@ "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "node_modules/luxon": { - "version": "3.4.4", - "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", - "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==", + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz", + "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==", "engines": { "node": ">=12" } @@ -235,14 +248,6 @@ "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==" }, - "node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -251,12 +256,20 @@ }, "dependencies": { "@actions/core": { - "version": "1.10.1", - "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.1.tgz", - "integrity": "sha512-3lBR9EDAY+iYIpTnTIXmWcNbX3T2kCkAEQGIQx4NVQ0575nk2k3GRZDTPQG+vVtS2izSLmINlxXf0uLtnrTP+g==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.11.1.tgz", + "integrity": "sha512-hXJCSrkwfA46Vd9Z3q4cpEpHB1rL5NG04+/rbqW9d3+CSvtB1tYe8UTpAlixa1vj0m/ULglfEK2UKxMGxCxv5A==", "requires": { - "@actions/http-client": "^2.0.1", - "uuid": "^8.3.2" + "@actions/exec": "^1.1.1", + "@actions/http-client": "^2.0.1" + } + }, + "@actions/exec": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.1.1.tgz", + "integrity": "sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==", + "requires": { + "@actions/io": "^1.0.1" } }, "@actions/github": { @@ -279,6 +292,11 @@ "undici": "^5.25.4" } }, + "@actions/io": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.1.3.tgz", + "integrity": "sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q==" + }, "@fastify/busboy": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.0.0.tgz", @@ -395,9 +413,9 @@ "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "luxon": { - "version": "3.4.4", - "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", - "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==" + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz", + "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==" }, "once": { "version": "1.4.0", @@ -425,11 +443,6 @@ "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==" }, - "uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" - }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/dev/coverage-action/package.json b/dev/coverage-action/package.json index 0a8794d2a2..1d03fe3dab 100644 --- a/dev/coverage-action/package.json +++ b/dev/coverage-action/package.json @@ -6,9 +6,9 @@ "author": "IETF Trust", "license": "BSD-3-Clause", "dependencies": { - "@actions/core": "1.10.1", + "@actions/core": "1.11.1", "@actions/github": "6.0.0", "lodash": "4.17.21", - "luxon": "3.4.4" + "luxon": "3.5.0" } } diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json index 7b86986d56..f636a45f05 100644 --- a/dev/deploy-to-container/package-lock.json +++ b/dev/deploy-to-container/package-lock.json @@ -11,7 +11,7 @@ "nanoid": "5.0.7", "nanoid-dictionary": "5.0.0-beta.1", "slugify": "1.6.6", - "tar": "^7.4.0", + "tar": "^7.4.3", "yargs": "^17.7.2" }, "engines": { @@ -788,9 +788,9 @@ } }, "node_modules/tar": { - "version": "7.4.0", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.0.tgz", - "integrity": "sha512-XQs0S8fuAkQWuqhDeCdMlJXDX80D7EOVLDPVFkna9yQfzS+PHKgfxcei0jf6/+QAWcjqrnC8uM3fSAnrQl+XYg==", + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", + "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", @@ -1503,9 +1503,9 @@ } }, "tar": { - "version": "7.4.0", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.0.tgz", - "integrity": "sha512-XQs0S8fuAkQWuqhDeCdMlJXDX80D7EOVLDPVFkna9yQfzS+PHKgfxcei0jf6/+QAWcjqrnC8uM3fSAnrQl+XYg==", + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", + "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", "requires": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json index 2f582dd2a7..be77fa5cce 100644 --- a/dev/deploy-to-container/package.json +++ b/dev/deploy-to-container/package.json @@ -7,7 +7,7 @@ "nanoid": "5.0.7", "nanoid-dictionary": "5.0.0-beta.1", "slugify": "1.6.6", - "tar": "^7.4.0", + "tar": "^7.4.3", "yargs": "^17.7.2" }, "engines": { diff --git a/dev/diff/package-lock.json b/dev/diff/package-lock.json index 3500ccec48..87aaf3f16f 100644 --- a/dev/diff/package-lock.json +++ b/dev/diff/package-lock.json @@ -15,9 +15,9 @@ "keypress": "^0.2.1", "listr2": "^6.6.1", "lodash-es": "^4.17.21", - "luxon": "^3.4.4", + "luxon": "^3.5.0", "pretty-bytes": "^6.1.1", - "tar": "^7.4.0", + "tar": "^7.4.3", "yargs": "^17.7.2" }, "engines": { @@ -1060,9 +1060,9 @@ } }, "node_modules/luxon": { - "version": "3.4.4", - "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", - "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==", + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz", + "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==", "engines": { "node": ">=12" } @@ -1493,9 +1493,9 @@ } }, "node_modules/tar": { - "version": "7.4.0", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.0.tgz", - "integrity": "sha512-XQs0S8fuAkQWuqhDeCdMlJXDX80D7EOVLDPVFkna9yQfzS+PHKgfxcei0jf6/+QAWcjqrnC8uM3fSAnrQl+XYg==", + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", + "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", @@ -2410,9 +2410,9 @@ "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==" }, "luxon": { - "version": "3.4.4", - "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", - "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==" + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz", + "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==" }, "mimic-fn": { "version": "2.1.0", @@ -2691,9 +2691,9 @@ } }, "tar": { - "version": "7.4.0", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.0.tgz", - "integrity": "sha512-XQs0S8fuAkQWuqhDeCdMlJXDX80D7EOVLDPVFkna9yQfzS+PHKgfxcei0jf6/+QAWcjqrnC8uM3fSAnrQl+XYg==", + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", + "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", "requires": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", diff --git a/dev/diff/package.json b/dev/diff/package.json index 1b5540e346..4e0e1be8f4 100644 --- a/dev/diff/package.json +++ b/dev/diff/package.json @@ -11,9 +11,9 @@ "keypress": "^0.2.1", "listr2": "^6.6.1", "lodash-es": "^4.17.21", - "luxon": "^3.4.4", + "luxon": "^3.5.0", "pretty-bytes": "^6.1.1", - "tar": "^7.4.0", + "tar": "^7.4.3", "yargs": "^17.7.2" }, "engines": { diff --git a/ietf/api/urls.py b/ietf/api/urls.py index 396b3813d6..48525dfda2 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -59,7 +59,7 @@ # Email alias listing url(r'^person/email/$', api_views.active_email_list), # Draft submission API - url(r'^submit/?$', submit_views.api_submit), + url(r'^submit/?$', submit_views.api_submit_tombstone), # Draft upload API url(r'^submission/?$', submit_views.api_submission), # Draft submission state API diff --git a/ietf/iesg/tests.py b/ietf/iesg/tests.py index 42e838ebdf..86910bc0ce 100644 --- a/ietf/iesg/tests.py +++ b/ietf/iesg/tests.py @@ -87,9 +87,18 @@ def test_milestones_needing_review_ordering(self): group=dated_group, person=Person.objects.get(user__username='ad'), ) - dated_milestones = DatedGroupMilestoneFactory.create_batch( - 2, group=dated_group, state_id="review" - ) + dated_milestones = [ + DatedGroupMilestoneFactory( + group=dated_group, + state_id="review", + desc="This is the description of one dated group milestone", + ), + DatedGroupMilestoneFactory( + group=dated_group, + state_id="review", + desc="This is the description of another dated group milestone", + ), + ] dated_milestones[0].due -= datetime.timedelta(days=1) # make this one earlier dated_milestones[0].save() @@ -99,9 +108,18 @@ def test_milestones_needing_review_ordering(self): group=dateless_group, person=Person.objects.get(user__username='ad'), ) - dateless_milestones = DatelessGroupMilestoneFactory.create_batch( - 2, group=dateless_group, state_id="review" - ) + dateless_milestones = [ + DatelessGroupMilestoneFactory( + group=dateless_group, + state_id="review", + desc="This is the description of one dateless group milestone", + ), + DatelessGroupMilestoneFactory( + group=dateless_group, + state_id="review", + desc="This is the description of another dateless group milestone", + ), + ] url = urlreverse("ietf.iesg.views.milestones_needing_review") self.client.login(username="ad", password="ad+password") @@ -111,17 +129,29 @@ def test_milestones_needing_review_ordering(self): # check order-by-date dated_tbody = pq(f'td:contains("{dated_milestones[0].desc}")').closest("tbody") - next_td = dated_tbody.find('td:contains("Next")') - self.assertEqual(next_td.siblings()[0].text.strip(), dated_milestones[0].desc) - last_td = dated_tbody.find('td:contains("Last")') - self.assertEqual(last_td.siblings()[0].text.strip(), dated_milestones[1].desc) + rows = list(dated_tbody.items("tr")) # keep as pyquery objects + self.assertTrue(rows[0].find('td:first:contains("Last")')) # Last milestone shown first + self.assertFalse(rows[0].find('td:first:contains("Next")')) + self.assertTrue(rows[0].find(f'td:contains("{dated_milestones[1].desc}")')) + self.assertFalse(rows[0].find(f'td:contains("{dated_milestones[0].desc}")')) + + self.assertFalse(rows[1].find('td:first:contains("Last")')) # Last milestone shown first + self.assertTrue(rows[1].find('td:first:contains("Next")')) + self.assertFalse(rows[1].find(f'td:contains("{dated_milestones[1].desc}")')) + self.assertTrue(rows[1].find(f'td:contains("{dated_milestones[0].desc}")')) # check order-by-order dateless_tbody = pq(f'td:contains("{dateless_milestones[0].desc}")').closest("tbody") - next_td = dateless_tbody.find('td:contains("Next")') - self.assertEqual(next_td.siblings()[0].text.strip(), dateless_milestones[0].desc) - last_td = dateless_tbody.find('td:contains("Last")') - self.assertEqual(last_td.siblings()[0].text.strip(), dateless_milestones[1].desc) + rows = list(dateless_tbody.items("tr")) # keep as pyquery objects + self.assertTrue(rows[0].find('td:first:contains("Last")')) # Last milestone shown first + self.assertFalse(rows[0].find('td:first:contains("Next")')) + self.assertTrue(rows[0].find(f'td:contains("{dateless_milestones[1].desc}")')) + self.assertFalse(rows[0].find(f'td:contains("{dateless_milestones[0].desc}")')) + + self.assertFalse(rows[1].find('td:first:contains("Last")')) # Last milestone shown first + self.assertTrue(rows[1].find('td:first:contains("Next")')) + self.assertFalse(rows[1].find(f'td:contains("{dateless_milestones[1].desc}")')) + self.assertTrue(rows[1].find(f'td:contains("{dateless_milestones[0].desc}")')) def test_review_decisions(self): diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 211cdec9a5..6e6c4dfe23 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -1840,7 +1840,7 @@ def agenda_extract_slide(item): "id": item.id, "title": item.title, "rev": item.rev, - "url": item.get_versionless_href(), + "url": item.get_href(), "ext": item.file_extension(), } diff --git a/ietf/person/name.py b/ietf/person/name.py index dc57f58f4b..0dbeaa9b99 100644 --- a/ietf/person/name.py +++ b/ietf/person/name.py @@ -59,7 +59,7 @@ def name_parts(name): last = parts[0] if len(parts) >= 2: # Handle reverse-order names with uppercase surname correctly - if len(first)>1 and re.search("^[A-Z-]+$", first): + if len(first)>1 and re.search("^[A-Z-]+$", first) and first != "JP": first, last = last, first.capitalize() # Handle exception for RFC Editor if (prefix, first, middle, last, suffix) == ('', 'Editor', '', 'Rfc', ''): diff --git a/ietf/secr/sreq/tests.py b/ietf/secr/sreq/tests.py index 7fb13f1796..847b993e1c 100644 --- a/ietf/secr/sreq/tests.py +++ b/ietf/secr/sreq/tests.py @@ -13,9 +13,10 @@ from ietf.meeting.models import Session, ResourceAssociation, SchedulingEvent, Constraint from ietf.meeting.factories import MeetingFactory, SessionFactory from ietf.name.models import ConstraintName, TimerangeName +from ietf.person.factories import PersonFactory from ietf.person.models import Person from ietf.secr.sreq.forms import SessionForm -from ietf.utils.mail import outbox, empty_outbox, get_payload_text +from ietf.utils.mail import outbox, empty_outbox, get_payload_text, send_mail from ietf.utils.timezone import date_today @@ -78,6 +79,32 @@ def test_cancel(self): self.assertRedirects(r,reverse('ietf.secr.sreq.views.main')) self.assertEqual(SchedulingEvent.objects.filter(session=session).order_by('-id')[0].status_id, 'deleted') + def test_cancel_notification_msg(self): + to = "" + subject = "Dummy subject" + template = "sreq/session_cancel_notification.txt" + meeting = MeetingFactory(type_id="ietf", date=date_today()) + requester = PersonFactory(name="James O'Rourke", user__username="jimorourke") + context = {"meeting": meeting, "requester": requester} + cc = "cc.a@example.com, cc.b@example.com" + bcc = "bcc@example.com" + + msg = send_mail( + None, + to, + None, + subject, + template, + context, + cc=cc, + bcc=bcc, + ) + self.assertEqual(requester.name, "James O'Rourke") # note ' (single quote) in the name + self.assertIn( + f"A request to cancel a meeting session has just been submitted by {requester.name}.", + get_payload_text(msg), + ) + def test_edit(self): meeting = MeetingFactory(type_id='ietf', date=date_today()) mars = RoleFactory(name_id='chair', person__user__username='marschairman', group__acronym='mars').group @@ -701,6 +728,33 @@ def test_request_notification(self): self.assertNotIn('1 Hour, 1 Hour, 1 Hour', notification_payload) self.assertNotIn('The third session requires your approval', notification_payload) + def test_request_notification_msg(self): + to = "" + subject = "Dummy subject" + template = "sreq/session_request_notification.txt" + header = "A new" + meeting = MeetingFactory(type_id="ietf", date=date_today()) + requester = PersonFactory(name="James O'Rourke", user__username="jimorourke") + context = {"header": header, "meeting": meeting, "requester": requester} + cc = "cc.a@example.com, cc.b@example.com" + bcc = "bcc@example.com" + + msg = send_mail( + None, + to, + None, + subject, + template, + context, + cc=cc, + bcc=bcc, + ) + self.assertEqual(requester.name, "James O'Rourke") # note ' (single quote) in the name + self.assertIn( + f"{header} meeting session request has just been submitted by {requester.name}.", + get_payload_text(msg), + ) + def test_request_notification_third_session(self): meeting = MeetingFactory(type_id='ietf', date=date_today()) ad = Person.objects.get(user__username='ad') diff --git a/ietf/secr/templates/sreq/session_cancel_notification.txt b/ietf/secr/templates/sreq/session_cancel_notification.txt index 3e6dd43f69..8aee6c89db 100644 --- a/ietf/secr/templates/sreq/session_cancel_notification.txt +++ b/ietf/secr/templates/sreq/session_cancel_notification.txt @@ -1,4 +1,3 @@ -{% load ams_filters %} - -A request to cancel a meeting session has just been submitted by {{ requester }}. +{% autoescape off %}{% load ams_filters %} +A request to cancel a meeting session has just been submitted by {{ requester }}.{% endautoescape %} diff --git a/ietf/secr/templates/sreq/session_request_notification.txt b/ietf/secr/templates/sreq/session_request_notification.txt index a41f202447..75f2cbbae4 100644 --- a/ietf/secr/templates/sreq/session_request_notification.txt +++ b/ietf/secr/templates/sreq/session_request_notification.txt @@ -1,5 +1,5 @@ -{% load ams_filters %} +{% autoescape off %}{% load ams_filters %} {% filter wordwrap:78 %}{{ header }} meeting session request has just been submitted by {{ requester }}.{% endfilter %} -{% include "includes/session_info.txt" %} +{% include "includes/session_info.txt" %}{% endautoescape %} diff --git a/ietf/static/css/document_html_txt.scss b/ietf/static/css/document_html_txt.scss index b0fec7c4d6..a5991056c9 100644 --- a/ietf/static/css/document_html_txt.scss +++ b/ietf/static/css/document_html_txt.scss @@ -344,7 +344,7 @@ div:is(.artwork, .sourcecode) pre { flex: 0 0 content; margin: 0; max-width: 72ch; - overflow: auto; + overflow: auto clip; } div:is(.artwork, .sourcecode) .pilcrow { flex: 0 0 1ch; diff --git a/ietf/submit/forms.py b/ietf/submit/forms.py index 4e5644b36e..2781d3365a 100644 --- a/ietf/submit/forms.py +++ b/ietf/submit/forms.py @@ -2,19 +2,16 @@ # -*- coding: utf-8 -*- -import io import os import re import datetime import email import sys import tempfile -import xml2rfc from contextlib import ExitStack from email.utils import formataddr from typing import Tuple -from unidecode import unidecode from django import forms from django.conf import settings @@ -37,10 +34,8 @@ from ietf.submit.utils import validate_submission_name, validate_submission_rev, validate_submission_document_date, remote_ip from ietf.submit.parsers.plain_parser import PlainParser from ietf.submit.parsers.xml_parser import XMLParser -from ietf.utils import log from ietf.utils.draft import PlaintextDraft from ietf.utils.fields import ModelMultipleChoiceField -from ietf.utils.text import normalize_text from ietf.utils.timezone import date_today from ietf.utils.xmldraft import InvalidXMLError, XMLDraft, XMLParseError @@ -371,273 +366,6 @@ def deduce_group(name): return None -class DeprecatedSubmissionBaseUploadForm(SubmissionBaseUploadForm): - def clean(self): - def format_messages(where, e, log): - out = log.write_out.getvalue().splitlines() - err = log.write_err.getvalue().splitlines() - m = str(e) - if m: - m = [ m ] - else: - import traceback - typ, val, tb = sys.exc_info() - m = traceback.format_exception(typ, val, tb) - m = [ l.replace('\n ', ':\n ') for l in m ] - msgs = [s for s in (["Error from xml2rfc (%s):" % (where,)] + m + out + err) if s] - return msgs - - if self.shutdown and not has_role(self.request.user, "Secretariat"): - raise forms.ValidationError(self.cutoff_warning) - - for ext in self.formats: - f = self.cleaned_data.get(ext, None) - if not f: - continue - self.file_types.append('.%s' % ext) - if not ('.txt' in self.file_types or '.xml' in self.file_types): - if not self.errors: - raise forms.ValidationError('Unexpected submission file types; found %s, but %s is required' % (', '.join(self.file_types), ' or '.join(self.base_formats))) - - #debug.show('self.cleaned_data["xml"]') - if self.cleaned_data.get('xml'): - #if not self.cleaned_data.get('txt'): - xml_file = self.cleaned_data.get('xml') - file_name = {} - xml2rfc.log.write_out = io.StringIO() # open(os.devnull, "w") - xml2rfc.log.write_err = io.StringIO() # open(os.devnull, "w") - tfn = None - with ExitStack() as stack: - @stack.callback - def cleanup(): # called when context exited, even in case of exception - if tfn is not None: - os.unlink(tfn) - - # We need to write the xml file to disk in order to hand it - # over to the xml parser. XXX FIXME: investigate updating - # xml2rfc to be able to work with file handles to in-memory - # files. - name, ext = os.path.splitext(os.path.basename(xml_file.name)) - with tempfile.NamedTemporaryFile(prefix=name+'-', - suffix='.xml', - mode='wb+', - delete=False) as tf: - tfn = tf.name - for chunk in xml_file.chunks(): - tf.write(chunk) - - parser = xml2rfc.XmlRfcParser(str(tfn), quiet=True) - # --- Parse the xml --- - try: - self.xmltree = parser.parse(remove_comments=False) - # If we have v2, run it through v2v3. Keep track of the submitted version, though. - self.xmlroot = self.xmltree.getroot() - self.xml_version = self.xmlroot.get('version', '2') - if self.xml_version == '2': - v2v3 = xml2rfc.V2v3XmlWriter(self.xmltree) - self.xmltree.tree = v2v3.convert2to3() - self.xmlroot = self.xmltree.getroot() # update to the new root - - draftname = self.xmlroot.attrib.get('docName') - if draftname is None: - self.add_error('xml', "No docName attribute found in the xml root element") - name_error = validate_submission_name(draftname) - if name_error: - self.add_error('xml', name_error) # This is a critical and immediate failure - do not proceed with other validation. - else: - revmatch = re.search("-[0-9][0-9]$", draftname) - if revmatch: - self.revision = draftname[-2:] - self.filename = draftname[:-3] - else: - self.revision = None - self.filename = draftname - self.title = self.xmlroot.findtext('front/title').strip() - if type(self.title) is str: - self.title = unidecode(self.title) - self.title = normalize_text(self.title) - self.abstract = (self.xmlroot.findtext('front/abstract') or '').strip() - if type(self.abstract) is str: - self.abstract = unidecode(self.abstract) - author_info = self.xmlroot.findall('front/author') - for author in author_info: - info = { - "name": author.attrib.get('fullname'), - "email": author.findtext('address/email'), - "affiliation": author.findtext('organization'), - } - elem = author.find('address/postal/country') - if elem != None: - ascii_country = elem.get('ascii', None) - info['country'] = ascii_country if ascii_country else elem.text - - for item in info: - if info[item]: - info[item] = info[item].strip() - self.authors.append(info) - - # --- Prep the xml --- - file_name['xml'] = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s%s' % (self.filename, self.revision, ext)) - try: - prep = xml2rfc.PrepToolWriter(self.xmltree, quiet=True, liberal=True, keep_pis=[xml2rfc.V3_PI_TARGET]) - prep.options.accept_prepped = True - self.xmltree.tree = prep.prep() - if self.xmltree.tree == None: - self.add_error('xml', "Error from xml2rfc (prep): %s" % prep.errors) - except Exception as e: - msgs = format_messages('prep', e, xml2rfc.log) - self.add_error('xml', msgs) - - # --- Convert to txt --- - if not ('txt' in self.cleaned_data and self.cleaned_data['txt']): - file_name['txt'] = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.txt' % (self.filename, self.revision)) - try: - writer = xml2rfc.TextWriter(self.xmltree, quiet=True) - writer.options.accept_prepped = True - writer.write(file_name['txt']) - log.log("In %s: xml2rfc %s generated %s from %s (version %s)" % - ( os.path.dirname(file_name['xml']), - xml2rfc.__version__, - os.path.basename(file_name['txt']), - os.path.basename(file_name['xml']), - self.xml_version)) - except Exception as e: - msgs = format_messages('txt', e, xml2rfc.log) - log.log('\n'.join(msgs)) - self.add_error('xml', msgs) - - # --- Convert to html --- - try: - file_name['html'] = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.html' % (self.filename, self.revision)) - writer = xml2rfc.HtmlWriter(self.xmltree, quiet=True) - writer.write(file_name['html']) - self.file_types.append('.html') - log.log("In %s: xml2rfc %s generated %s from %s (version %s)" % - ( os.path.dirname(file_name['xml']), - xml2rfc.__version__, - os.path.basename(file_name['html']), - os.path.basename(file_name['xml']), - self.xml_version)) - except Exception as e: - msgs = format_messages('html', e, xml2rfc.log) - self.add_error('xml', msgs) - - except Exception as e: - try: - msgs = format_messages('txt', e, xml2rfc.log) - log.log('\n'.join(msgs)) - self.add_error('xml', msgs) - except Exception: - self.add_error('xml', "An exception occurred when trying to process the XML file: %s" % e) - - # The following errors are likely noise if we have previous field - # errors: - if self.errors: - raise forms.ValidationError('') - - if self.cleaned_data.get('txt'): - # try to parse it - txt_file = self.cleaned_data['txt'] - txt_file.seek(0) - bytes = txt_file.read() - txt_file.seek(0) - try: - text = bytes.decode(PlainParser.encoding) - self.parsed_draft = PlaintextDraft(text, txt_file.name) - if self.filename == None: - self.filename = self.parsed_draft.filename - elif self.filename != self.parsed_draft.filename: - self.add_error('txt', "Inconsistent name information: xml:%s, txt:%s" % (self.filename, self.parsed_draft.filename)) - if self.revision == None: - self.revision = self.parsed_draft.revision - elif self.revision != self.parsed_draft.revision: - self.add_error('txt', "Inconsistent revision information: xml:%s, txt:%s" % (self.revision, self.parsed_draft.revision)) - if self.title == None: - self.title = self.parsed_draft.get_title() - elif self.title != self.parsed_draft.get_title(): - self.add_error('txt', "Inconsistent title information: xml:%s, txt:%s" % (self.title, self.parsed_draft.get_title())) - except (UnicodeDecodeError, LookupError) as e: - self.add_error('txt', 'Failed decoding the uploaded file: "%s"' % str(e)) - - rev_error = validate_submission_rev(self.filename, self.revision) - if rev_error: - raise forms.ValidationError(rev_error) - - # The following errors are likely noise if we have previous field - # errors: - if self.errors: - raise forms.ValidationError('') - - if not self.filename: - raise forms.ValidationError("Could not extract a valid Internet-Draft name from the upload. " - "To fix this in a text upload, please make sure that the full Internet-Draft name including " - "revision number appears centered on its own line below the document title on the " - "first page. In an xml upload, please make sure that the top-level " - "element has a docName attribute which provides the full Internet-Draft name including " - "revision number.") - - if not self.revision: - raise forms.ValidationError("Could not extract a valid Internet-Draft revision from the upload. " - "To fix this in a text upload, please make sure that the full Internet-Draft name including " - "revision number appears centered on its own line below the document title on the " - "first page. In an xml upload, please make sure that the top-level " - "element has a docName attribute which provides the full Internet-Draft name including " - "revision number.") - - if not self.title: - raise forms.ValidationError("Could not extract a valid title from the upload") - - if self.cleaned_data.get('txt') or self.cleaned_data.get('xml'): - # check group - self.group = self.deduce_group(self.filename) - - # check existing - existing = Submission.objects.filter(name=self.filename, rev=self.revision).exclude(state__in=("posted", "cancel", "waiting-for-draft")) - if existing: - raise forms.ValidationError(mark_safe('A submission with same name and revision is currently being processed. Check the status here.' % urlreverse("ietf.submit.views.submission_status", kwargs={ 'submission_id': existing[0].pk }))) - - # cut-off - if self.revision == '00' and self.in_first_cut_off: - raise forms.ValidationError(mark_safe(self.cutoff_warning)) - - # check thresholds - today = date_today() - - self.check_submissions_thresholds( - "for the Internet-Draft %s" % self.filename, - dict(name=self.filename, rev=self.revision, submission_date=today), - settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME, settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME_SIZE, - ) - self.check_submissions_thresholds( - "for the same submitter", - dict(remote_ip=self.remote_ip, submission_date=today), - settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER, settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER_SIZE, - ) - if self.group: - self.check_submissions_thresholds( - "for the group \"%s\"" % (self.group.acronym), - dict(group=self.group, submission_date=today), - settings.IDSUBMIT_MAX_DAILY_SAME_GROUP, settings.IDSUBMIT_MAX_DAILY_SAME_GROUP_SIZE, - ) - self.check_submissions_thresholds( - "across all submitters", - dict(submission_date=today), - settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS, settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE, - ) - - return super().clean() - - -class DeprecatedSubmissionAutoUploadForm(DeprecatedSubmissionBaseUploadForm): - """Full-service upload form, replaced by the asynchronous version""" - user = forms.EmailField(required=True) - - def __init__(self, request, *args, **kwargs): - super(DeprecatedSubmissionAutoUploadForm, self).__init__(request, *args, **kwargs) - self.formats = ['xml', ] - self.base_formats = ['xml', ] - - class SubmissionManualUploadForm(SubmissionBaseUploadForm): txt = forms.FileField(label='.txt format', required=False) formats = SubmissionBaseUploadForm.formats + ('txt',) @@ -676,6 +404,7 @@ def clean_txt(self): ) return txt_file + class SubmissionAutoUploadForm(SubmissionBaseUploadForm): user = forms.EmailField(required=True) replaces = forms.CharField(required=False, max_length=1000, strip=True) diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index b48168f8a6..ed28c7ef02 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -44,7 +44,7 @@ from ietf.meeting.factories import MeetingFactory from ietf.name.models import DraftSubmissionStateName, FormalLanguageName from ietf.person.models import Person -from ietf.person.factories import UserFactory, PersonFactory, EmailFactory +from ietf.person.factories import UserFactory, PersonFactory from ietf.submit.factories import SubmissionFactory, SubmissionExtResourceFactory from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm from ietf.submit.models import Submission, Preapproval, SubmissionExtResource @@ -2345,6 +2345,12 @@ def setUp(self): super().setUp() MeetingFactory(type_id='ietf', date=date_today()+datetime.timedelta(days=60)) + def test_api_submit_tombstone(self): + """Tombstone for obsolete API endpoint should return 410 Gone""" + url = urlreverse("ietf.submit.views.api_submit_tombstone") + self.assertEqual(self.client.get(url).status_code, 410) + self.assertEqual(self.client.post(url).status_code, 410) + def test_upload_draft(self): """api_submission accepts a submission and queues it for processing""" url = urlreverse('ietf.submit.views.api_submission') @@ -3191,141 +3197,6 @@ def test_cancel_stale_submissions(self): self.assertEqual(subm.state_id, "cancel") self.assertEqual(subm.submissionevent_set.count(), 2) - -class ApiSubmitTests(BaseSubmitTestCase): - def setUp(self): - super().setUp() - # break early in case of missing configuration - self.assertTrue(os.path.exists(settings.IDSUBMIT_IDNITS_BINARY)) - MeetingFactory(type_id='ietf', date=date_today()+datetime.timedelta(days=60)) - - def do_post_submission(self, rev, author=None, name=None, group=None, email=None, title=None, year=None): - url = urlreverse('ietf.submit.views.api_submit') - if author is None: - author = PersonFactory() - if name is None: - slug = re.sub('[^a-z0-9-]+', '', author.ascii_parts()[3].lower()) - name = 'draft-%s-foo' % slug - if email is None: - email = author.user.username - # submit - data = {} - data['xml'], author = submission_file(f'{name}-{rev}', f'{name}-{rev}.xml', group, "test_submission.xml", author=author, email=email, title=title, year=year) - data['user'] = email - r = self.client.post(url, data) - return r, author, name - - def test_api_submit_info(self): - url = urlreverse('ietf.submit.views.api_submit') - r = self.client.get(url) - expected = "A simplified Internet-Draft submission interface, intended for automation" - self.assertContains(r, expected, status_code=200) - - def test_api_submit_bad_method(self): - url = urlreverse('ietf.submit.views.api_submit') - r = self.client.put(url) - self.assertEqual(r.status_code, 405) - - def test_api_submit_ok(self): - r, author, name = self.do_post_submission('00') - expected = "Upload of %s OK, confirmation requests sent to:\n %s" % (name, author.formatted_email().replace('\n','')) - self.assertContains(r, expected, status_code=200) - - def test_api_submit_secondary_email_active(self): - person = PersonFactory() - email = EmailFactory(person=person) - r, author, name = self.do_post_submission('00', author=person, email=email.address) - for expected in [ - "Upload of %s OK, confirmation requests sent to:" % (name, ), - author.formatted_email().replace('\n',''), - ]: - self.assertContains(r, expected, status_code=200) - - def test_api_submit_secondary_email_inactive(self): - person = PersonFactory() - prim = person.email() - prim.primary = True - prim.save() - email = EmailFactory(person=person, active=False) - r, author, name = self.do_post_submission('00', author=person, email=email.address) - expected = "No such user: %s" % email.address - self.assertContains(r, expected, status_code=400) - - def test_api_submit_no_user(self): - email='nonexistant.user@example.org' - r, author, name = self.do_post_submission('00', email=email) - expected = "No such user: %s" % email - self.assertContains(r, expected, status_code=400) - - def test_api_submit_no_person(self): - user = UserFactory() - email = user.username - r, author, name = self.do_post_submission('00', email=email) - expected = "No person with username %s" % email - self.assertContains(r, expected, status_code=400) - - def test_api_submit_wrong_revision(self): - r, author, name = self.do_post_submission('01') - expected = "Invalid revision (revision 00 is expected)" - self.assertContains(r, expected, status_code=400) - - def test_api_submit_update_existing_submissiondocevent_rev(self): - draft, _ = create_draft_submission_with_rev_mismatch(rev='01') - r, _, __ = self.do_post_submission(rev='01', name=draft.name) - expected = "Submission failed" - self.assertContains(r, expected, status_code=409) - - def test_api_submit_update_later_submissiondocevent_rev(self): - draft, _ = create_draft_submission_with_rev_mismatch(rev='02') - r, _, __ = self.do_post_submission(rev='01', name=draft.name) - expected = "Submission failed" - self.assertContains(r, expected, status_code=409) - - def test_api_submit_pending_submission(self): - r, author, name = self.do_post_submission('00') - expected = "Upload of" - self.assertContains(r, expected, status_code=200) - r, author, name = self.do_post_submission('00', author=author, name=name) - expected = "A submission with same name and revision is currently being processed" - self.assertContains(r, expected, status_code=400) - - def test_api_submit_no_title(self): - r, author, name = self.do_post_submission('00', title=" ") - expected = "Could not extract a valid title from the upload" - self.assertContains(r, expected, status_code=400) - - def test_api_submit_failed_idnits(self): - # `year` on the next line must be leap year or this test will fail every Feb 29 - r, author, name = self.do_post_submission('00', year="2012") - expected = "Document date must be within 3 days of submission date" - self.assertContains(r, expected, status_code=400) - - def test_api_submit_keeps_extresources(self): - """API submit should not disturb doc external resources - - Tests that the submission inherits the existing doc's docextresource_set. - Relies on separate testing that Submission external_resources will be - handled appropriately. - """ - draft = WgDraftFactory() - - # add an external resource - self.assertEqual(draft.docextresource_set.count(), 0) - extres = draft.docextresource_set.create( - name_id='faq', - display_name='this is a display name', - value='https://example.com/faq-for-test.html', - ) - - r, _, __ = self.do_post_submission('01', name=draft.name) - self.assertEqual(r.status_code, 200) - # draft = Document.objects.get(pk=draft.pk) # update the draft - sub = Submission.objects.get(name=draft.name) - self.assertEqual( - [str(r) for r in sub.external_resources.all()], - [str(extres)], - ) - class RefsTests(BaseSubmitTestCase): diff --git a/ietf/submit/views.py b/ietf/submit/views.py index 6f23ba49d4..3f745741e4 100644 --- a/ietf/submit/views.py +++ b/ietf/submit/views.py @@ -1,7 +1,5 @@ # Copyright The IETF Trust 2011-2020, All Rights Reserved # -*- coding: utf-8 -*- - - import re import datetime @@ -29,17 +27,15 @@ from ietf.mailtrigger.utils import gather_address_lists from ietf.person.models import Email from ietf.submit.forms import (SubmissionAutoUploadForm, AuthorForm, SubmitterForm, EditSubmissionForm, - PreapprovalForm, ReplacesForm, - DeprecatedSubmissionAutoUploadForm, SubmissionManualUploadForm) + PreapprovalForm, ReplacesForm, SubmissionManualUploadForm) from ietf.submit.mail import send_full_url, send_manual_post_request from ietf.submit.models import (Submission, Preapproval, SubmissionExtResource, DraftSubmissionStateName ) from ietf.submit.tasks import process_uploaded_submission_task, process_and_accept_uploaded_submission_task, poke from ietf.submit.utils import ( approvable_submissions_for_user, preapprovals_for_user, recently_approved_by_user, validate_submission, create_submission_event, docevent_from_submission, - post_submission, cancel_submission, rename_submission_files, remove_submission_files, get_draft_meta, - get_submission, fill_in_submission, apply_checkers, save_files, clear_existing_files, - check_submission_revision_consistency, accept_submission, accept_submission_requires_group_approval, + post_submission, cancel_submission, rename_submission_files, remove_submission_files, + get_submission, save_files, clear_existing_files, accept_submission, accept_submission_requires_group_approval, accept_submission_requires_prev_auth_approval, update_submission_external_resources) from ietf.stats.utils import clean_country_name from ietf.utils.accesstoken import generate_access_token @@ -187,97 +183,14 @@ def api_submission_status(request, submission_id): @csrf_exempt -def api_submit(request): - "Automated submission entrypoint" - submission = None - def err(code, text): - return HttpResponse(text, status=code, content_type='text/plain') - - if request.method == 'GET': - return render(request, 'submit/api_submit_info.html') - elif request.method == 'POST': - exception = None - try: - form = DeprecatedSubmissionAutoUploadForm(request, data=request.POST, files=request.FILES) - if form.is_valid(): - log('got valid submission form for %s' % form.filename) - username = form.cleaned_data['user'] - user = User.objects.filter(username__iexact=username) - if user.count() == 0: - # See if a secondary login was being used - email = Email.objects.filter(address=username, active=True) - # The error messages don't talk about 'email', as the field we're - # looking at is still the 'username' field. - if email.count() == 0: - return err(400, "No such user: %s" % username) - elif email.count() > 1: - return err(500, "Multiple matching accounts for %s" % username) - email = email.first() - if not hasattr(email, 'person'): - return err(400, "No person matches %s" % username) - person = email.person - if not hasattr(person, 'user'): - return err(400, "No user matches: %s" % username) - user = person.user - elif user.count() > 1: - return err(500, "Multiple matching accounts for %s" % username) - else: - user = user.first() - if not hasattr(user, 'person'): - return err(400, "No person with username %s" % username) - - saved_files = save_files(form) - authors, abstract, file_name, file_size = get_draft_meta(form, saved_files) - for a in authors: - if not a['email']: - raise ValidationError("Missing email address for author %s" % a) - - submission = get_submission(form) - fill_in_submission(form, submission, authors, abstract, file_size) - apply_checkers(submission, file_name) - - create_submission_event(request, submission, desc="Uploaded submission via api_submit") - - errors = validate_submission(submission) - if errors: - raise ValidationError(errors) - - # must do this after validate_submission() or data needed for check may be invalid - if check_submission_revision_consistency(submission): - return err( 409, "Submission failed due to a document revision inconsistency error " - "in the database. Please contact the secretariat for assistance.") - - errors = [ c.message for c in submission.checks.all() if c.passed==False ] - if errors: - raise ValidationError(errors) - - if not username.lower() in [ a['email'].lower() for a in authors ]: - raise ValidationError('Submitter %s is not one of the document authors' % user.username) - - submission.submitter = user.person.formatted_email() - sent_to = accept_submission(submission, request) +def api_submit_tombstone(request): + """Tombstone for removed automated submission entrypoint""" + return render( + request, + 'submit/api_submit_info.html', + status=410, # Gone + ) - return HttpResponse( - "Upload of %s OK, confirmation requests sent to:\n %s" % (submission.name, ',\n '.join(sent_to)), - content_type="text/plain") - else: - raise ValidationError(form.errors) - except IOError as e: - exception = e - return err(500, "IO Error: %s" % str(e)) - except ValidationError as e: - exception = e - return err(400, "Validation Error: %s" % str(e)) - except Exception as e: - exception = e - raise - return err(500, "Exception: %s" % str(e)) - finally: - if exception and submission: - remove_submission_files(submission) - submission.delete() - else: - return err(405, "Method not allowed") def tool_instructions(request): return render(request, 'submit/tool_instructions.html', {'selected': 'instructions'}) diff --git a/ietf/templates/api/index.html b/ietf/templates/api/index.html index 8373a387f9..e21a50101a 100644 --- a/ietf/templates/api/index.html +++ b/ietf/templates/api/index.html @@ -9,7 +9,7 @@

Framework API

This section describes the autogenerated read-only API towards the database tables. See also the - Internet-Draft submission API description + Internet-Draft submission API description and the IESG ballot position API description

diff --git a/ietf/templates/submit/api_submit_info.html b/ietf/templates/submit/api_submit_info.html index cd0d52410b..75fc1abfc2 100644 --- a/ietf/templates/submit/api_submit_info.html +++ b/ietf/templates/submit/api_submit_info.html @@ -1,56 +1,13 @@ {% extends "base.html" %} -{# Copyright The IETF Trust 2015-2022, All Rights Reserved #} +{# Copyright The IETF Trust 2015-2024, All Rights Reserved #} {% load origin ietf_filters %} -{% block title %}I-D submission API instructions{% endblock %} +{% block title %}Obsolete I-D submission API notice{% endblock %} {% block content %} {% origin %} -

Internet-Draft submission API instructions

+

Obsolete Internet-Draft submission API notice

- Note: API endpoint described here is known to have a slow response time or to fail - due to timeout for some Internet-Draft submissions, particularly those with large file sizes. - It is recommended to use the new API endpoint - instead for increased reliability. + The API endpoint previously available here is obsolete and is no longer supported. + Please use the new API endpoint + instead.

-

- A simplified Internet-Draft submission interface, intended for automation, - is available at {% absurl 'ietf.submit.views.api_submit' %}. -

-

- The interface accepts only XML uploads that can be processed on the server, and - requires the user to have a datatracker account. A successful submit still requires - the same email confirmation round-trip as submissions done through the regular - submission tool. -

-

- This interface does not provide all the options which the regular submission tool does. - Some limitations: -

-
    -
  • Only XML-only uploads are supported, not text or combined.
    • -
  • Document replacement information cannot be supplied.
    • -
  • - The server expects multipart/form-data, supported by curl but not by wget. -
    • -
-

- It takes two parameters: -

-
    -
  • - user which is the user login -
    • -
  • - xml, which is the submitted file -
    • -
-

- It returns an appropriate http result code, and a brief explanatory text message. -

-

- Here is an example: -

- 
-$ curl -S -F "user=user.name@example.com" -F "xml=@~/draft-user-example.xml" {% absurl 'ietf.submit.views.api_submit' %}
-Upload of draft-user-example OK, confirmation requests sent to:
-User Name <user.name@example.com>
-{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ietf/utils/mail.py b/ietf/utils/mail.py index 4585fdb846..5417161451 100644 --- a/ietf/utils/mail.py +++ b/ietf/utils/mail.py @@ -19,11 +19,13 @@ from email.header import Header, decode_header from email import message_from_bytes, message_from_string from email import charset as Charset +from typing import Optional from django.conf import settings from django.contrib import messages from django.core.exceptions import ImproperlyConfigured, ValidationError from django.core.validators import validate_email +from django.http import HttpRequest from django.template.loader import render_to_string from django.template import Context,RequestContext from django.utils import timezone @@ -64,6 +66,18 @@ def add_headers(msg): msg['From'] = settings.DEFAULT_FROM_EMAIL return msg + +def decode_header_value(value: str) -> str: + """Decode a header value + + Easier-to-use wrapper around email.message.decode_header() + """ + return "".join( + part.decode(charset if charset else "utf-8") if isinstance(part, bytes) else part + for part, charset in decode_header(value) + ) + + class SMTPSomeRefusedRecipients(smtplib.SMTPException): def __init__(self, message, original_msg, refusals): @@ -251,8 +265,7 @@ def parseaddr(addr): """ - addr = ''.join( [ ( s.decode(m) if m else s.decode()) if isinstance(s, bytes) else s for (s,m) in decode_header(addr) ] ) - name, addr = simple_parseaddr(addr) + name, addr = simple_parseaddr(decode_header_value(addr)) return name, addr def excludeaddrs(addrlist, exlist): @@ -330,18 +343,45 @@ def condition_message(to, frm, subject, msg, cc, extra): msg['Message-ID'] = make_msgid() -def show_that_mail_was_sent(request,leadline,msg,bcc): - if request and request.user: - from ietf.ietfauth.utils import has_role - if has_role(request.user,['Area Director','Secretariat','IANA','RFC Editor','ISE','IAD','IRTF Chair','WG Chair','RG Chair','WG Secretary','RG Secretary']): - info = "%s at %s %s\n" % (leadline,timezone.now().strftime("%Y-%m-%d %H:%M:%S"),settings.TIME_ZONE) - info += "Subject: %s\n" % force_str(msg.get('Subject','[no subject]')) - info += "To: %s\n" % msg.get('To','[no to]') - if msg.get('Cc'): - info += "Cc: %s\n" % msg.get('Cc') - if bcc: - info += "Bcc: %s\n" % bcc - messages.info(request,info,extra_tags='preformatted',fail_silently=True) +def show_that_mail_was_sent(request: HttpRequest, leadline: str, msg: Message, bcc: Optional[str]): + if request and request.user: + from ietf.ietfauth.utils import has_role + + if has_role( + request.user, + [ + "Area Director", + "Secretariat", + "IANA", + "RFC Editor", + "ISE", + "IAD", + "IRTF Chair", + "WG Chair", + "RG Chair", + "WG Secretary", + "RG Secretary", + ], + ): + subject = decode_header_value(msg.get("Subject", "[no subject]")) + _to = decode_header_value(msg.get("To", "[no to]")) + info_lines = [ + f"{leadline} at {timezone.now():%Y-%m-%d %H:%M:%S %Z}", + f"Subject: {subject}", + f"To: {_to}", + ] + cc = msg.get("Cc", None) + if cc: + info_lines.append(f"Cc: {decode_header_value(cc)}") + if bcc: + info_lines.append(f"Bcc: {decode_header_value(bcc)}") + messages.info( + request, + "\n".join(info_lines), + extra_tags="preformatted", + fail_silently=True, + ) + def save_as_message(request, msg, bcc): by = ((request and request.user and not request.user.is_anonymous and request.user.person) diff --git a/ietf/utils/meetecho.py b/ietf/utils/meetecho.py index 2f5f146766..0dbf75736a 100644 --- a/ietf/utils/meetecho.py +++ b/ietf/utils/meetecho.py @@ -481,12 +481,10 @@ def delete_conference(self, conf: Conference): class SlidesManager(Manager): """Interface between Datatracker models and Meetecho API - Note: The URL we send comes from get_versionless_href(). This should match what we use as the - URL in api_get_session_materials(). Additionally, it _must_ give the right result for a Document - instance that has not yet been persisted to the database. This is because upload_session_slides() - (as of 2024-03-07) SessionPresentations before saving its updated Documents. This means, for - example, using get_absolute_url() will cause bugs. (We should refactor upload_session_slides() to - avoid this requirement.) + Note: the URL sent for a slide deck comes from DocumentInfo.get_href() and includes the revision + of the slides being sent. Be sure that 1) the URL matches what api_get_session_materials() returns + for the slides; and 2) the URL is valid if it is fetched immediately - possibly even before the call + to SlidesManager.add() or send_update() returns. """ def __init__(self, api_config): @@ -521,7 +519,7 @@ def add(self, session: "Session", slides: "Document", order: int): deck={ "id": slides.pk, "title": slides.title, - "url": slides.get_versionless_href(), # see above note re: get_versionless_href() + "url": slides.get_href(), "rev": slides.rev, "order": order, } @@ -575,7 +573,7 @@ def send_update(self, session: "Session"): { "id": deck.document.pk, "title": deck.document.title, - "url": deck.document.get_versionless_href(), # see note above re: get_versionless_href() + "url": deck.document.get_href(), "rev": deck.document.rev, "order": deck.order, } diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index decdd778d9..0a1986a608 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -11,10 +11,11 @@ import shutil import types -from mock import patch +from mock import call, patch from pyquery import PyQuery from typing import Dict, List # pyflakes:ignore +from email.message import Message from email.mime.image import MIMEImage from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText @@ -32,6 +33,7 @@ from django.template.defaulttags import URLNode from django.template.loader import get_template, render_to_string from django.templatetags.static import StaticNode +from django.test import RequestFactory from django.urls import reverse as urlreverse import debug # pyflakes:ignore @@ -42,7 +44,15 @@ from ietf.utils.draft import PlaintextDraft, getmeta from ietf.utils.fields import SearchableField from ietf.utils.log import unreachable, assertion -from ietf.utils.mail import send_mail_preformatted, send_mail_text, send_mail_mime, outbox, get_payload_text +from ietf.utils.mail import ( + send_mail_preformatted, + send_mail_text, + send_mail_mime, + outbox, + get_payload_text, + decode_header_value, + show_that_mail_was_sent, +) from ietf.utils.test_runner import get_template_paths, set_coverage_checking from ietf.utils.test_utils import TestCase, unicontent from ietf.utils.text import parse_unicode @@ -109,6 +119,135 @@ def test_send_mail_preformatted(self): recv = outbox[-1] self.assertEqual(recv['Fuzz'], 'bucket, monger') + +class MailUtilsTests(TestCase): + def test_decode_header_value(self): + self.assertEqual( + decode_header_value("cake"), + "cake", + "decodes simple string value", + ) + self.assertEqual( + decode_header_value("=?utf-8?b?8J+Ogg==?="), + "\U0001f382", + "decodes single utf-8-encoded part", + ) + self.assertEqual( + decode_header_value("=?utf-8?b?8J+Ogg==?= = =?macintosh?b?jYxrjg==?="), + "\U0001f382 = çåké", + "decodes a value with non-utf-8 encodings", + ) + + # Patch in a side_effect so we can distinguish values that came from decode_header_value. + @patch("ietf.utils.mail.decode_header_value", side_effect=lambda s: f"decoded-{s}") + @patch("ietf.utils.mail.messages") + def test_show_that_mail_was_sent(self, mock_messages, mock_decode_header_value): + request = RequestFactory().get("/some/path") + request.user = object() # just needs to exist + msg = Message() + msg["To"] = "to-value" + msg["Subject"] = "subject-value" + msg["Cc"] = "cc-value" + with patch("ietf.ietfauth.utils.has_role", return_value=True): + show_that_mail_was_sent(request, "mail was sent", msg, "bcc-value") + self.assertCountEqual( + mock_decode_header_value.call_args_list, + [call("to-value"), call("subject-value"), call("cc-value"), call("bcc-value")], + ) + self.assertEqual(mock_messages.info.call_args[0][0], request) + self.assertIn("mail was sent", mock_messages.info.call_args[0][1]) + self.assertIn("decoded-subject-value", mock_messages.info.call_args[0][1]) + self.assertIn("decoded-to-value", mock_messages.info.call_args[0][1]) + self.assertIn("decoded-cc-value", mock_messages.info.call_args[0][1]) + self.assertIn("decoded-bcc-value", mock_messages.info.call_args[0][1]) + mock_messages.reset_mock() + mock_decode_header_value.reset_mock() + + # no bcc + with patch("ietf.ietfauth.utils.has_role", return_value=True): + show_that_mail_was_sent(request, "mail was sent", msg, None) + self.assertCountEqual( + mock_decode_header_value.call_args_list, + [call("to-value"), call("subject-value"), call("cc-value")], + ) + self.assertEqual(mock_messages.info.call_args[0][0], request) + self.assertIn("mail was sent", mock_messages.info.call_args[0][1]) + self.assertIn("decoded-subject-value", mock_messages.info.call_args[0][1]) + self.assertIn("decoded-to-value", mock_messages.info.call_args[0][1]) + self.assertIn("decoded-cc-value", mock_messages.info.call_args[0][1]) + # Note: here and below - when using assertNotIn(), leaving off the "decoded-" prefix + # proves that neither the original value nor the decoded value appear. + self.assertNotIn("bcc-value", mock_messages.info.call_args[0][1]) + mock_messages.reset_mock() + mock_decode_header_value.reset_mock() + + # no cc + del msg["Cc"] + with patch("ietf.ietfauth.utils.has_role", return_value=True): + show_that_mail_was_sent(request, "mail was sent", msg, None) + self.assertCountEqual( + mock_decode_header_value.call_args_list, + [call("to-value"), call("subject-value")], + ) + self.assertEqual(mock_messages.info.call_args[0][0], request) + self.assertIn("mail was sent", mock_messages.info.call_args[0][1]) + self.assertIn("decoded-subject-value", mock_messages.info.call_args[0][1]) + self.assertIn("decoded-to-value", mock_messages.info.call_args[0][1]) + self.assertNotIn("cc-value", mock_messages.info.call_args[0][1]) + self.assertNotIn("bcc-value", mock_messages.info.call_args[0][1]) + mock_messages.reset_mock() + mock_decode_header_value.reset_mock() + + # no to + del msg["To"] + with patch("ietf.ietfauth.utils.has_role", return_value=True): + show_that_mail_was_sent(request, "mail was sent", msg, None) + self.assertCountEqual( + mock_decode_header_value.call_args_list, + [call("[no to]"), call("subject-value")], + ) + self.assertEqual(mock_messages.info.call_args[0][0], request) + self.assertIn("mail was sent", mock_messages.info.call_args[0][1]) + self.assertIn("decoded-subject-value", mock_messages.info.call_args[0][1]) + self.assertIn("decoded-[no to]", mock_messages.info.call_args[0][1]) + self.assertNotIn("to-value", mock_messages.info.call_args[0][1]) + self.assertNotIn("cc-value", mock_messages.info.call_args[0][1]) + self.assertNotIn("bcc-value", mock_messages.info.call_args[0][1]) + mock_messages.reset_mock() + mock_decode_header_value.reset_mock() + + # no subject + del msg["Subject"] + with patch("ietf.ietfauth.utils.has_role", return_value=True): + show_that_mail_was_sent(request, "mail was sent", msg, None) + self.assertCountEqual( + mock_decode_header_value.call_args_list, + [call("[no to]"), call("[no subject]")], + ) + self.assertEqual(mock_messages.info.call_args[0][0], request) + self.assertIn("mail was sent", mock_messages.info.call_args[0][1]) + self.assertIn("decoded-[no subject]", mock_messages.info.call_args[0][1]) + self.assertNotIn("subject-value", mock_messages.info.call_args[0][1]) + self.assertIn("decoded-[no to]", mock_messages.info.call_args[0][1]) + self.assertNotIn("to-value", mock_messages.info.call_args[0][1]) + self.assertNotIn("cc-value", mock_messages.info.call_args[0][1]) + self.assertNotIn("bcc-value", mock_messages.info.call_args[0][1]) + mock_messages.reset_mock() + mock_decode_header_value.reset_mock() + + # user does not have role + with patch("ietf.ietfauth.utils.has_role", return_value=False): + show_that_mail_was_sent(request, "mail was sent", msg, None) + self.assertFalse(mock_messages.called) + + # no user + request.user = None + with patch("ietf.ietfauth.utils.has_role", return_value=True) as mock_has_role: + show_that_mail_was_sent(request, "mail was sent", msg, None) + self.assertFalse(mock_messages.called) + self.assertFalse(mock_has_role.called) + + class TestSMTPServer(TestCase): def test_address_rejected(self): diff --git a/ietf/utils/tests_meetecho.py b/ietf/utils/tests_meetecho.py index 1aef5894e2..a10ac68c27 100644 --- a/ietf/utils/tests_meetecho.py +++ b/ietf/utils/tests_meetecho.py @@ -558,7 +558,7 @@ def test_add(self, mock_add, mock_wg_token): deck={ "id": slides_doc.pk, "title": slides_doc.title, - "url": slides_doc.get_versionless_href(), + "url": slides_doc.get_href(session.meeting), "rev": slides_doc.rev, "order": 13, }, @@ -597,7 +597,7 @@ def test_delete(self, mock_delete, mock_update, mock_wg_token): { "id": slides_doc.pk, "title": slides_doc.title, - "url": slides_doc.get_versionless_href(), + "url": slides_doc.get_href(session.meeting), "rev": slides_doc.rev, "order": 1, }, @@ -635,7 +635,7 @@ def test_revise(self, mock_add, mock_delete, mock_wg_token): deck={ "id": slides_doc.pk, "title": slides_doc.title, - "url": slides_doc.get_versionless_href(), + "url": slides_doc.get_href(slides.session.meeting), "rev": slides_doc.rev, "order": 23, }, @@ -660,7 +660,7 @@ def test_send_update(self, mock_send_update, mock_wg_token): { "id": slides.document_id, "title": slides.document.title, - "url": slides.document.get_versionless_href(), + "url": slides.document.get_href(slides.session.meeting), "rev": slides.document.rev, "order": 0, } diff --git a/playwright/tests/status/status.spec.js b/playwright/tests/status/status.spec.js index 7b3b90bfa4..daef7a88f1 100644 --- a/playwright/tests/status/status.spec.js +++ b/playwright/tests/status/status.spec.js @@ -20,6 +20,10 @@ test.describe('site status', () => { by: 'Exile is a cool Amiga game' } + test.beforeEach(({ browserName }) => { + test.skip(browserName === 'firefox', 'bypassing flaky tests on Firefox') + }) + test('Renders server status as Notification', async ({ page }) => { await page.route('/status/latest.json', route => { route.fulfill({