From 0c5c812441f665589044a00626d2a22b37343f8b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 Apr 2024 08:00:47 +0200 Subject: [PATCH 1/5] Bump pytest from 8.1.1 to 8.2.0 (#674) Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.1.1 to 8.2.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.1.1...8.2.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/tests.txt b/requirements/tests.txt index b8c25ff0..e6ea7656 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,5 +1,5 @@ coveralls[yaml]==3.3.1 -pytest==8.1.1 +pytest==8.2.0 pytest-cov==5.0.0 pytest-mock==3.14.0 wptserve==4.0.2 From ba6d934e96ac878e1bd94441729b7ad546012f04 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 Apr 2024 08:07:51 +0200 Subject: [PATCH 2/5] Bump coveralls[yaml] from 3.3.1 to 4.0.0 (#673) Bumps [coveralls[yaml]](https://github.com/TheKevJames/coveralls-python) from 3.3.1 to 4.0.0. - [Release notes](https://github.com/TheKevJames/coveralls-python/releases) - [Changelog](https://github.com/TheKevJames/coveralls-python/blob/master/CHANGELOG.md) - [Commits](https://github.com/TheKevJames/coveralls-python/compare/3.3.1...4.0.0) --- updated-dependencies: - dependency-name: coveralls[yaml] dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/tests.txt b/requirements/tests.txt index e6ea7656..b2946979 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,4 +1,4 @@ -coveralls[yaml]==3.3.1 +coveralls[yaml]==4.0.0 pytest==8.2.0 pytest-cov==5.0.0 pytest-mock==3.14.0 From 7345795189d703358d2ace6bc9973c8a4e8feb7b Mon Sep 17 00:00:00 2001 From: piri-p Date: Mon, 6 May 2024 09:20:07 +0200 Subject: [PATCH 3/5] Raise NotFoundError if no daily build can be found for latest build date (#672) --- mozdownload/scraper.py | 2 ++ tests/daily_scraper/test_invalid_platform.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 tests/daily_scraper/test_invalid_platform.py diff --git a/mozdownload/scraper.py b/mozdownload/scraper.py index a9ae5d2f..763d5d39 100755 --- a/mozdownload/scraper.py +++ b/mozdownload/scraper.py @@ -451,6 +451,8 @@ def get_latest_build_date(self): months.entries[-1] + '/') parser = self._create_directory_parser(url) parser.entries = parser.filter(r'.*%s' % self.platform_regex) + if not parser.entries: + raise errors.NotFoundError('No builds have been found', url) parser.entries.sort() date = ''.join(parser.entries[-1].split('-')[:6]) diff --git a/tests/daily_scraper/test_invalid_platform.py b/tests/daily_scraper/test_invalid_platform.py new file mode 100644 index 00000000..70c721fd --- /dev/null +++ b/tests/daily_scraper/test_invalid_platform.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import pytest + +from mozdownload import DailyScraper +import mozdownload.errors as errors + +@pytest.mark.parametrize('args', [ + ({'application': 'fenix', 'platform': 'mac'}), +]) +def test_invalid_platform(httpd, tmpdir, args): + """Testing download scenarios with invalid platform parameters for DailyScraper""" + + with pytest.raises(errors.NotFoundError): + DailyScraper(destination=str(tmpdir), base_url=httpd.get_url(), **args) From f9c501c3c818d0d67ea76d1c598e721d797fad5c Mon Sep 17 00:00:00 2001 From: piri-p Date: Mon, 6 May 2024 10:38:11 +0200 Subject: [PATCH 4/5] Add Fenix locale handling (#671) --- mozdownload/scraper.py | 11 +++++------ tests/daily_scraper/test_daily_scraper.py | 3 +++ tests/release_scraper/test_release_scraper.py | 3 +++ tests/release_scraper/test_release_scraper_latest.py | 3 +++ 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/mozdownload/scraper.py b/mozdownload/scraper.py index 763d5d39..07e40faf 100755 --- a/mozdownload/scraper.py +++ b/mozdownload/scraper.py @@ -124,13 +124,12 @@ def __init__(self, destination=None, platform=None, self.destination = destination or os.getcwd() - if not locale: - if application in APPLICATIONS_MULTI_LOCALE: - self.locale = 'multi' - else: - self.locale = 'en-US' - else: + if application in APPLICATIONS_MULTI_LOCALE: + self.locale = 'multi' + elif locale: self.locale = locale + else: + self.locale = 'en-US' self.locale_build = self.locale not in ('en-US', 'multi') self.platform = platform or self.detect_platform() diff --git a/tests/daily_scraper/test_daily_scraper.py b/tests/daily_scraper/test_daily_scraper.py index 72ff95bf..aba48b59 100644 --- a/tests/daily_scraper/test_daily_scraper.py +++ b/tests/daily_scraper/test_daily_scraper.py @@ -122,6 +122,9 @@ ({'application': 'fenix', 'platform': 'android-x86_64', 'date': '2022-11-14'}, '2022-11-14-17-01-36-fenix-108.0b1.multi.android-x86_64.apk', 'fenix/nightly/2022/11/2022-11-14-17-01-36-fenix-108.0b1-android-x86_64/fenix-108.0b1.multi.android-x86_64.apk'), + ({'application': 'fenix', 'platform': 'android-arm64-v8a', 'date': '2022-11-14', 'locale': 'de'}, + '2022-11-14-17-01-36-fenix-108.0b1.multi.android-arm64-v8a.apk', + 'fenix/nightly/2022/11/2022-11-14-17-01-36-fenix-108.0b1-android-arm64-v8a/fenix-108.0b1.multi.android-arm64-v8a.apk'), ] @pytest.mark.parametrize("args,filename,url", firefox_tests + thunderbird_tests + fenix_tests) diff --git a/tests/release_scraper/test_release_scraper.py b/tests/release_scraper/test_release_scraper.py index d6ed8618..0865b6d0 100644 --- a/tests/release_scraper/test_release_scraper.py +++ b/tests/release_scraper/test_release_scraper.py @@ -26,6 +26,9 @@ ({'application': 'fenix', 'platform': 'android-x86_64', 'version': '120.1.0'}, 'fenix-120.1.0.multi.android-x86_64.apk', 'fenix/releases/120.1.0/android/fenix-120.1.0-android-x86_64/fenix-120.1.0.multi.android-x86_64.apk'), + ({'application': 'fenix', 'platform': 'android-arm64-v8a', 'version': '120.1.0', 'locale': 'de'}, + 'fenix-120.1.0.multi.android-arm64-v8a.apk', + 'fenix/releases/120.1.0/android/fenix-120.1.0-android-arm64-v8a/fenix-120.1.0.multi.android-arm64-v8a.apk'), ({'platform': 'win32', 'version': '23.0.1'}, 'firefox-23.0.1.en-US.win32.exe', 'firefox/releases/23.0.1/win32/en-US/Firefox Setup 23.0.1.exe'), diff --git a/tests/release_scraper/test_release_scraper_latest.py b/tests/release_scraper/test_release_scraper_latest.py index 66419997..b356f851 100644 --- a/tests/release_scraper/test_release_scraper_latest.py +++ b/tests/release_scraper/test_release_scraper_latest.py @@ -38,6 +38,9 @@ ({'application': 'fenix', 'platform': 'android-x86_64', 'version': 'latest-beta'}, 'fenix-120.0b9.multi.android-x86_64.apk', 'fenix/releases/120.0b9/android/fenix-120.0b9-android-x86_64/fenix-120.0b9.multi.android-x86_64.apk'), + ({'application': 'fenix', 'platform': 'android-arm64-v8a', 'version': 'latest', 'locale': 'de'}, + 'fenix-120.1.0.multi.android-arm64-v8a.apk', + 'fenix/releases/120.1.0/android/fenix-120.1.0-android-arm64-v8a/fenix-120.1.0.multi.android-arm64-v8a.apk'), ({'application': 'firefox', 'platform': 'linux', 'version': 'latest'}, 'firefox-23.0.1.en-US.linux.tar.bz2', 'firefox/releases/23.0.1/linux-i686/en-US/firefox-23.0.1.tar.bz2'), From 6f22139e42a1f12c9ddcec2a4b50adb724f75dcf Mon Sep 17 00:00:00 2001 From: piri-p Date: Mon, 6 May 2024 15:59:53 +0200 Subject: [PATCH 5/5] Add consistent build number error notification for scrapers (#121) (#677) --- mozdownload/scraper.py | 9 ++++++++- tests/daily_scraper/test_invalid_build.py | 19 ++++++++++++++++++ .../test_release_candidate_scraper_indices.py | 2 +- .../test_invalid_build_tinderbox.py | 20 +++++++++++++++++++ 4 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 tests/daily_scraper/test_invalid_build.py create mode 100644 tests/tinderbox_scraper/test_invalid_build_tinderbox.py diff --git a/mozdownload/scraper.py b/mozdownload/scraper.py index 07e40faf..fc6cc4e7 100755 --- a/mozdownload/scraper.py +++ b/mozdownload/scraper.py @@ -541,6 +541,10 @@ def get_build_info_for_date(self, date, build_index=None): build_index -= 1 if not build_index or self.is_build_dir(build): break + + if build_index >= len(parser.entries): + raise errors.NotFoundError('Specified build number has not been found ', url) + self.logger.info('Selected build: %s' % parser.entries[build_index]) return (parser.entries, build_index) @@ -762,7 +766,7 @@ def get_build_info(self): self.build_index = 0 self.logger.info('Selected build: build%s' % self.build_number) else: - raise errors.NotSupportedError('Selected build not available') + raise errors.NotFoundError('Specified build number has not been found ', url) @property def candidate_build_list_regex(self): @@ -985,6 +989,9 @@ def get_build_info_for_index(self, build_index=None): if not build_index or self.is_build_dir(build): break + if build_index >= len(parser.entries): + raise errors.NotFoundError('Specified build number has not been found ', url) + self.logger.info('Selected build: %s' % parser.entries[build_index]) return (parser.entries, build_index) diff --git a/tests/daily_scraper/test_invalid_build.py b/tests/daily_scraper/test_invalid_build.py new file mode 100644 index 00000000..a14c8119 --- /dev/null +++ b/tests/daily_scraper/test_invalid_build.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import pytest + +from mozdownload import DailyScraper +import mozdownload.errors as errors + +@pytest.mark.parametrize('args', [ + ({'application': 'firefox', 'date': '2013-07-02', 'build_number': 3}), +]) +def test_invalid_build(httpd, tmpdir, args): + """Testing download scenarios with invalid branch parameters for DailyScraper""" + + with pytest.raises(errors.NotFoundError): + DailyScraper(destination=str(tmpdir), base_url=httpd.get_url(), **args) diff --git a/tests/release_candidate_scraper/test_release_candidate_scraper_indices.py b/tests/release_candidate_scraper/test_release_candidate_scraper_indices.py index 97673ef5..02b367d6 100644 --- a/tests/release_candidate_scraper/test_release_candidate_scraper_indices.py +++ b/tests/release_candidate_scraper/test_release_candidate_scraper_indices.py @@ -30,5 +30,5 @@ def test_invalid_build_number(httpd, tmpdir): args = {'application': 'firefox', 'build_number': '2', 'platform': 'linux', 'version': '23.0.1'} - with pytest.raises(errors.NotSupportedError): + with pytest.raises(errors.NotFoundError): ReleaseCandidateScraper(destination=tmpdir, base_url=httpd.get_url(), **args) diff --git a/tests/tinderbox_scraper/test_invalid_build_tinderbox.py b/tests/tinderbox_scraper/test_invalid_build_tinderbox.py new file mode 100644 index 00000000..1d291545 --- /dev/null +++ b/tests/tinderbox_scraper/test_invalid_build_tinderbox.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import pytest + +from mozdownload import TinderboxScraper +import mozdownload.errors as errors + + +@pytest.mark.parametrize("args", [ + ({'application': 'firefox', 'branch': 'mozilla-central', 'build_number': '4', 'date': '2013-07-23'}), +]) +def test_invalid_build_tinderbox(httpd, tmpdir, args): + """Testing download scenarios with invalid parameters for TinderboxScraper""" + + with pytest.raises(errors.NotFoundError): + TinderboxScraper(destination=str(tmpdir), base_url=httpd.get_url(), **args)