From 75990f356ed6d79601ecf3ecd96e48c3932acb47 Mon Sep 17 00:00:00 2001 From: Raiya Jessa Date: Tue, 28 Mar 2023 09:24:01 -0700 Subject: [PATCH] Add isort configuration file (#1054) * add .isort.cfg * add comment about known_first_party modules * Remove redundant config * Fixup after rebase --------- Co-authored-by: Madison Swain-Bowden --- .isort.cfg | 6 ++++++ justfile | 4 ++-- openverse_catalog/dags/common/loader/loader.py | 1 + openverse_catalog/dags/common/loader/s3.py | 1 + openverse_catalog/dags/common/loader/sql.py | 3 ++- openverse_catalog/dags/common/popularity/sql.py | 1 + openverse_catalog/dags/common/requester.py | 3 ++- openverse_catalog/dags/common/sql.py | 1 + openverse_catalog/dags/data_refresh/dag_factory.py | 1 + .../dags/data_refresh/data_refresh_task_factory.py | 3 ++- .../data_refresh/refresh_popularity_metrics_task_factory.py | 1 + .../dags/data_refresh/refresh_view_data_task_factory.py | 1 + .../dags/database/image_expiration_workflow.py | 1 + .../database/recreate_popularity_calculation_dag_factory.py | 1 + .../dags/database/report_pending_reported_media.py | 1 + .../dags/database/terminate_long_queries_workflow.py | 1 + openverse_catalog/dags/maintenance/add_license_url.py | 3 ++- .../dags/maintenance/airflow_log_cleanup_workflow.py | 1 + openverse_catalog/dags/maintenance/check_silenced_dags.py | 1 + .../dags/maintenance/flickr_audit_sub_provider_workflow.py | 1 + .../pr_review_reminders/pr_review_reminders_dag.py | 1 + openverse_catalog/dags/oauth2/authorize_dag.py | 3 ++- openverse_catalog/dags/oauth2/token_refresh_dag.py | 3 ++- openverse_catalog/dags/providers/factory_utils.py | 1 + .../dags/providers/provider_api_scripts/brooklyn_museum.py | 1 + .../dags/providers/provider_api_scripts/europeana.py | 3 ++- .../dags/providers/provider_api_scripts/flickr.py | 1 + .../dags/providers/provider_api_scripts/freesound.py | 5 +++-- .../dags/providers/provider_api_scripts/inaturalist.py | 1 + .../dags/providers/provider_api_scripts/jamendo.py | 3 ++- .../providers/provider_api_scripts/metropolitan_museum.py | 3 ++- .../dags/providers/provider_api_scripts/nypl.py | 1 + .../provider_api_scripts/provider_data_ingester.py | 1 + .../dags/providers/provider_api_scripts/rawpixel.py | 1 + .../dags/providers/provider_api_scripts/smithsonian.py | 3 ++- .../providers/provider_api_scripts/wikimedia_commons.py | 1 + .../dags/providers/provider_api_scripts/wordpress.py | 1 + openverse_catalog/dags/providers/provider_dag_factory.py | 1 + openverse_catalog/dags/providers/provider_workflows.py | 3 ++- .../utilities/dag_doc_gen/dag_doc_generation.py | 1 + tests/dags/common/conftest.py | 2 +- tests/dags/common/licenses/test_licenses.py | 3 ++- tests/dags/common/loader/test_loader.py | 1 + tests/dags/common/loader/test_reporting.py | 1 + tests/dags/common/loader/test_s3.py | 1 + tests/dags/common/loader/test_sql.py | 5 +++-- tests/dags/common/popularity/test_sql.py | 1 + .../common/sensors/test_single_run_external_dags_sensor.py | 1 + tests/dags/common/storage/test_audio.py | 2 +- tests/dags/common/storage/test_columns.py | 1 + tests/dags/common/storage/test_image.py | 1 + tests/dags/common/storage/test_media.py | 1 + tests/dags/common/storage/test_util.py | 1 + tests/dags/common/test_extensions.py | 1 + tests/dags/common/test_github.py | 3 ++- tests/dags/common/test_requester.py | 2 +- tests/dags/common/test_slack.py | 1 + tests/dags/common/test_sql.py | 4 ++-- tests/dags/common/test_urls.py | 3 ++- tests/dags/conftest.py | 3 ++- tests/dags/data_refresh/test_dag_factory.py | 1 + tests/dags/data_refresh/test_reporting.py | 1 + tests/dags/database/test_report_pending_reported_media.py | 1 + tests/dags/maintenance/test_check_silenced_dags.py | 2 +- tests/dags/maintenance/test_flickr_audit_sub_provider.py | 1 + tests/dags/maintenance/test_rotate_db_snapshots.py | 1 + tests/dags/oauth2/test_oauth2.py | 2 +- .../providers/provider_api_scripts/test_brooklyn_museum.py | 1 + .../providers/provider_api_scripts/test_cleveland_museum.py | 1 + tests/dags/providers/provider_api_scripts/test_europeana.py | 1 + .../providers/provider_api_scripts/test_finnish_museums.py | 1 + tests/dags/providers/provider_api_scripts/test_flickr.py | 1 + tests/dags/providers/provider_api_scripts/test_freesound.py | 1 + .../dags/providers/provider_api_scripts/test_inaturalist.py | 1 + tests/dags/providers/provider_api_scripts/test_jamendo.py | 1 + .../provider_api_scripts/test_metropolitan_museum.py | 1 + .../providers/provider_api_scripts/test_museum_victoria.py | 1 + tests/dags/providers/provider_api_scripts/test_nappy.py | 1 + tests/dags/providers/provider_api_scripts/test_nypl.py | 1 + tests/dags/providers/provider_api_scripts/test_phylopic.py | 1 + .../provider_api_scripts/test_provider_data_ingester.py | 2 +- tests/dags/providers/provider_api_scripts/test_rawpixel.py | 1 + .../providers/provider_api_scripts/test_science_museum.py | 1 + .../dags/providers/provider_api_scripts/test_smithsonian.py | 1 + tests/dags/providers/provider_api_scripts/test_smk.py | 1 + tests/dags/providers/provider_api_scripts/test_stocksnap.py | 1 + .../provider_api_scripts/test_wikimedia_commons.py | 1 + tests/dags/providers/provider_api_scripts/test_wordpress.py | 1 + tests/dags/providers/test_factory_utils.py | 2 +- tests/dags/providers/test_provider_dag_factory.py | 2 +- tests/dags/providers/test_provider_workflows.py | 2 +- tests/dags/test_dag_parsing.py | 1 + 92 files changed, 118 insertions(+), 32 deletions(-) create mode 100644 .isort.cfg diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 000000000..d2bbc8a43 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,6 @@ +[settings] +profile=black + +sections=FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER +# These submodules are all added to the PYTHONPATH by Airflow, so we need to add them individually here. +known_first_party=common,data_refresh,database,maintenance,oauth2,providers,retired diff --git a/justfile b/justfile index 5a02d90fa..504a1e992 100644 --- a/justfile +++ b/justfile @@ -73,8 +73,8 @@ deploy: @just up # Run pre-commit on all files -lint: - pre-commit run --all-files +lint hook="": + pre-commit run {{ hook }} --all-files # Load any dependencies necessary for actions on the stack without running the webserver _deps: diff --git a/openverse_catalog/dags/common/loader/loader.py b/openverse_catalog/dags/common/loader/loader.py index a4095db84..93390a001 100644 --- a/openverse_catalog/dags/common/loader/loader.py +++ b/openverse_catalog/dags/common/loader/loader.py @@ -1,4 +1,5 @@ from airflow.models.abstractoperator import AbstractOperator + from common.loader import paths, s3, sql from common.loader.paths import _extract_media_type from common.loader.reporting import RecordMetrics diff --git a/openverse_catalog/dags/common/loader/s3.py b/openverse_catalog/dags/common/loader/s3.py index 509b85850..7410bac67 100644 --- a/openverse_catalog/dags/common/loader/s3.py +++ b/openverse_catalog/dags/common/loader/s3.py @@ -4,6 +4,7 @@ from airflow.exceptions import AirflowSkipException from airflow.providers.amazon.aws.hooks.s3 import S3Hook + from common.loader import paths diff --git a/openverse_catalog/dags/common/loader/sql.py b/openverse_catalog/dags/common/loader/sql.py index 9cec55915..1773ca844 100644 --- a/openverse_catalog/dags/common/loader/sql.py +++ b/openverse_catalog/dags/common/loader/sql.py @@ -2,6 +2,8 @@ from textwrap import dedent from airflow.models.abstractoperator import AbstractOperator +from psycopg2.errors import InvalidTextRepresentation + from common.constants import AUDIO, IMAGE, MediaType from common.loader import provider_details as prov from common.loader.paths import _extract_media_type @@ -15,7 +17,6 @@ CURRENT_IMAGE_TSV_COLUMNS, required_columns, ) -from psycopg2.errors import InvalidTextRepresentation logger = logging.getLogger(__name__) diff --git a/openverse_catalog/dags/common/popularity/sql.py b/openverse_catalog/dags/common/popularity/sql.py index 2c3e04073..d5c5f53a7 100644 --- a/openverse_catalog/dags/common/popularity/sql.py +++ b/openverse_catalog/dags/common/popularity/sql.py @@ -3,6 +3,7 @@ from textwrap import dedent from airflow.models.abstractoperator import AbstractOperator + from common.constants import AUDIO, IMAGE from common.loader.sql import TABLE_NAMES from common.sql import PostgresHook diff --git a/openverse_catalog/dags/common/requester.py b/openverse_catalog/dags/common/requester.py index 160e22207..6ce81aff1 100644 --- a/openverse_catalog/dags/common/requester.py +++ b/openverse_catalog/dags/common/requester.py @@ -2,11 +2,12 @@ import time from collections.abc import Callable -import oauth2 import requests from airflow.exceptions import AirflowException from requests.exceptions import JSONDecodeError +import oauth2 + # pytest_socket will not be available in production, so we must create a shim for # that specific exception so this code does not error out. diff --git a/openverse_catalog/dags/common/sql.py b/openverse_catalog/dags/common/sql.py index cba305e8b..0a544b7a3 100644 --- a/openverse_catalog/dags/common/sql.py +++ b/openverse_catalog/dags/common/sql.py @@ -8,6 +8,7 @@ from airflow.providers.postgres.hooks.postgres import ( PostgresHook as UpstreamPostgresHook, ) + from common.constants import DAG_DEFAULT_ARGS, POSTGRES_CONN_ID diff --git a/openverse_catalog/dags/data_refresh/dag_factory.py b/openverse_catalog/dags/data_refresh/dag_factory.py index 836f0e9b6..7dc7698c6 100644 --- a/openverse_catalog/dags/data_refresh/dag_factory.py +++ b/openverse_catalog/dags/data_refresh/dag_factory.py @@ -40,6 +40,7 @@ from airflow.settings import SASession from airflow.utils.session import provide_session from airflow.utils.state import State + from common.constants import ( DAG_DEFAULT_ARGS, OPENLEDGER_API_CONN_ID, diff --git a/openverse_catalog/dags/data_refresh/data_refresh_task_factory.py b/openverse_catalog/dags/data_refresh/data_refresh_task_factory.py index 0b853169c..4c30c345f 100644 --- a/openverse_catalog/dags/data_refresh/data_refresh_task_factory.py +++ b/openverse_catalog/dags/data_refresh/data_refresh_task_factory.py @@ -57,10 +57,11 @@ from airflow.providers.http.operators.http import SimpleHttpOperator from airflow.providers.http.sensors.http import HttpSensor from airflow.utils.task_group import TaskGroup +from requests import Response + from common.constants import XCOM_PULL_TEMPLATE from common.sensors.single_run_external_dags_sensor import SingleRunExternalDAGsSensor from data_refresh.data_refresh_types import DataRefresh -from requests import Response logger = logging.getLogger(__name__) diff --git a/openverse_catalog/dags/data_refresh/refresh_popularity_metrics_task_factory.py b/openverse_catalog/dags/data_refresh/refresh_popularity_metrics_task_factory.py index 7247f9bf4..ea1ca1911 100644 --- a/openverse_catalog/dags/data_refresh/refresh_popularity_metrics_task_factory.py +++ b/openverse_catalog/dags/data_refresh/refresh_popularity_metrics_task_factory.py @@ -10,6 +10,7 @@ """ from airflow.operators.python import PythonOperator from airflow.utils.task_group import TaskGroup + from common.constants import POSTGRES_CONN_ID from common.popularity import sql from data_refresh import reporting diff --git a/openverse_catalog/dags/data_refresh/refresh_view_data_task_factory.py b/openverse_catalog/dags/data_refresh/refresh_view_data_task_factory.py index 1a1220a7a..69fc4a07c 100644 --- a/openverse_catalog/dags/data_refresh/refresh_view_data_task_factory.py +++ b/openverse_catalog/dags/data_refresh/refresh_view_data_task_factory.py @@ -12,6 +12,7 @@ """ from airflow.operators.python import PythonOperator from airflow.utils.trigger_rule import TriggerRule + from common.constants import POSTGRES_CONN_ID from common.popularity import sql from data_refresh import reporting diff --git a/openverse_catalog/dags/database/image_expiration_workflow.py b/openverse_catalog/dags/database/image_expiration_workflow.py index 14f941c11..07040cb34 100644 --- a/openverse_catalog/dags/database/image_expiration_workflow.py +++ b/openverse_catalog/dags/database/image_expiration_workflow.py @@ -9,6 +9,7 @@ from airflow import DAG from airflow.operators.python import PythonOperator + from common.constants import DAG_DEFAULT_ARGS, POSTGRES_CONN_ID from common.loader import sql diff --git a/openverse_catalog/dags/database/recreate_popularity_calculation_dag_factory.py b/openverse_catalog/dags/database/recreate_popularity_calculation_dag_factory.py index bae487a47..2770fab50 100644 --- a/openverse_catalog/dags/database/recreate_popularity_calculation_dag_factory.py +++ b/openverse_catalog/dags/database/recreate_popularity_calculation_dag_factory.py @@ -10,6 +10,7 @@ """ from airflow import DAG from airflow.operators.python import PythonOperator + from common.constants import DAG_DEFAULT_ARGS, POSTGRES_CONN_ID from common.popularity import sql from data_refresh.data_refresh_types import DATA_REFRESH_CONFIGS, DataRefresh diff --git a/openverse_catalog/dags/database/report_pending_reported_media.py b/openverse_catalog/dags/database/report_pending_reported_media.py index e0534d95f..61f041691 100644 --- a/openverse_catalog/dags/database/report_pending_reported_media.py +++ b/openverse_catalog/dags/database/report_pending_reported_media.py @@ -18,6 +18,7 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator, TaskInstance from airflow.operators.python import PythonOperator + from common import slack from common.constants import ( DAG_DEFAULT_ARGS, diff --git a/openverse_catalog/dags/database/terminate_long_queries_workflow.py b/openverse_catalog/dags/database/terminate_long_queries_workflow.py index c411335d6..663974621 100644 --- a/openverse_catalog/dags/database/terminate_long_queries_workflow.py +++ b/openverse_catalog/dags/database/terminate_long_queries_workflow.py @@ -11,6 +11,7 @@ from airflow import DAG from airflow.exceptions import AirflowSkipException from airflow.operators.python import PythonOperator + from common.constants import ( DAG_DEFAULT_ARGS, OPENLEDGER_API_CONN_ID, diff --git a/openverse_catalog/dags/maintenance/add_license_url.py b/openverse_catalog/dags/maintenance/add_license_url.py index fc36d66ea..5e3b3072f 100644 --- a/openverse_catalog/dags/maintenance/add_license_url.py +++ b/openverse_catalog/dags/maintenance/add_license_url.py @@ -21,13 +21,14 @@ from airflow.operators.python import PythonOperator from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.utils.trigger_rule import TriggerRule +from psycopg2._json import Json + from common.constants import DAG_DEFAULT_ARGS, POSTGRES_CONN_ID, XCOM_PULL_TEMPLATE from common.licenses import get_license_info_from_license_pair from common.loader.sql import RETURN_ROW_COUNT from common.slack import send_message from common.sql import PostgresHook from providers.provider_dag_factory import AWS_CONN_ID, OPENVERSE_BUCKET -from psycopg2._json import Json DAG_ID = "add_license_url" diff --git a/openverse_catalog/dags/maintenance/airflow_log_cleanup_workflow.py b/openverse_catalog/dags/maintenance/airflow_log_cleanup_workflow.py index 8e916169d..23f2876d6 100644 --- a/openverse_catalog/dags/maintenance/airflow_log_cleanup_workflow.py +++ b/openverse_catalog/dags/maintenance/airflow_log_cleanup_workflow.py @@ -25,6 +25,7 @@ from airflow.configuration import conf from airflow.models import DAG from airflow.operators.python import PythonOperator + from common import log_cleanup from common.constants import DAG_DEFAULT_ARGS diff --git a/openverse_catalog/dags/maintenance/check_silenced_dags.py b/openverse_catalog/dags/maintenance/check_silenced_dags.py index f130f9789..868e0b323 100644 --- a/openverse_catalog/dags/maintenance/check_silenced_dags.py +++ b/openverse_catalog/dags/maintenance/check_silenced_dags.py @@ -28,6 +28,7 @@ from airflow.exceptions import AirflowException, AirflowSkipException from airflow.models import DAG, Variable from airflow.operators.python import PythonOperator + from common.constants import DAG_DEFAULT_ARGS from common.github import GitHubAPI from common.slack import SilencedSlackNotification, send_alert diff --git a/openverse_catalog/dags/maintenance/flickr_audit_sub_provider_workflow.py b/openverse_catalog/dags/maintenance/flickr_audit_sub_provider_workflow.py index f23ecf520..81250f4ab 100644 --- a/openverse_catalog/dags/maintenance/flickr_audit_sub_provider_workflow.py +++ b/openverse_catalog/dags/maintenance/flickr_audit_sub_provider_workflow.py @@ -13,6 +13,7 @@ from airflow.exceptions import AirflowSkipException from airflow.models import Variable from airflow.operators.python import PythonOperator + from common.constants import DAG_DEFAULT_ARGS from common.loader import provider_details as prov from common.requester import DelayedRequester diff --git a/openverse_catalog/dags/maintenance/pr_review_reminders/pr_review_reminders_dag.py b/openverse_catalog/dags/maintenance/pr_review_reminders/pr_review_reminders_dag.py index 590a9d256..e5050c472 100644 --- a/openverse_catalog/dags/maintenance/pr_review_reminders/pr_review_reminders_dag.py +++ b/openverse_catalog/dags/maintenance/pr_review_reminders/pr_review_reminders_dag.py @@ -25,6 +25,7 @@ from airflow.models import DAG from airflow.operators.python import PythonOperator + from common.constants import DAG_DEFAULT_ARGS from maintenance.pr_review_reminders import pr_review_reminders diff --git a/openverse_catalog/dags/oauth2/authorize_dag.py b/openverse_catalog/dags/oauth2/authorize_dag.py index 6405496ff..a02238ed9 100644 --- a/openverse_catalog/dags/oauth2/authorize_dag.py +++ b/openverse_catalog/dags/oauth2/authorize_dag.py @@ -1,9 +1,10 @@ """# OAuth Provider Authorization""" from datetime import datetime -import oauth2 from airflow.models import DAG from airflow.operators.python import PythonOperator + +import oauth2 from common.constants import DAG_DEFAULT_ARGS diff --git a/openverse_catalog/dags/oauth2/token_refresh_dag.py b/openverse_catalog/dags/oauth2/token_refresh_dag.py index 93f1a0b6e..56c65309a 100644 --- a/openverse_catalog/dags/oauth2/token_refresh_dag.py +++ b/openverse_catalog/dags/oauth2/token_refresh_dag.py @@ -1,9 +1,10 @@ """# OAuth Provider Token Refresh""" from datetime import datetime -import oauth2 from airflow.models import DAG from airflow.operators.python import PythonOperator + +import oauth2 from common.constants import DAG_DEFAULT_ARGS diff --git a/openverse_catalog/dags/providers/factory_utils.py b/openverse_catalog/dags/providers/factory_utils.py index ca66e51ef..eccc73ad8 100644 --- a/openverse_catalog/dags/providers/factory_utils.py +++ b/openverse_catalog/dags/providers/factory_utils.py @@ -5,6 +5,7 @@ from airflow.models import DagRun, TaskInstance from airflow.utils.dates import cron_presets + from common.constants import MediaType from common.storage.media import MediaStore from providers.provider_api_scripts.provider_data_ingester import ProviderDataIngester diff --git a/openverse_catalog/dags/providers/provider_api_scripts/brooklyn_museum.py b/openverse_catalog/dags/providers/provider_api_scripts/brooklyn_museum.py index a63670b8c..f1eb94980 100644 --- a/openverse_catalog/dags/providers/provider_api_scripts/brooklyn_museum.py +++ b/openverse_catalog/dags/providers/provider_api_scripts/brooklyn_museum.py @@ -2,6 +2,7 @@ import lxml.html as html from airflow.models import Variable + from common import constants from common.licenses import get_license_info from common.loader import provider_details as prov diff --git a/openverse_catalog/dags/providers/provider_api_scripts/europeana.py b/openverse_catalog/dags/providers/provider_api_scripts/europeana.py index 0fcd7ae94..5f52ea4d4 100644 --- a/openverse_catalog/dags/providers/provider_api_scripts/europeana.py +++ b/openverse_catalog/dags/providers/provider_api_scripts/europeana.py @@ -13,8 +13,9 @@ import logging from datetime import datetime, timedelta, timezone -import common from airflow.models import Variable + +import common from common.licenses import get_license_info from common.loader import provider_details as prov from providers.provider_api_scripts.provider_data_ingester import ProviderDataIngester diff --git a/openverse_catalog/dags/providers/provider_api_scripts/flickr.py b/openverse_catalog/dags/providers/provider_api_scripts/flickr.py index f3f357ca6..85d026e82 100644 --- a/openverse_catalog/dags/providers/provider_api_scripts/flickr.py +++ b/openverse_catalog/dags/providers/provider_api_scripts/flickr.py @@ -16,6 +16,7 @@ import lxml.html as html from airflow.models import Variable + from common import constants from common.licenses import get_license_info from common.loader import provider_details as prov diff --git a/openverse_catalog/dags/providers/provider_api_scripts/freesound.py b/openverse_catalog/dags/providers/provider_api_scripts/freesound.py index 7328cd848..03572a262 100644 --- a/openverse_catalog/dags/providers/provider_api_scripts/freesound.py +++ b/openverse_catalog/dags/providers/provider_api_scripts/freesound.py @@ -16,13 +16,14 @@ from datetime import datetime from airflow.models import Variable +from requests.exceptions import ConnectionError, SSLError +from retry import retry + from common import constants from common.licenses.licenses import get_license_info from common.loader import provider_details as prov from common.requester import RetriesExceeded from providers.provider_api_scripts.provider_data_ingester import ProviderDataIngester -from requests.exceptions import ConnectionError, SSLError -from retry import retry logger = logging.getLogger(__name__) diff --git a/openverse_catalog/dags/providers/provider_api_scripts/inaturalist.py b/openverse_catalog/dags/providers/provider_api_scripts/inaturalist.py index 5295303f4..fa889857b 100644 --- a/openverse_catalog/dags/providers/provider_api_scripts/inaturalist.py +++ b/openverse_catalog/dags/providers/provider_api_scripts/inaturalist.py @@ -32,6 +32,7 @@ from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.utils.task_group import TaskGroup from airflow.utils.trigger_rule import TriggerRule + from common.constants import IMAGE, POSTGRES_CONN_ID, XCOM_PULL_TEMPLATE from common.loader import provider_details, reporting, sql from common.sql import PGExecuteQueryOperator, PostgresHook diff --git a/openverse_catalog/dags/providers/provider_api_scripts/jamendo.py b/openverse_catalog/dags/providers/provider_api_scripts/jamendo.py index aa90c2780..a5725d2f7 100644 --- a/openverse_catalog/dags/providers/provider_api_scripts/jamendo.py +++ b/openverse_catalog/dags/providers/provider_api_scripts/jamendo.py @@ -18,8 +18,9 @@ import logging from urllib.parse import parse_qs, urlencode, urlsplit -import common from airflow.models import Variable + +import common from common import constants from common.licenses import get_license_info from common.loader import provider_details as prov diff --git a/openverse_catalog/dags/providers/provider_api_scripts/metropolitan_museum.py b/openverse_catalog/dags/providers/provider_api_scripts/metropolitan_museum.py index 7b848e82e..5b0cdecbf 100644 --- a/openverse_catalog/dags/providers/provider_api_scripts/metropolitan_museum.py +++ b/openverse_catalog/dags/providers/provider_api_scripts/metropolitan_museum.py @@ -28,9 +28,10 @@ import argparse import logging +from provider_data_ingester import ProviderDataIngester + from common.licenses import get_license_info from common.loader import provider_details as prov -from provider_data_ingester import ProviderDataIngester logging.basicConfig( diff --git a/openverse_catalog/dags/providers/provider_api_scripts/nypl.py b/openverse_catalog/dags/providers/provider_api_scripts/nypl.py index d594ffe78..f86e0222e 100644 --- a/openverse_catalog/dags/providers/provider_api_scripts/nypl.py +++ b/openverse_catalog/dags/providers/provider_api_scripts/nypl.py @@ -3,6 +3,7 @@ from urllib.parse import parse_qs, urlparse from airflow.models import Variable + from common import constants from common.licenses import get_license_info from common.loader import provider_details as prov diff --git a/openverse_catalog/dags/providers/provider_api_scripts/provider_data_ingester.py b/openverse_catalog/dags/providers/provider_api_scripts/provider_data_ingester.py index edaf56fb6..d6ba0cf28 100644 --- a/openverse_catalog/dags/providers/provider_api_scripts/provider_data_ingester.py +++ b/openverse_catalog/dags/providers/provider_api_scripts/provider_data_ingester.py @@ -7,6 +7,7 @@ from airflow.exceptions import AirflowException from airflow.models import Variable + from common.requester import DelayedRequester from common.storage.media import MediaStore from common.storage.util import get_media_store_class diff --git a/openverse_catalog/dags/providers/provider_api_scripts/rawpixel.py b/openverse_catalog/dags/providers/provider_api_scripts/rawpixel.py index 2cbe81e22..acde719bb 100644 --- a/openverse_catalog/dags/providers/provider_api_scripts/rawpixel.py +++ b/openverse_catalog/dags/providers/provider_api_scripts/rawpixel.py @@ -20,6 +20,7 @@ from urllib.parse import urlencode from airflow.models import Variable + from common import constants from common.licenses import NO_LICENSE_FOUND, get_license_info from common.loader import provider_details as prov diff --git a/openverse_catalog/dags/providers/provider_api_scripts/smithsonian.py b/openverse_catalog/dags/providers/provider_api_scripts/smithsonian.py index 519ca7619..04d619835 100644 --- a/openverse_catalog/dags/providers/provider_api_scripts/smithsonian.py +++ b/openverse_catalog/dags/providers/provider_api_scripts/smithsonian.py @@ -12,11 +12,12 @@ from airflow.exceptions import AirflowException from airflow.models import Variable +from retry import retry + from common import constants from common.licenses import get_license_info from common.loader import provider_details as prov from providers.provider_api_scripts.provider_data_ingester import ProviderDataIngester -from retry import retry logger = logging.getLogger(__name__) diff --git a/openverse_catalog/dags/providers/provider_api_scripts/wikimedia_commons.py b/openverse_catalog/dags/providers/provider_api_scripts/wikimedia_commons.py index 662dd2a7d..79716be81 100644 --- a/openverse_catalog/dags/providers/provider_api_scripts/wikimedia_commons.py +++ b/openverse_catalog/dags/providers/provider_api_scripts/wikimedia_commons.py @@ -113,6 +113,7 @@ from types import MappingProxyType import lxml.html as html + from common.constants import AUDIO, IMAGE from common.licenses import get_license_info from common.loader import provider_details as prov diff --git a/openverse_catalog/dags/providers/provider_api_scripts/wordpress.py b/openverse_catalog/dags/providers/provider_api_scripts/wordpress.py index 62754cdb8..1731dcd15 100644 --- a/openverse_catalog/dags/providers/provider_api_scripts/wordpress.py +++ b/openverse_catalog/dags/providers/provider_api_scripts/wordpress.py @@ -12,6 +12,7 @@ import logging import lxml.html as html + from common import constants from common.licenses import get_license_info from common.loader import provider_details as prov diff --git a/openverse_catalog/dags/providers/provider_dag_factory.py b/openverse_catalog/dags/providers/provider_dag_factory.py index fbfe8ff4a..ebc326324 100644 --- a/openverse_catalog/dags/providers/provider_dag_factory.py +++ b/openverse_catalog/dags/providers/provider_dag_factory.py @@ -74,6 +74,7 @@ from airflow.operators.python import PythonOperator from airflow.utils.task_group import TaskGroup from airflow.utils.trigger_rule import TriggerRule + from common.constants import DAG_DEFAULT_ARGS, XCOM_PULL_TEMPLATE from common.loader import loader, reporting, s3, sql from providers.factory_utils import date_partition_for_prefix, pull_media_wrapper diff --git a/openverse_catalog/dags/providers/provider_workflows.py b/openverse_catalog/dags/providers/provider_workflows.py index 3519033ae..f14454c1e 100644 --- a/openverse_catalog/dags/providers/provider_workflows.py +++ b/openverse_catalog/dags/providers/provider_workflows.py @@ -6,6 +6,8 @@ from datetime import datetime, timedelta from airflow.models import Variable +from typing_extensions import NotRequired, TypedDict + from providers.provider_api_scripts.brooklyn_museum import BrooklynMuseumDataIngester from providers.provider_api_scripts.cleveland_museum import ClevelandDataIngester from providers.provider_api_scripts.europeana import EuropeanaDataIngester @@ -29,7 +31,6 @@ WikimediaCommonsDataIngester, ) from providers.provider_api_scripts.wordpress import WordPressDataIngester -from typing_extensions import NotRequired, TypedDict logger = logging.getLogger(__name__) diff --git a/openverse_catalog/utilities/dag_doc_gen/dag_doc_generation.py b/openverse_catalog/utilities/dag_doc_gen/dag_doc_generation.py index 8550eb945..2590ab9ec 100644 --- a/openverse_catalog/utilities/dag_doc_gen/dag_doc_generation.py +++ b/openverse_catalog/utilities/dag_doc_gen/dag_doc_generation.py @@ -19,6 +19,7 @@ from typing import NamedTuple from airflow.models import DAG, DagBag + from providers.provider_workflows import PROVIDER_WORKFLOWS, ProviderWorkflow diff --git a/tests/dags/common/conftest.py b/tests/dags/common/conftest.py index 1bbbf8775..aa55b39c1 100644 --- a/tests/dags/common/conftest.py +++ b/tests/dags/common/conftest.py @@ -7,9 +7,9 @@ from airflow import DAG from airflow.models.abstractoperator import AbstractOperator from airflow.operators.python import PythonOperator + from common.constants import POSTGRES_CONN_ID from common.sql import PGExecuteQueryOperator, PostgresHook - from tests.dags.common.loader.test_s3 import ACCESS_KEY, S3_LOCAL_ENDPOINT, SECRET_KEY diff --git a/tests/dags/common/licenses/test_licenses.py b/tests/dags/common/licenses/test_licenses.py index 7af605517..adcb8809e 100644 --- a/tests/dags/common/licenses/test_licenses.py +++ b/tests/dags/common/licenses/test_licenses.py @@ -1,9 +1,10 @@ import logging from unittest.mock import patch -import common.urls import pytest import tldextract + +import common.urls from common.licenses import licenses diff --git a/tests/dags/common/loader/test_loader.py b/tests/dags/common/loader/test_loader.py index 6419625b1..10c0dd6f2 100644 --- a/tests/dags/common/loader/test_loader.py +++ b/tests/dags/common/loader/test_loader.py @@ -1,6 +1,7 @@ from unittest import mock import pytest + from common.loader import loader from common.loader.reporting import RecordMetrics diff --git a/tests/dags/common/loader/test_reporting.py b/tests/dags/common/loader/test_reporting.py index eb7eb4495..102617138 100644 --- a/tests/dags/common/loader/test_reporting.py +++ b/tests/dags/common/loader/test_reporting.py @@ -1,6 +1,7 @@ from unittest import mock import pytest + from common.loader.reporting import ( RecordMetrics, clean_duration, diff --git a/tests/dags/common/loader/test_s3.py b/tests/dags/common/loader/test_s3.py index 1a5733b71..3b935af0b 100644 --- a/tests/dags/common/loader/test_s3.py +++ b/tests/dags/common/loader/test_s3.py @@ -4,6 +4,7 @@ import pytest from airflow.exceptions import AirflowSkipException from airflow.models import TaskInstance + from common.loader import s3 diff --git a/tests/dags/common/loader/test_sql.py b/tests/dags/common/loader/test_sql.py index 3a60a9dc5..74c690655 100644 --- a/tests/dags/common/loader/test_sql.py +++ b/tests/dags/common/loader/test_sql.py @@ -8,13 +8,14 @@ import psycopg2 import pytest from airflow.models import TaskInstance +from flaky import flaky +from psycopg2.errors import InvalidTextRepresentation + from common.constants import IMAGE from common.loader import sql from common.loader.sql import TSV_COLUMNS, create_column_definitions from common.storage import columns as col from common.storage.db_columns import IMAGE_TABLE_COLUMNS -from flaky import flaky -from psycopg2.errors import InvalidTextRepresentation POSTGRES_CONN_ID = os.getenv("TEST_CONN_ID") diff --git a/tests/dags/common/popularity/test_sql.py b/tests/dags/common/popularity/test_sql.py index 239322d81..41ca0fca2 100644 --- a/tests/dags/common/popularity/test_sql.py +++ b/tests/dags/common/popularity/test_sql.py @@ -7,6 +7,7 @@ import psycopg2 import pytest + from common.popularity import sql diff --git a/tests/dags/common/sensors/test_single_run_external_dags_sensor.py b/tests/dags/common/sensors/test_single_run_external_dags_sensor.py index 142aee650..b1b1445f0 100644 --- a/tests/dags/common/sensors/test_single_run_external_dags_sensor.py +++ b/tests/dags/common/sensors/test_single_run_external_dags_sensor.py @@ -9,6 +9,7 @@ from airflow.utils.state import State from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType + from common.sensors.single_run_external_dags_sensor import SingleRunExternalDAGsSensor diff --git a/tests/dags/common/storage/test_audio.py b/tests/dags/common/storage/test_audio.py index a8fb61fe9..36cd6cf3b 100644 --- a/tests/dags/common/storage/test_audio.py +++ b/tests/dags/common/storage/test_audio.py @@ -1,9 +1,9 @@ import logging import pytest + from common.licenses import LicenseInfo from common.storage import audio - from tests.dags.common.storage import test_media diff --git a/tests/dags/common/storage/test_columns.py b/tests/dags/common/storage/test_columns.py index 12c40e396..cfbe8ba49 100644 --- a/tests/dags/common/storage/test_columns.py +++ b/tests/dags/common/storage/test_columns.py @@ -2,6 +2,7 @@ import string import tldextract + from common import urls from common.storage import columns from common.storage.columns import Datatype diff --git a/tests/dags/common/storage/test_image.py b/tests/dags/common/storage/test_image.py index 35b363320..ecf0ad159 100644 --- a/tests/dags/common/storage/test_image.py +++ b/tests/dags/common/storage/test_image.py @@ -1,6 +1,7 @@ import logging import pytest + from common import urls from common.licenses import LicenseInfo from common.storage import image diff --git a/tests/dags/common/storage/test_media.py b/tests/dags/common/storage/test_media.py index 842163910..b04a56de1 100644 --- a/tests/dags/common/storage/test_media.py +++ b/tests/dags/common/storage/test_media.py @@ -6,6 +6,7 @@ from unittest.mock import patch import pytest + from common import urls from common.licenses import LicenseInfo, get_license_info from common.loader import provider_details as prov diff --git a/tests/dags/common/storage/test_util.py b/tests/dags/common/storage/test_util.py index d5a3263ee..1d9f30726 100644 --- a/tests/dags/common/storage/test_util.py +++ b/tests/dags/common/storage/test_util.py @@ -1,6 +1,7 @@ import logging import pytest + from common.storage import util from common.storage.audio import AudioStore from common.storage.image import ImageStore diff --git a/tests/dags/common/test_extensions.py b/tests/dags/common/test_extensions.py index 17da58b97..e1a021381 100644 --- a/tests/dags/common/test_extensions.py +++ b/tests/dags/common/test_extensions.py @@ -1,4 +1,5 @@ import pytest + from common import extensions diff --git a/tests/dags/common/test_github.py b/tests/dags/common/test_github.py index dae9c47ee..fccd527a1 100644 --- a/tests/dags/common/test_github.py +++ b/tests/dags/common/test_github.py @@ -1,8 +1,9 @@ import pook import pytest -from common.github import GitHubAPI from requests import HTTPError +from common.github import GitHubAPI + SAMPLE_PAT = "foobar" diff --git a/tests/dags/common/test_requester.py b/tests/dags/common/test_requester.py index bc63c20db..3669919fa 100644 --- a/tests/dags/common/test_requester.py +++ b/tests/dags/common/test_requester.py @@ -4,9 +4,9 @@ import pytest import requests -from common import requester from requests_oauthlib import OAuth2Session +from common import requester from tests.dags.conftest import FAKE_OAUTH_PROVIDER_NAME diff --git a/tests/dags/common/test_slack.py b/tests/dags/common/test_slack.py index bcbf13869..f58354d6a 100644 --- a/tests/dags/common/test_slack.py +++ b/tests/dags/common/test_slack.py @@ -3,6 +3,7 @@ import pytest from airflow.exceptions import AirflowNotFoundException + from common.slack import ( SLACK_ALERTS_CONN_ID, SlackMessage, diff --git a/tests/dags/common/test_sql.py b/tests/dags/common/test_sql.py index 4d2169d61..d66efca46 100644 --- a/tests/dags/common/test_sql.py +++ b/tests/dags/common/test_sql.py @@ -3,10 +3,10 @@ from datetime import datetime, timedelta import pytest -from common.constants import DAG_DEFAULT_ARGS, POSTGRES_CONN_ID -from common.sql import PGExecuteQueryOperator, PostgresHook from psycopg2.errors import QueryCanceled +from common.constants import DAG_DEFAULT_ARGS, POSTGRES_CONN_ID +from common.sql import PGExecuteQueryOperator, PostgresHook from tests.conftest import mark_extended diff --git a/tests/dags/common/test_urls.py b/tests/dags/common/test_urls.py index e9f7cdebd..c64f013b4 100644 --- a/tests/dags/common/test_urls.py +++ b/tests/dags/common/test_urls.py @@ -3,9 +3,10 @@ import pytest import requests -from common import urls from requests import RequestException +from common import urls + logging.basicConfig( format="%(asctime)s - %(name)s - %(levelname)s: %(message)s", level=logging.DEBUG diff --git a/tests/dags/conftest.py b/tests/dags/conftest.py index 8ff82f945..d994ae672 100644 --- a/tests/dags/conftest.py +++ b/tests/dags/conftest.py @@ -1,9 +1,10 @@ from unittest import mock import pytest -from oauth2 import oauth2 from requests import Response +from oauth2 import oauth2 + FAKE_OAUTH_PROVIDER_NAME = "fakeprovider" diff --git a/tests/dags/data_refresh/test_dag_factory.py b/tests/dags/data_refresh/test_dag_factory.py index 5ba34fe0f..fa5d31c68 100644 --- a/tests/dags/data_refresh/test_dag_factory.py +++ b/tests/dags/data_refresh/test_dag_factory.py @@ -7,6 +7,7 @@ from airflow.utils.state import State from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType + from data_refresh import dag_factory from data_refresh.dag_factory import ( REFRESH_MATERIALIZED_VIEW_TASK_ID, diff --git a/tests/dags/data_refresh/test_reporting.py b/tests/dags/data_refresh/test_reporting.py index d5c1cfeec..1aaf2df1c 100644 --- a/tests/dags/data_refresh/test_reporting.py +++ b/tests/dags/data_refresh/test_reporting.py @@ -1,4 +1,5 @@ import pytest + from data_refresh.reporting import report_record_difference, report_status diff --git a/tests/dags/database/test_report_pending_reported_media.py b/tests/dags/database/test_report_pending_reported_media.py index 55941efbc..9e7c2ca72 100644 --- a/tests/dags/database/test_report_pending_reported_media.py +++ b/tests/dags/database/test_report_pending_reported_media.py @@ -1,6 +1,7 @@ from unittest import mock import pytest + from database.report_pending_reported_media import report_actionable_records diff --git a/tests/dags/maintenance/test_check_silenced_dags.py b/tests/dags/maintenance/test_check_silenced_dags.py index 7ed42859a..00448d05a 100644 --- a/tests/dags/maintenance/test_check_silenced_dags.py +++ b/tests/dags/maintenance/test_check_silenced_dags.py @@ -2,12 +2,12 @@ import pytest from airflow.exceptions import AirflowException, AirflowSkipException + from maintenance.check_silenced_dags import ( check_configuration, get_dags_with_closed_issues, get_issue_info, ) - from tests.factories.github import make_issue diff --git a/tests/dags/maintenance/test_flickr_audit_sub_provider.py b/tests/dags/maintenance/test_flickr_audit_sub_provider.py index 8a7cdc2ab..963668dab 100644 --- a/tests/dags/maintenance/test_flickr_audit_sub_provider.py +++ b/tests/dags/maintenance/test_flickr_audit_sub_provider.py @@ -2,6 +2,7 @@ import pytest from airflow.exceptions import AirflowSkipException + from maintenance.flickr_audit_sub_provider_workflow import ( FlickrSubProviderAuditor, audit_flickr_sub_providers, diff --git a/tests/dags/maintenance/test_rotate_db_snapshots.py b/tests/dags/maintenance/test_rotate_db_snapshots.py index 0a0a19be5..80a1a1610 100644 --- a/tests/dags/maintenance/test_rotate_db_snapshots.py +++ b/tests/dags/maintenance/test_rotate_db_snapshots.py @@ -4,6 +4,7 @@ import boto3 import pytest + from maintenance.rotate_db_snapshots import delete_previous_snapshots diff --git a/tests/dags/oauth2/test_oauth2.py b/tests/dags/oauth2/test_oauth2.py index 1a7b2be51..8639eef4b 100644 --- a/tests/dags/oauth2/test_oauth2.py +++ b/tests/dags/oauth2/test_oauth2.py @@ -2,8 +2,8 @@ import pytest from airflow.exceptions import AirflowSkipException -from oauth2 import oauth2 +from oauth2 import oauth2 from tests.dags.conftest import FAKE_OAUTH_PROVIDER_NAME diff --git a/tests/dags/providers/provider_api_scripts/test_brooklyn_museum.py b/tests/dags/providers/provider_api_scripts/test_brooklyn_museum.py index 252b0e7e6..1e9ad6c36 100644 --- a/tests/dags/providers/provider_api_scripts/test_brooklyn_museum.py +++ b/tests/dags/providers/provider_api_scripts/test_brooklyn_museum.py @@ -3,6 +3,7 @@ from unittest.mock import patch import pytest + from common.licenses import LicenseInfo from common.loader import provider_details as prov from common.storage.image import ImageStore diff --git a/tests/dags/providers/provider_api_scripts/test_cleveland_museum.py b/tests/dags/providers/provider_api_scripts/test_cleveland_museum.py index 0e101627f..40d2ad2d9 100644 --- a/tests/dags/providers/provider_api_scripts/test_cleveland_museum.py +++ b/tests/dags/providers/provider_api_scripts/test_cleveland_museum.py @@ -5,6 +5,7 @@ import pytest import requests + from common.licenses import LicenseInfo from common.loader import provider_details as prov from common.storage.image import ImageStore diff --git a/tests/dags/providers/provider_api_scripts/test_europeana.py b/tests/dags/providers/provider_api_scripts/test_europeana.py index 538995655..3f972ec7c 100644 --- a/tests/dags/providers/provider_api_scripts/test_europeana.py +++ b/tests/dags/providers/provider_api_scripts/test_europeana.py @@ -2,6 +2,7 @@ import os import pytest + from common.licenses import LicenseInfo, get_license_info from providers.provider_api_scripts.europeana import ( EuropeanaDataIngester, diff --git a/tests/dags/providers/provider_api_scripts/test_finnish_museums.py b/tests/dags/providers/provider_api_scripts/test_finnish_museums.py index 741d0baa3..c081cbec0 100644 --- a/tests/dags/providers/provider_api_scripts/test_finnish_museums.py +++ b/tests/dags/providers/provider_api_scripts/test_finnish_museums.py @@ -5,6 +5,7 @@ from unittest.mock import patch import pytest + from common.licenses import LicenseInfo from common.loader import provider_details as prov from common.storage.image import ImageStore diff --git a/tests/dags/providers/provider_api_scripts/test_flickr.py b/tests/dags/providers/provider_api_scripts/test_flickr.py index ab85e2c3a..d568f1ece 100644 --- a/tests/dags/providers/provider_api_scripts/test_flickr.py +++ b/tests/dags/providers/provider_api_scripts/test_flickr.py @@ -4,6 +4,7 @@ from unittest import mock import pytest + from common.licenses import LicenseInfo from providers.provider_api_scripts.flickr import FlickrDataIngester diff --git a/tests/dags/providers/provider_api_scripts/test_freesound.py b/tests/dags/providers/provider_api_scripts/test_freesound.py index edb7df769..c92fe1bce 100644 --- a/tests/dags/providers/provider_api_scripts/test_freesound.py +++ b/tests/dags/providers/provider_api_scripts/test_freesound.py @@ -3,6 +3,7 @@ from unittest.mock import MagicMock, patch import pytest + from common.licenses.licenses import LicenseInfo from providers.provider_api_scripts.freesound import FreesoundDataIngester diff --git a/tests/dags/providers/provider_api_scripts/test_inaturalist.py b/tests/dags/providers/provider_api_scripts/test_inaturalist.py index a20a5a46a..de7ec907f 100644 --- a/tests/dags/providers/provider_api_scripts/test_inaturalist.py +++ b/tests/dags/providers/provider_api_scripts/test_inaturalist.py @@ -6,6 +6,7 @@ import pytest from airflow.exceptions import AirflowSkipException from airflow.models import TaskInstance + from common.constants import IMAGE from common.loader.reporting import RecordMetrics from common.sql import PostgresHook diff --git a/tests/dags/providers/provider_api_scripts/test_jamendo.py b/tests/dags/providers/provider_api_scripts/test_jamendo.py index c2686b322..c7e744059 100644 --- a/tests/dags/providers/provider_api_scripts/test_jamendo.py +++ b/tests/dags/providers/provider_api_scripts/test_jamendo.py @@ -3,6 +3,7 @@ from unittest.mock import patch import pytest + from common.licenses import LicenseInfo from providers.provider_api_scripts.jamendo import JamendoDataIngester diff --git a/tests/dags/providers/provider_api_scripts/test_metropolitan_museum.py b/tests/dags/providers/provider_api_scripts/test_metropolitan_museum.py index b0d2c503e..458e91905 100644 --- a/tests/dags/providers/provider_api_scripts/test_metropolitan_museum.py +++ b/tests/dags/providers/provider_api_scripts/test_metropolitan_museum.py @@ -5,6 +5,7 @@ import pytest import requests + from common.licenses import LicenseInfo from providers.provider_api_scripts.metropolitan_museum import MetMuseumDataIngester diff --git a/tests/dags/providers/provider_api_scripts/test_museum_victoria.py b/tests/dags/providers/provider_api_scripts/test_museum_victoria.py index 50c69ca51..2ca648c27 100644 --- a/tests/dags/providers/provider_api_scripts/test_museum_victoria.py +++ b/tests/dags/providers/provider_api_scripts/test_museum_victoria.py @@ -4,6 +4,7 @@ from unittest.mock import patch import pytest + from common.licenses import LicenseInfo, get_license_info from common.loader import provider_details as prov from common.storage.image import ImageStore diff --git a/tests/dags/providers/provider_api_scripts/test_nappy.py b/tests/dags/providers/provider_api_scripts/test_nappy.py index 6341d6e86..2b4566a70 100644 --- a/tests/dags/providers/provider_api_scripts/test_nappy.py +++ b/tests/dags/providers/provider_api_scripts/test_nappy.py @@ -3,6 +3,7 @@ from pathlib import Path import pytest + from common.constants import IMAGE from common.licenses import get_license_info from providers.provider_api_scripts.nappy import NappyDataIngester diff --git a/tests/dags/providers/provider_api_scripts/test_nypl.py b/tests/dags/providers/provider_api_scripts/test_nypl.py index 8208f5342..7b7b23420 100644 --- a/tests/dags/providers/provider_api_scripts/test_nypl.py +++ b/tests/dags/providers/provider_api_scripts/test_nypl.py @@ -3,6 +3,7 @@ from unittest.mock import patch import pytest + from common.licenses import LicenseInfo from providers.provider_api_scripts.nypl import ( NyplDataIngester, diff --git a/tests/dags/providers/provider_api_scripts/test_phylopic.py b/tests/dags/providers/provider_api_scripts/test_phylopic.py index 833480de4..cb91f18de 100644 --- a/tests/dags/providers/provider_api_scripts/test_phylopic.py +++ b/tests/dags/providers/provider_api_scripts/test_phylopic.py @@ -3,6 +3,7 @@ from unittest.mock import patch import pytest + from common.licenses import LicenseInfo from providers.provider_api_scripts.phylopic import PhylopicDataIngester diff --git a/tests/dags/providers/provider_api_scripts/test_provider_data_ingester.py b/tests/dags/providers/provider_api_scripts/test_provider_data_ingester.py index 7e193abb6..553227b6b 100644 --- a/tests/dags/providers/provider_api_scripts/test_provider_data_ingester.py +++ b/tests/dags/providers/provider_api_scripts/test_provider_data_ingester.py @@ -5,13 +5,13 @@ import pytest import requests from airflow.exceptions import AirflowException + from common.requester import RetriesExceeded from common.storage.audio import AudioStore, MockAudioStore from common.storage.image import ImageStore, MockImageStore from providers.provider_api_scripts.provider_data_ingester import ( AggregateIngestionError, ) - from tests.dags.providers.provider_api_scripts.resources.provider_data_ingester.mock_provider_data_ingester import ( AUDIO_PROVIDER, EXPECTED_BATCH_DATA, diff --git a/tests/dags/providers/provider_api_scripts/test_rawpixel.py b/tests/dags/providers/provider_api_scripts/test_rawpixel.py index f543e0442..003465835 100644 --- a/tests/dags/providers/provider_api_scripts/test_rawpixel.py +++ b/tests/dags/providers/provider_api_scripts/test_rawpixel.py @@ -2,6 +2,7 @@ from pathlib import Path import pytest + from common.licenses import LicenseInfo from common.loader.provider_details import ImageCategory from providers.provider_api_scripts.rawpixel import RawpixelDataIngester diff --git a/tests/dags/providers/provider_api_scripts/test_science_museum.py b/tests/dags/providers/provider_api_scripts/test_science_museum.py index b9dc56b18..433b5721f 100644 --- a/tests/dags/providers/provider_api_scripts/test_science_museum.py +++ b/tests/dags/providers/provider_api_scripts/test_science_museum.py @@ -5,6 +5,7 @@ from unittest.mock import patch import pytest + from common.licenses import LicenseInfo from common.loader import provider_details as prov from common.storage.image import ImageStore diff --git a/tests/dags/providers/provider_api_scripts/test_smithsonian.py b/tests/dags/providers/provider_api_scripts/test_smithsonian.py index 4509883ee..d96df29c2 100644 --- a/tests/dags/providers/provider_api_scripts/test_smithsonian.py +++ b/tests/dags/providers/provider_api_scripts/test_smithsonian.py @@ -4,6 +4,7 @@ import pytest from airflow.exceptions import AirflowException + from providers.provider_api_scripts.smithsonian import SmithsonianDataIngester diff --git a/tests/dags/providers/provider_api_scripts/test_smk.py b/tests/dags/providers/provider_api_scripts/test_smk.py index 4d8f0a721..7f3230289 100644 --- a/tests/dags/providers/provider_api_scripts/test_smk.py +++ b/tests/dags/providers/provider_api_scripts/test_smk.py @@ -2,6 +2,7 @@ from pathlib import Path import pytest + from common.licenses import LicenseInfo from providers.provider_api_scripts.smk import SmkDataIngester diff --git a/tests/dags/providers/provider_api_scripts/test_stocksnap.py b/tests/dags/providers/provider_api_scripts/test_stocksnap.py index e7a74518e..d7caf9ffb 100644 --- a/tests/dags/providers/provider_api_scripts/test_stocksnap.py +++ b/tests/dags/providers/provider_api_scripts/test_stocksnap.py @@ -4,6 +4,7 @@ from unittest.mock import patch import pytest + from common.licenses import get_license_info from common.loader import provider_details as prov from common.storage.image import ImageStore diff --git a/tests/dags/providers/provider_api_scripts/test_wikimedia_commons.py b/tests/dags/providers/provider_api_scripts/test_wikimedia_commons.py index 3d61c748e..15bfd0c1e 100644 --- a/tests/dags/providers/provider_api_scripts/test_wikimedia_commons.py +++ b/tests/dags/providers/provider_api_scripts/test_wikimedia_commons.py @@ -4,6 +4,7 @@ from unittest.mock import Mock, patch import pytest + from common.constants import IMAGE from common.licenses import get_license_info from providers.provider_api_scripts.wikimedia_commons import ( diff --git a/tests/dags/providers/provider_api_scripts/test_wordpress.py b/tests/dags/providers/provider_api_scripts/test_wordpress.py index 54311bcbd..b1341bb45 100644 --- a/tests/dags/providers/provider_api_scripts/test_wordpress.py +++ b/tests/dags/providers/provider_api_scripts/test_wordpress.py @@ -3,6 +3,7 @@ from unittest.mock import MagicMock, patch import pytest + from providers.provider_api_scripts.wordpress import WordPressDataIngester diff --git a/tests/dags/providers/test_factory_utils.py b/tests/dags/providers/test_factory_utils.py index 1a0201059..8da5390a9 100644 --- a/tests/dags/providers/test_factory_utils.py +++ b/tests/dags/providers/test_factory_utils.py @@ -3,8 +3,8 @@ import pytest from airflow.models import DagRun, TaskInstance -from providers import factory_utils +from providers import factory_utils from tests.dags.common.test_resources.fake_provider_data_ingester import ( FakeDataIngester, ) diff --git a/tests/dags/providers/test_provider_dag_factory.py b/tests/dags/providers/test_provider_dag_factory.py index c9a3f398e..ee0a85c08 100644 --- a/tests/dags/providers/test_provider_dag_factory.py +++ b/tests/dags/providers/test_provider_dag_factory.py @@ -9,10 +9,10 @@ from airflow.operators.empty import EmptyOperator from airflow.utils.session import create_session from pendulum import now + from providers import provider_dag_factory from providers.provider_reingestion_workflows import ProviderReingestionWorkflow from providers.provider_workflows import ProviderWorkflow - from tests.conftest import mark_extended from tests.dags.providers.provider_api_scripts.resources.provider_data_ingester.mock_provider_data_ingester import ( MockAudioOnlyProviderDataIngester, diff --git a/tests/dags/providers/test_provider_workflows.py b/tests/dags/providers/test_provider_workflows.py index bb6472e42..694ac8902 100644 --- a/tests/dags/providers/test_provider_workflows.py +++ b/tests/dags/providers/test_provider_workflows.py @@ -2,8 +2,8 @@ from unittest import mock import pytest -from providers.provider_workflows import ProviderWorkflow, get_time_override +from providers.provider_workflows import ProviderWorkflow, get_time_override from tests.dags.providers.provider_api_scripts.resources.provider_data_ingester.mock_provider_data_ingester import ( MockAudioOnlyProviderDataIngester, MockImageOnlyProviderDataIngester, diff --git a/tests/dags/test_dag_parsing.py b/tests/dags/test_dag_parsing.py index 4202194bf..6fd30ac08 100644 --- a/tests/dags/test_dag_parsing.py +++ b/tests/dags/test_dag_parsing.py @@ -2,6 +2,7 @@ import pytest from airflow.models import DagBag + from common.constants import MEDIA_TYPES from providers.provider_reingestion_workflows import ( PROVIDER_REINGESTION_WORKFLOWS as REINGESTION_WORKFLOW_CONFIGS,