Add isort configuration file (#1054)

* add .isort.cfg

* add comment about known_first_party modules

* Remove redundant config

* Fixup after rebase

---------

Co-authored-by: Madison Swain-Bowden <[email protected]>

.isort.cfg

@@ -0,0 +1,6 @@
+[settings]
+profile=black
+
+sections=FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER
+# These submodules are all added to the PYTHONPATH by Airflow, so we need to add them individually here.
+known_first_party=common,data_refresh,database,maintenance,oauth2,providers,retired

justfile

@@ -73,8 +73,8 @@ deploy:
     @just up
 
 # Run pre-commit on all files
-lint:
-    pre-commit run --all-files
+lint hook="":
+    pre-commit run {{ hook }} --all-files
 
 # Load any dependencies necessary for actions on the stack without running the webserver
 _deps:

openverse_catalog/dags/common/loader/loader.py

@@ -1,4 +1,5 @@
 from airflow.models.abstractoperator import AbstractOperator
+
 from common.loader import paths, s3, sql
 from common.loader.paths import _extract_media_type
 from common.loader.reporting import RecordMetrics

openverse_catalog/dags/common/loader/s3.py

@@ -4,6 +4,7 @@
 
 from airflow.exceptions import AirflowSkipException
 from airflow.providers.amazon.aws.hooks.s3 import S3Hook
+
 from common.loader import paths
 
 

openverse_catalog/dags/common/loader/sql.py

@@ -2,6 +2,8 @@
 from textwrap import dedent
 
 from airflow.models.abstractoperator import AbstractOperator
+from psycopg2.errors import InvalidTextRepresentation
+
 from common.constants import AUDIO, IMAGE, MediaType
 from common.loader import provider_details as prov
 from common.loader.paths import _extract_media_type
@@ -15,7 +17,6 @@
     CURRENT_IMAGE_TSV_COLUMNS,
     required_columns,
 )
-from psycopg2.errors import InvalidTextRepresentation
 
 
 logger = logging.getLogger(__name__)

openverse_catalog/dags/common/popularity/sql.py

@@ -3,6 +3,7 @@
 from textwrap import dedent
 
 from airflow.models.abstractoperator import AbstractOperator
+
 from common.constants import AUDIO, IMAGE
 from common.loader.sql import TABLE_NAMES
 from common.sql import PostgresHook

openverse_catalog/dags/common/requester.py

@@ -2,11 +2,12 @@
 import time
 from collections.abc import Callable
 
-import oauth2
 import requests
 from airflow.exceptions import AirflowException
 from requests.exceptions import JSONDecodeError
 
+import oauth2
+
 
 # pytest_socket will not be available in production, so we must create a shim for
 # that specific exception so this code does not error out.

openverse_catalog/dags/common/sql.py

@@ -8,6 +8,7 @@
 from airflow.providers.postgres.hooks.postgres import (
     PostgresHook as UpstreamPostgresHook,
 )
+
 from common.constants import DAG_DEFAULT_ARGS, POSTGRES_CONN_ID
 
 

openverse_catalog/dags/data_refresh/dag_factory.py

@@ -40,6 +40,7 @@
 from airflow.settings import SASession
 from airflow.utils.session import provide_session
 from airflow.utils.state import State
+
 from common.constants import (
     DAG_DEFAULT_ARGS,
     OPENLEDGER_API_CONN_ID,

openverse_catalog/dags/data_refresh/data_refresh_task_factory.py

@@ -57,10 +57,11 @@
 from airflow.providers.http.operators.http import SimpleHttpOperator
 from airflow.providers.http.sensors.http import HttpSensor
 from airflow.utils.task_group import TaskGroup
+from requests import Response
+
 from common.constants import XCOM_PULL_TEMPLATE
 from common.sensors.single_run_external_dags_sensor import SingleRunExternalDAGsSensor
 from data_refresh.data_refresh_types import DataRefresh
-from requests import Response
 
 
 logger = logging.getLogger(__name__)

openverse_catalog/dags/data_refresh/refresh_popularity_metrics_task_factory.py

@@ -10,6 +10,7 @@
 """
 from airflow.operators.python import PythonOperator
 from airflow.utils.task_group import TaskGroup
+
 from common.constants import POSTGRES_CONN_ID
 from common.popularity import sql
 from data_refresh import reporting

openverse_catalog/dags/data_refresh/refresh_view_data_task_factory.py

@@ -12,6 +12,7 @@
 """
 from airflow.operators.python import PythonOperator
 from airflow.utils.trigger_rule import TriggerRule
+
 from common.constants import POSTGRES_CONN_ID
 from common.popularity import sql
 from data_refresh import reporting

openverse_catalog/dags/database/image_expiration_workflow.py

@@ -9,6 +9,7 @@
 
 from airflow import DAG
 from airflow.operators.python import PythonOperator
+
 from common.constants import DAG_DEFAULT_ARGS, POSTGRES_CONN_ID
 from common.loader import sql
 

openverse_catalog/dags/database/recreate_popularity_calculation_dag_factory.py

@@ -10,6 +10,7 @@
 """
 from airflow import DAG
 from airflow.operators.python import PythonOperator
+
 from common.constants import DAG_DEFAULT_ARGS, POSTGRES_CONN_ID
 from common.popularity import sql
 from data_refresh.data_refresh_types import DATA_REFRESH_CONFIGS, DataRefresh

openverse_catalog/dags/database/report_pending_reported_media.py

@@ -18,6 +18,7 @@
 from airflow.exceptions import AirflowException
 from airflow.models import BaseOperator, TaskInstance
 from airflow.operators.python import PythonOperator
+
 from common import slack
 from common.constants import (
     DAG_DEFAULT_ARGS,

openverse_catalog/dags/database/terminate_long_queries_workflow.py

@@ -11,6 +11,7 @@
 from airflow import DAG
 from airflow.exceptions import AirflowSkipException
 from airflow.operators.python import PythonOperator
+
 from common.constants import (
     DAG_DEFAULT_ARGS,
     OPENLEDGER_API_CONN_ID,

openverse_catalog/dags/maintenance/add_license_url.py

@@ -21,13 +21,14 @@
 from airflow.operators.python import PythonOperator
 from airflow.providers.amazon.aws.hooks.s3 import S3Hook
 from airflow.utils.trigger_rule import TriggerRule
+from psycopg2._json import Json
+
 from common.constants import DAG_DEFAULT_ARGS, POSTGRES_CONN_ID, XCOM_PULL_TEMPLATE
 from common.licenses import get_license_info_from_license_pair
 from common.loader.sql import RETURN_ROW_COUNT
 from common.slack import send_message
 from common.sql import PostgresHook
 from providers.provider_dag_factory import AWS_CONN_ID, OPENVERSE_BUCKET
-from psycopg2._json import Json
 
 
 DAG_ID = "add_license_url"

openverse_catalog/dags/maintenance/airflow_log_cleanup_workflow.py

@@ -25,6 +25,7 @@
 from airflow.configuration import conf
 from airflow.models import DAG
 from airflow.operators.python import PythonOperator
+
 from common import log_cleanup
 from common.constants import DAG_DEFAULT_ARGS
 

openverse_catalog/dags/maintenance/check_silenced_dags.py

@@ -28,6 +28,7 @@
 from airflow.exceptions import AirflowException, AirflowSkipException
 from airflow.models import DAG, Variable
 from airflow.operators.python import PythonOperator
+
 from common.constants import DAG_DEFAULT_ARGS
 from common.github import GitHubAPI
 from common.slack import SilencedSlackNotification, send_alert

openverse_catalog/dags/maintenance/flickr_audit_sub_provider_workflow.py

@@ -13,6 +13,7 @@
 from airflow.exceptions import AirflowSkipException
 from airflow.models import Variable
 from airflow.operators.python import PythonOperator
+
 from common.constants import DAG_DEFAULT_ARGS
 from common.loader import provider_details as prov
 from common.requester import DelayedRequester

openverse_catalog/dags/maintenance/pr_review_reminders/pr_review_reminders_dag.py

@@ -25,6 +25,7 @@
 
 from airflow.models import DAG
 from airflow.operators.python import PythonOperator
+
 from common.constants import DAG_DEFAULT_ARGS
 from maintenance.pr_review_reminders import pr_review_reminders
 

openverse_catalog/dags/oauth2/authorize_dag.py

@@ -1,9 +1,10 @@
 """# OAuth Provider Authorization"""
 from datetime import datetime
 
-import oauth2
 from airflow.models import DAG
 from airflow.operators.python import PythonOperator
+
+import oauth2
 from common.constants import DAG_DEFAULT_ARGS
 
 

openverse_catalog/dags/oauth2/token_refresh_dag.py

@@ -1,9 +1,10 @@
 """# OAuth Provider Token Refresh"""
 from datetime import datetime
 
-import oauth2
 from airflow.models import DAG
 from airflow.operators.python import PythonOperator
+
+import oauth2
 from common.constants import DAG_DEFAULT_ARGS
 
 

openverse_catalog/dags/providers/factory_utils.py

@@ -5,6 +5,7 @@
 
 from airflow.models import DagRun, TaskInstance
 from airflow.utils.dates import cron_presets
+
 from common.constants import MediaType
 from common.storage.media import MediaStore
 from providers.provider_api_scripts.provider_data_ingester import ProviderDataIngester

openverse_catalog/dags/providers/provider_api_scripts/brooklyn_museum.py

@@ -2,6 +2,7 @@
 
 import lxml.html as html
 from airflow.models import Variable
+
 from common import constants
 from common.licenses import get_license_info
 from common.loader import provider_details as prov

openverse_catalog/dags/providers/provider_api_scripts/europeana.py

@@ -13,8 +13,9 @@
 import logging
 from datetime import datetime, timedelta, timezone
 
-import common
 from airflow.models import Variable
+
+import common
 from common.licenses import get_license_info
 from common.loader import provider_details as prov
 from providers.provider_api_scripts.provider_data_ingester import ProviderDataIngester

openverse_catalog/dags/providers/provider_api_scripts/flickr.py

@@ -16,6 +16,7 @@
 
 import lxml.html as html
 from airflow.models import Variable
+
 from common import constants
 from common.licenses import get_license_info
 from common.loader import provider_details as prov

openverse_catalog/dags/providers/provider_api_scripts/freesound.py

@@ -16,13 +16,14 @@
 from datetime import datetime
 
 from airflow.models import Variable
+from requests.exceptions import ConnectionError, SSLError
+from retry import retry
+
 from common import constants
 from common.licenses.licenses import get_license_info
 from common.loader import provider_details as prov
 from common.requester import RetriesExceeded
 from providers.provider_api_scripts.provider_data_ingester import ProviderDataIngester
-from requests.exceptions import ConnectionError, SSLError
-from retry import retry
 
 
 logger = logging.getLogger(__name__)

openverse_catalog/dags/providers/provider_api_scripts/inaturalist.py

@@ -32,6 +32,7 @@
 from airflow.providers.amazon.aws.hooks.s3 import S3Hook
 from airflow.utils.task_group import TaskGroup
 from airflow.utils.trigger_rule import TriggerRule
+
 from common.constants import IMAGE, POSTGRES_CONN_ID, XCOM_PULL_TEMPLATE
 from common.loader import provider_details, reporting, sql
 from common.sql import PGExecuteQueryOperator, PostgresHook

openverse_catalog/dags/providers/provider_api_scripts/jamendo.py

@@ -18,8 +18,9 @@
 import logging
 from urllib.parse import parse_qs, urlencode, urlsplit
 
-import common
 from airflow.models import Variable
+
+import common
 from common import constants
 from common.licenses import get_license_info
 from common.loader import provider_details as prov

openverse_catalog/dags/providers/provider_api_scripts/metropolitan_museum.py

@@ -28,9 +28,10 @@
 import argparse
 import logging
 
+from provider_data_ingester import ProviderDataIngester
+
 from common.licenses import get_license_info
 from common.loader import provider_details as prov
-from provider_data_ingester import ProviderDataIngester
 
 
 logging.basicConfig(

openverse_catalog/dags/providers/provider_api_scripts/nypl.py

@@ -3,6 +3,7 @@
 from urllib.parse import parse_qs, urlparse
 
 from airflow.models import Variable
+
 from common import constants
 from common.licenses import get_license_info
 from common.loader import provider_details as prov

openverse_catalog/dags/providers/provider_api_scripts/provider_data_ingester.py

@@ -7,6 +7,7 @@
 
 from airflow.exceptions import AirflowException
 from airflow.models import Variable
+
 from common.requester import DelayedRequester
 from common.storage.media import MediaStore
 from common.storage.util import get_media_store_class

openverse_catalog/dags/providers/provider_api_scripts/rawpixel.py

@@ -20,6 +20,7 @@
 from urllib.parse import urlencode
 
 from airflow.models import Variable
+
 from common import constants
 from common.licenses import NO_LICENSE_FOUND, get_license_info
 from common.loader import provider_details as prov

openverse_catalog/dags/providers/provider_api_scripts/smithsonian.py

@@ -12,11 +12,12 @@
 
 from airflow.exceptions import AirflowException
 from airflow.models import Variable
+from retry import retry
+
 from common import constants
 from common.licenses import get_license_info
 from common.loader import provider_details as prov
 from providers.provider_api_scripts.provider_data_ingester import ProviderDataIngester
-from retry import retry
 
 
 logger = logging.getLogger(__name__)

openverse_catalog/dags/providers/provider_api_scripts/wikimedia_commons.py

@@ -113,6 +113,7 @@
 from types import MappingProxyType
 
 import lxml.html as html
+
 from common.constants import AUDIO, IMAGE
 from common.licenses import get_license_info
 from common.loader import provider_details as prov

openverse_catalog/dags/providers/provider_api_scripts/wordpress.py

@@ -12,6 +12,7 @@
 import logging
 
 import lxml.html as html
+
 from common import constants
 from common.licenses import get_license_info
 from common.loader import provider_details as prov

openverse_catalog/dags/providers/provider_dag_factory.py

@@ -74,6 +74,7 @@
 from airflow.operators.python import PythonOperator
 from airflow.utils.task_group import TaskGroup
 from airflow.utils.trigger_rule import TriggerRule
+
 from common.constants import DAG_DEFAULT_ARGS, XCOM_PULL_TEMPLATE
 from common.loader import loader, reporting, s3, sql
 from providers.factory_utils import date_partition_for_prefix, pull_media_wrapper

openverse_catalog/dags/providers/provider_workflows.py

@@ -6,6 +6,8 @@
 from datetime import datetime, timedelta
 
 from airflow.models import Variable
+from typing_extensions import NotRequired, TypedDict
+
 from providers.provider_api_scripts.brooklyn_museum import BrooklynMuseumDataIngester
 from providers.provider_api_scripts.cleveland_museum import ClevelandDataIngester
 from providers.provider_api_scripts.europeana import EuropeanaDataIngester
@@ -29,7 +31,6 @@
     WikimediaCommonsDataIngester,
 )
 from providers.provider_api_scripts.wordpress import WordPressDataIngester
-from typing_extensions import NotRequired, TypedDict
 
 
 logger = logging.getLogger(__name__)

openverse_catalog/utilities/dag_doc_gen/dag_doc_generation.py

@@ -19,6 +19,7 @@
 from typing import NamedTuple
 
 from airflow.models import DAG, DagBag
+
 from providers.provider_workflows import PROVIDER_WORKFLOWS, ProviderWorkflow