Enhance pre-commit hooks with flake8 linting (kubeflow#2195)

* Add pre-commit linter Signed-off-by: Ignas Baranauskas <[email protected]> * Fix flake8 linting issues Signed-off-by: Ignas Baranauskas <[email protected]> --------- Signed-off-by: Ignas Baranauskas <[email protected]>
andreyvelich · Aug 12, 2024 · ff19a10 · ff19a10
1 parent 2561b52
commit ff19a10
Show file tree

Hide file tree

Showing 7 changed files with 36 additions and 23 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -16,17 +16,31 @@ repos:
   - repo: https://github.com/psf/black
     rev: 24.2.0
     hooks:
-    - id: black
-      files: (sdk|examples)/.*
-      exclude: |
-        (?x)^(
-          /*kubeflow_org_v1*|
-          __init__.py|
-          api_client.py|
-          configuration.py|
-          exceptions.py|
-          rest.py
-        )$
+      - id: black
+        files: (sdk|examples)/.*
+        exclude: |
+          (?x)^(
+            /*kubeflow_org_v1*|
+            __init__.py|
+            api_client.py|
+            configuration.py|
+            exceptions.py|
+            rest.py
+          )$
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.1.1
+    hooks:
+      - id: flake8
+        files: ^(sdk|examples)/.*$
+        exclude: |
+          (?x)^(
+            .*kubeflow_org_v1.*|
+            __init__\.py|
+            api_client\.py|
+            configuration\.py|
+            exceptions\.py|
+            rest\.py
+          )$
 exclude: |
   (?x)^(
     pkg/apis/kubeflow.org/v1/openapi_generated.go|

diff --git a/examples/pytorch/elastic/echo/echo.py b/examples/pytorch/elastic/echo/echo.py
@@ -3,12 +3,10 @@
 import os
 import pprint
 import sys
-import time
 
 import torch.distributed as dist
 
 if __name__ == "__main__":
-
     env_dict = {
         k: os.environ[k]
         for k in (

diff --git a/examples/pytorch/elastic/imagenet/imagenet.py b/examples/pytorch/elastic/imagenet/imagenet.py
@@ -348,7 +348,7 @@ def load_checkpoint(
 
         # max_epoch == -1 means no one has checkpointed return base state
         if max_epoch == -1:
-            print(f"=> no workers have checkpoints, starting from epoch 0")
+            print("=> no workers have checkpoints, starting from epoch 0")
             return state
 
         # broadcast the state from max_rank (which has the most up-to-date state)
@@ -370,7 +370,7 @@ def load_checkpoint(
             blob = torch.as_tensor(raw_blob, dtype=torch.uint8)
 
         dist.broadcast(blob, src=max_rank, group=pg)
-        print(f"=> done broadcasting checkpoint")
+        print("=> done broadcasting checkpoint")
 
         if rank != max_rank:
             with io.BytesIO(blob.numpy()) as f:
@@ -380,7 +380,7 @@ def load_checkpoint(
         # wait till everyone has loaded the checkpoint
         dist.barrier(group=pg)
 
-    print(f"=> done restoring from previous checkpoint")
+    print("=> done restoring from previous checkpoint")
     return state
 
 

diff --git a/sdk/python/kubeflow/storage_initializer/s3.py b/sdk/python/kubeflow/storage_initializer/s3.py
@@ -1,5 +1,4 @@
 from dataclasses import dataclass
-from dataclasses import field
 import json
 import os
 from urllib.parse import urlparse
@@ -71,4 +70,4 @@ def download_dataset(self):
             # Download the file
             file_path = os.path.sep.join(path_components[1:])
             bucket.download_file(obj_key, os.path.join(VOLUME_PATH_DATASET, file_path))
-        print(f"Files downloaded")
+        print("Files downloaded")
diff --git a/sdk/python/kubeflow/training/api/training_client.py b/sdk/python/kubeflow/training/api/training_client.py
@@ -168,8 +168,8 @@ def train(
                 and dataset. You can configure PVC size and storage class name in this argument.
         """
         try:
-            import peft
-            import transformers
+            import peft  # noqa: F401
+            import transformers  # noqa: F401
         except ImportError:
             raise ImportError(
                 "Train API dependencies not installed. "

diff --git a/sdk/python/kubeflow/training/api/training_client_test.py b/sdk/python/kubeflow/training/api/training_client_test.py
@@ -225,8 +225,10 @@ def __init__(self, kind) -> None:
             "replica_type": constants.REPLICA_TYPE_MASTER.lower(),
             "replica_index": 0,
         },
-        f"{constants.JOB_NAME_LABEL}={TEST_NAME},{constants.JOB_ROLE_LABEL}={constants.JOB_ROLE_MASTER}"
-        f",{constants.REPLICA_TYPE_LABEL}={constants.REPLICA_TYPE_MASTER.lower()},{constants.REPLICA_INDEX_LABEL}=0",
+        f"{constants.JOB_NAME_LABEL}={TEST_NAME},"
+        f"{constants.JOB_ROLE_LABEL}={constants.JOB_ROLE_MASTER},"
+        f"{constants.REPLICA_TYPE_LABEL}={constants.REPLICA_TYPE_MASTER.lower()},"
+        f"{constants.REPLICA_INDEX_LABEL}=0",
         LIST_RESPONSE,
     ),
     (

diff --git a/sdk/python/kubeflow/training/constants/constants.py b/sdk/python/kubeflow/training/constants/constants.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Dict, Union
+from typing import Union
 
 from kubeflow.storage_initializer.constants import INIT_CONTAINER_MOUNT_PATH
 from kubeflow.training import models