Update example

huggingface · Sep 22, 2023 · 83f37c6 · 83f37c6
1 parent d53e9a8
commit 83f37c6
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 9 deletions.
diff --git a/.github/workflows/test_trainium_examples.yml b/.github/workflows/test_trainium_examples.yml
@@ -4,15 +4,17 @@ on:
   workflow_dispatch:
     inputs: 
       coverage: 
-        description: The coverage of the models to test, useful to perform filtering
+        description: Coverage
+        type: choice
         options:
           - all
           - high
           - middle
           - low
         required: true
       model_size: 
-        description: The size of the models to tests
+        description: Size of models
+        type: choice
         options:
           - regular
           - tiny

diff --git a/tests/test_examples.py b/tests/test_examples.py
@@ -83,9 +83,9 @@ class Coverage(str, Enum):
     ALL = "all"
 
 
+USE_VENV = string_to_bool(os.environ.get("USE_VENV", "true"))
 COVERAGE = Coverage(os.environ.get("COVERAGE", "all"))
 RUN_TINY = string_to_bool(os.environ.get("RUN_TINY", "false"))
-USE_VENV = string_to_bool(os.environ.get("USE_VENV", "true"))
 
 MODELS_TO_TEST_MAPPING = {
     "albert": (
@@ -210,7 +210,7 @@ def _get_supported_models_for_script(
     "run_mlm": _get_supported_models_for_script(MODELS_TO_TEST_MAPPING, MODEL_FOR_MASKED_LM_MAPPING),
     "run_swag": _get_supported_models_for_script(MODELS_TO_TEST_MAPPING, MODEL_FOR_MULTIPLE_CHOICE_MAPPING),
     "run_qa": _get_supported_models_for_script(
-        MODELS_TO_TEST_MAPPING, MODEL_FOR_QUESTION_ANSWERING_MAPPING, to_exclude={"bart"}
+        MODELS_TO_TEST_MAPPING, MODEL_FOR_QUESTION_ANSWERING_MAPPING, to_exclude={"gpt2", "gpt_neo", "bart", "t5"}
     ),
     "run_summarization": _get_supported_models_for_script(
         MODELS_TO_TEST_MAPPING, MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING, to_exclude={"marian", "m2m_100"}
@@ -219,10 +219,10 @@ def _get_supported_models_for_script(
         MODELS_TO_TEST_MAPPING, MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
     ),
     "run_glue": _get_supported_models_for_script(
-        MODELS_TO_TEST_MAPPING, MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING, to_exclude={"bart", "gpt2", "gpt_neo"}
+        MODELS_TO_TEST_MAPPING, MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING, to_exclude={"gpt2", "gpt_neo", "bart", "t5"}
     ),
     "run_ner": _get_supported_models_for_script(
-        MODELS_TO_TEST_MAPPING, MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING, to_exclude={"gpt2"}
+        MODELS_TO_TEST_MAPPING, MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING, to_exclude={"gpt2", "gpt_neo"}
     ),
     "run_image_classification": _get_supported_models_for_script(
         MODELS_TO_TEST_MAPPING, MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING
@@ -386,10 +386,12 @@ def test(self):
 
                 if self.CHECK_THAT_LOSS_IS_DECREASING:
                     losses = ExampleTestMeta.parse_loss_from_log(stdout)
-                    allowed_miss_rate = 0.1
+                    allowed_miss_rate = 0.20
                     is_decreasing, moving_average_losses = ExampleTestMeta.check_that_loss_is_decreasing(
-                        losses,
-                        16,
+                        # The loss might stagnate at some point, so we only check that the first 200 losses are
+                        # decreasing on average.
+                        losses[200:],
+                        4,
                         allowed_miss_rate=allowed_miss_rate,
                     )
                     self.assertTrue(