[Integration-Testing] Fix for post-merge tests (#887)

Author: KSGulin

.github/workflows/Integrations-post-merge-check.yaml

@@ -1,4 +1,4 @@
-name: Integrations Testing
+name: Integrations Testing Post-Merge
 on: 
   push:
     branches:
@@ -33,6 +33,10 @@ jobs:
           repository: "neuralmagic/sparsezoo"
           path: "sparsezoo"
           ref: ${{needs.test-setup.outputs.branch}}
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.8
       - name: "⚙️ Install sparsezoo dependencies"
         run: pip3 install -U pip && pip3 install setuptools sparsezoo/
       - name: "Clean sparsezoo directory"
@@ -46,4 +50,4 @@ jobs:
       - name: "Install yolov5 integration"
         run: sparseml.yolov5.train --help
       - name: "🔬 Running integrations tests (cadence: commit}})"
-        run: make testinteg TARGETS=transformers,yolov5,image_classification
+        run: make testinteg TARGETS=yolov5,transformers,image_classification

.github/workflows/integrations-check.yaml

@@ -54,14 +54,12 @@ jobs:
           repository: "neuralmagic/sparsezoo"
           path: "sparsezoo"
           ref: ${{needs.test-setup.outputs.branch}}
-      - name: Set up Python 3.8}}
+      - name: Set up Python 3.8
         uses: actions/setup-python@v3
         with:
           python-version: 3.8
       - name: "⚙️ Install sparsezoo dependencies"
-        run: |
-          python --version
-          pip3 install -U pip && pip3 install setuptools sparsezoo/
+        run: pip3 install -U pip && pip3 install setuptools sparsezoo/
       - name: "Clean sparsezoo directory"
         run: rm -r sparsezoo/
       - name: "Upgrade protobuf version"

tests/integrations/base_tester.py

@@ -38,7 +38,6 @@
 from pydantic import BaseModel
 
 from tests.integrations.config import Config
-from tests.integrations.helpers import get_configs_with_cadence
 
 
 __all__ = [
@@ -258,7 +257,9 @@ class BaseIntegrationTester:
     stage is `train`, `export`, or `deploy` and name is a unique name to describe the
     test. This naming convention is used and enforced within the decorator
     `@skip_inactive_stage`
-    """
+
+    Adding the fixture below to each test class will parameterize the test over
+    the set of test configs that match the cadence setting.
 
     @pytest.fixture(
         params=get_configs_with_cadence(
@@ -267,6 +268,8 @@ class BaseIntegrationTester:
         ),
         scope="class",
     )
+    """
+
     def integration_manager(request):
         """
         Fixture with a lifecycle of:

tests/integrations/config.py

@@ -12,11 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+from pathlib import Path
 from typing import Dict, List, Optional
 
 from pydantic import BaseModel
 
 
+# path to sparseml root directory
+ROOT = Path(Path(__file__).resolve().parents[2])
+
 __all__ = ["Config"]
 
 
@@ -49,6 +54,15 @@ def __init__(
         self.test_args = config.pop("test_args", {})
         # whether to replace '_' with '-' for run keywords
         self.dashed_keywords = False
+
+        # Construct explicit path from relative path, using sparseml base directory
+        # as root
+        recipe = getattr(self.run_args, "recipe", None)
+        if recipe and not recipe.startswith("zoo:"):
+            self.run_args.recipe = str(
+                Path(os.path.join(ROOT, self.run_args.recipe)).resolve()
+            )
+
         self._validate_config()
 
     def create_command_script(self):

tests/integrations/helpers.py

@@ -54,6 +54,9 @@ def get_configs_with_cadence(cadence: str, dir_path: str = "."):
                     if line.split(":")[1].strip().strip('"').lower() == cadence:
                         matching_files.append(file)
                         break
+
+    print(f"\nFor {cadence} found matching files: {matching_files}")
+
     return matching_files
 
 

tests/integrations/image_classification/args.py

@@ -145,6 +145,9 @@ def __post_init__(self):
 
 
 class ImageClassificationExportArgs(_ImageClassificationBaseArgs):
+    model_tag: Optional[str] = Field(
+        Default=None, description="required - tag for model under save_dir"
+    )
     onnx_opset: int = Field(
         default=11, description="The onnx opset to use for exporting the model"
     )

tests/integrations/image_classification/configs/commit/test_ended_to_end_pruned_quantized_mobilenet_imagenette.yaml

@@ -0,0 +1,17 @@
+cadence: "commit"
+abridged: True
+train:
+  command_args:
+    dataset: imagenette
+    dataset_path: imagenette
+    recipe_path: tests/integrations/image_classification/configs/commit/recipe_short_prune_quant.md
+    arch_key: mobilenet
+    train_batch_size: 4
+    test_batch_size: 4
+    model_tag: mobilenet-imagenette-pruned
+    save_dir: image_classification-end_to_end-test
+export:
+  command_args:
+    num_classes: 10
+deploy:
+  command_args: null

tests/integrations/image_classification/configs/commit/test_ended_to_end_pruned_quantized_resnet50_imagenette.yaml

@@ -84,7 +84,9 @@ def capture_pre_run_state(self):
                 export_args.save_dir = self.save_dir.name
             else:
                 export_args.checkpoint_path = self.expected_checkpoint_path
-                export_args.save_dir = train_args.save_dir
+                export_args.save_dir = train_args.save_dir + "_exported"
+                export_args.model_tag = train_args.model_tag
+                export_args.arch_key = train_args.arch_key
 
         if "deploy" in self.configs:
             deploy_args = self.configs["deploy"].run_args
@@ -96,8 +98,8 @@ def capture_pre_run_state(self):
 
     def add_abridged_configs(self):
         if "train" in self.command_types:
-            self.configs["train"].max_train_steps = 10
-            self.configs["train"].max_eval_steps = 10
+            self.configs["train"].run_args.max_train_steps = 2
+            self.configs["train"].run_args.max_eval_steps = 2
 
     def teardown(self):
         """
@@ -181,7 +183,7 @@ def test_train_metrics(self, integration_manager):
     def test_export_onnx_graph(self, integration_manager):
         export_args = integration_manager.configs["export"]
         expected_onnx_path = os.path.join(
-            integration_manager.save_dir.name,
+            export_args.run_args.save_dir,
             export_args.run_args.model_tag,
             "model.onnx",
         )
@@ -201,7 +203,7 @@ def test_export_target_model(self, integration_manager):
             zoo_model = Zoo.load_model_from_stub(target_model_path)
             target_model_path = zoo_model.onnx_file.downloaded_path()
         export_model_path = os.path.join(
-            integration_manager.save_dir.name,
+            export_args.run_args.save_dir,
             export_args.run_args.model_tag,
             "model.onnx",
         )

tests/integrations/image_classification/test_image_classification.py

@@ -12,6 +12,8 @@ train:
    per_device_train_batch_size: 16
    max_seq_length: 384
    doc_stride: 128
+   per_device_train_batch_size: 4
+   per_device_eval_batch_size: 4
 export:
   command_args:
     task: qa

tests/integrations/transformers/configs/commit/test_end_to_end_quantized_question_answering.yaml

@@ -1,5 +1,4 @@
 cadence: "pre-commit"
-abridged: True
 train:
   task: Question-Answering
   command_args:

tests/integrations/transformers/configs/pre-commit/test_cli_question_answering.yaml

@@ -1,5 +1,4 @@
 cadence: "pre-commit"
-abridged: True
 train:
   task: Text-Classification
   command_args:

tests/integrations/transformers/configs/pre-commit/test_cli_text_classification.yaml

@@ -119,8 +119,8 @@ def save_stage_information(self, command_type):
 
     def add_abridged_configs(self):
         if "train" in self.command_types:
-            self.configs["train"].max_train_samples = 10
-            self.configs["train"].max_eval_samples = 10
+            self.configs["train"].run_args.max_train_samples = 2
+            self.configs["train"].run_args.max_eval_samples = 2
 
     def get_root_commands(self, raw_configs):
         self.task = (

tests/integrations/transformers/test_transformers.py

@@ -91,7 +91,7 @@ class Yolov5TrainArgs(BaseModel):
     cost_lr: bool = Field(default=False, description="cosine LR scheduler")
     label_smoothing: float = Field(default=0.0, description="Label smoothing epsilon")
     patience: int = Field(
-        default=100, description="EarlyStopping patience (epochs without improvement)"
+        default=0, description="EarlyStopping patience (epochs without improvement)"
     )
     freeze: str = Field(
         default="0", description="Freeze layers: backbone=10, first3=0 1 2"

tests/integrations/yolov5/args.py

@@ -4,10 +4,7 @@ train:
   command_args:
     weights: "zoo:cv/detection/yolov5-s/pytorch/ultralytics/coco/base-none"
     recipe: "tests/integrations/yolov5/configs/commit/recipe_short_prune_quant.md"
-  test_args: 
-    target_name: "map0.5"
-    target_mean: 52.5 
-    target_std: 3
+    batch_size: 4
 export:
   command_args:
     dynamic: True

tests/integrations/yolov5/configs/commit/test_ended_to_end_pruned_quantized.yaml

@@ -1,5 +1,4 @@
 cadence: "pre-commit"
-abridged: True
 train:
   command_args:
     weights: "zoo:cv/detection/yolov5-s/pytorch/ultralytics/coco/base-none"

tests/integrations/yolov5/configs/pre-commit/test_cli.yaml

@@ -100,9 +100,8 @@ def capture_pre_run_state(self):
 
     def add_abridged_configs(self):
         if "train" in self.command_types:
-            self.configs["train"].max_train_steps = 10
-            self.configs["train"].max_eval_steps = 10
-            self.configs["train"].data = "coco128.yaml"
+            self.configs["train"].run_args.max_train_steps = 2
+            self.configs["train"].run_args.max_eval_steps = 2
 
     def teardown(self):
         if "train" in self.command_types: