Run CI on Linux platform (#40)

Co-authored-by: Guang Yang <[email protected]>

Author: guangy10

.github/workflows/test_models.yml

@@ -11,39 +11,32 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build:
+  discover-tests:
+    runs-on: ubuntu-22.04
+    outputs:
+      model_names: ${{ steps.set-matrix.outputs.model_names }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Find model tests
+        id: set-matrix
+        run: |
+          # Find all test files and extract model names correctly
+          MODEL_NAMES=$(find tests/models -name "test_modeling_*.py" -type f | sed 's|tests/models/test_modeling_||' | sed 's|\.py$||' | paste -sd "," -)
+          echo "model_names=[\"${MODEL_NAMES//,/\",\"}\"]" >> $GITHUB_OUTPUT
+
+          # Display all discovered models
+          echo "Discovered models:"
+          echo "$MODEL_NAMES" | tr ',' '\n' | sort | awk '{print "- " $0}'
+
+  run-tests:
+    needs: discover-tests
     strategy:
       fail-fast: false
       matrix:
-        test-modeling:
-          # The model name MUST match with the tests/models/test_modeling_<name>.py
-          - common
-          - albert
-          - bert
-          - cvt
-          - deit
-          - distilbert
-          - dit
-          - efficientnet
-          - focalnet
-          - gemma
-          - gemma2
-          - llama
-          - mobilevit
-          - mobilevit2
-          - olmo
-          - phi4
-          - pvt
-          - qwen2
-          - roberta
-          - smollm
-          - swin
-          - t5
-          - vit
-          - whisper
+        test-modeling: ${{ fromJson(needs.discover-tests.outputs.model_names) }}
         executorch-version: ['0.4.0', '0.6.0rc', 'nightly']
-        python-version: ['3.10', '3.11', '3.12']
-        os: [macos-15]
+        python-version: ['3.11']
+        os: [macos-15, ubuntu-22.04]
 
     # Custom job name, now shortened and cleaner
     name: ${{ matrix.test-modeling }} (et=${{ matrix.executorch-version }}, py=${{ matrix.python-version }}, ${{ matrix.os }})
@@ -73,6 +66,5 @@ jobs:
           pip install '.[tests]'
           pip list
       - name: Run tests
-        working-directory: tests
         run: |
-          RUN_SLOW=1 pytest models/test_modeling_${{ matrix.test-modeling }}.py -s -vvvv --durations=0 --log-cli-level=INFO
+          RUN_SLOW=1 pytest tests/models/test_modeling_${{ matrix.test-modeling }}.py -s -vvvv --durations=0 --log-cli-level=INFO

tests/models/test_modeling_gemma.py

@@ -17,6 +17,7 @@
 import logging
 import os
 import subprocess
+import sys
 import tempfile
 import unittest
 
@@ -32,6 +33,10 @@
 from ..utils import check_causal_lm_output_quality
 
 
+is_linux_ci = sys.platform.startswith("linux") and os.environ.get("GITHUB_ACTIONS") == "true"
+
+
+@pytest.mark.skipif(is_linux_ci, reason="OOM on linux runner")
 class ExecuTorchModelIntegrationTest(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)

tests/models/test_modeling_gemma2.py

@@ -17,6 +17,7 @@
 import logging
 import os
 import subprocess
+import sys
 import tempfile
 import unittest
 
@@ -32,6 +33,10 @@
 from ..utils import check_causal_lm_output_quality
 
 
+is_linux_ci = sys.platform.startswith("linux") and os.environ.get("GITHUB_ACTIONS") == "true"
+
+
+@pytest.mark.skipif(is_linux_ci, reason="OOM on linux runner")
 class ExecuTorchModelIntegrationTest(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)

tests/models/test_modeling_llama.py

@@ -17,6 +17,7 @@
 import logging
 import os
 import subprocess
+import sys
 import tempfile
 import unittest
 
@@ -32,6 +33,9 @@
 from ..utils import check_causal_lm_output_quality
 
 
+is_linux_ci = sys.platform.startswith("linux") and os.environ.get("GITHUB_ACTIONS") == "true"
+
+
 class ExecuTorchModelIntegrationTest(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -43,15 +47,23 @@ def test_llama3_2_1b_export_to_executorch(self):
         task = "text-generation"
         recipe = "xnnpack"
         with tempfile.TemporaryDirectory() as tempdir:
+            out_dir = f"{tempdir}/executorch"
             subprocess.run(
-                f"optimum-cli export executorch --model {model_id} --task {task} --recipe {recipe} --output_dir {tempdir}/executorch",
+                f"optimum-cli export executorch --model {model_id} --task {task} --recipe {recipe} --output_dir {out_dir}",
                 shell=True,
                 check=True,
             )
-            self.assertTrue(os.path.exists(f"{tempdir}/executorch/model.pte"))
+            pte_full_path = f"{out_dir}/model.pte"
+            self.assertTrue(os.path.exists(pte_full_path))
+
+            # Explicitly delete the PTE file to free up disk space
+            if os.path.exists(pte_full_path):
+                os.remove(pte_full_path)
+            gc.collect()
 
     @slow
     @pytest.mark.run_slow
+    @pytest.mark.skipif(is_linux_ci, reason="OOM on linux runner")
     def test_llama3_2_1b_text_generation(self):
         # TODO: Switch to use meta-llama/Llama-3.2-1B once https://github.com/huggingface/optimum/issues/2127 is fixed
         # model_id = "lama/Llama-3.2-1B"