Transformers version bump (#70)

guangy10 · Guang Yang · web-flow · commit 6dc9aa2d507b · 2025-06-04T10:36:01.000-07:00
Co-authored-by: Guang Yang &lt;guangyang@fb.com&gt;
diff --git a/.github/workflows/test_models.yml b/.github/workflows/test_models.yml
@@ -51,25 +51,19 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies for ExecuTorch
         run: |
+          pip install '.[tests]'
           if [ "${{ matrix.executorch-version }}" == "nightly" ]; then
-            export NIGHTLY_VERSION=dev20250507
+            export NIGHTLY_VERSION=dev20250523
             pip install executorch==0.7.0.${NIGHTLY_VERSION} \
               torch==2.8.0.${NIGHTLY_VERSION} \
               torchvision==0.22.0.${NIGHTLY_VERSION} \
               torchaudio==2.6.0.${NIGHTLY_VERSION} \
-              torchao==0.12.0.${NIGHTLY_VERSION} \
+              torchao==0.12.0.dev20250528 \
               --extra-index-url "https://download.pytorch.org/whl/nightly/cpu"
+            pip install transformers==4.52.4
           else
             pip install executorch==${{ matrix.executorch-version }}
           fi
-          pip install '.[tests]'
-          if [ "${{ matrix.test-modeling }}" == "gemma3" ]; then
-            git clone https://github.com/huggingface/transformers.git
-            pushd transformers
-            git checkout a57274466f7f72efaa2662d1738cdaf28ae8071f
-            pip install -e .
-            popd
-          fi
           pip list
       - name: Run tests
         run: |
diff --git a/setup.py b/setup.py
@@ -14,7 +14,7 @@
 INSTALL_REQUIRE = [
     "optimum~=1.24",
     "executorch>=0.6.0",
-    "transformers==4.51.0",
+    "transformers==4.51.3",
 ]
 
 TESTS_REQUIRE = [
diff --git a/tests/models/test_modeling_gemma.py b/tests/models/test_modeling_gemma.py
@@ -17,24 +17,23 @@
 import logging
 import os
 import subprocess
-import sys
 import tempfile
 import unittest
 
 import pytest
+import torchao
 from executorch.extension.pybindings.portable_lib import ExecuTorchModule
+from packaging.version import parse
 from transformers import AutoTokenizer
 from transformers.testing_utils import slow
 
 from optimum.executorch import ExecuTorchModelForCausalLM
 
-from ..utils import check_causal_lm_output_quality
 
-
-is_linux_ci = sys.platform.startswith("linux") and os.environ.get("GITHUB_ACTIONS") == "true"
-
-
-@pytest.mark.skipif(is_linux_ci, reason="OOM on linux runner")
+@pytest.mark.skipif(
+    parse(torchao.__version__) < parse("0.11.0.dev0"),
+    reason="Only available on torchao >= 0.11.0.dev0",
+)
 class ExecuTorchModelIntegrationTest(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -48,7 +47,14 @@ def test_gemma_export_to_executorch(self):
         with tempfile.TemporaryDirectory() as tempdir:
             out_dir = f"{tempdir}/executorch"
             subprocess.run(
-                f"optimum-cli export executorch --model {model_id} --task {task} --recipe {recipe} --output_dir {out_dir}",
+                f"optimum-cli export executorch \
+                    --model {model_id} \
+                    --task {task} \
+                    --recipe {recipe} \
+                    --output_dir {tempdir}/executorch \
+                    --use_custom_sdpa \
+                    --qlinear \
+                    --qembedding",
                 shell=True,
                 check=True,
             )
@@ -62,14 +68,17 @@ def test_gemma_export_to_executorch(self):
 
     @slow
     @pytest.mark.run_slow
-    def test_gemma_text_generation_float16(self):
+    def test_gemma_text_generation_with_custom_sdpa_8da4w_8we(self):
         # TODO: Switch to use google/gemma-2b once https://github.com/huggingface/optimum/issues/2127 is fixed
         # model_id = "google/gemma-2b"
         model_id = "weqweasdas/RM-Gemma-2B"
+        # ExecuTorch model + custom sdpa + 8da4w linear quantization + int8 embedding quantization
+        kwargs = {"qlinear": True, "qembedding": True}
         model = ExecuTorchModelForCausalLM.from_pretrained(
             model_id,
             recipe="xnnpack",
-            **{"dtype": "float16"},
+            attn_implementation="custom_sdpa",
+            **kwargs,
         )
         self.assertIsInstance(model, ExecuTorchModelForCausalLM)
         self.assertIsInstance(model.model, ExecuTorchModule)
@@ -81,11 +90,3 @@ def test_gemma_text_generation_float16(self):
             max_seq_len=21,
         )
         logging.info(f"\nGenerated text:\n\t{generated_text}")
-        generated_tokens = tokenizer(generated_text, return_tensors="pt").input_ids
-
-        # Free memory before loading eager for quality check
-        del model
-        del tokenizer
-        gc.collect()
-
-        self.assertTrue(check_causal_lm_output_quality(model_id, generated_tokens))
diff --git a/tests/models/test_modeling_gemma2.py b/tests/models/test_modeling_gemma2.py
@@ -17,12 +17,13 @@
 import logging
 import os
 import subprocess
-import sys
 import tempfile
 import unittest
 
 import pytest
+import torchao
 from executorch.extension.pybindings.portable_lib import ExecuTorchModule
+from packaging.version import parse
 from transformers import AutoTokenizer
 from transformers.testing_utils import slow
 
@@ -31,10 +32,10 @@
 from ..utils import check_causal_lm_output_quality
 
 
-is_linux_ci = sys.platform.startswith("linux") and os.environ.get("GITHUB_ACTIONS") == "true"
-
-
-@pytest.mark.skipif(is_linux_ci, reason="OOM on linux runner")
+@pytest.mark.skipif(
+    parse(torchao.__version__) < parse("0.11.0.dev0"),
+    reason="Only available on torchao >= 0.11.0.dev0",
+)
 class ExecuTorchModelIntegrationTest(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -48,7 +49,14 @@ def test_gemma2_export_to_executorch(self):
         with tempfile.TemporaryDirectory() as tempdir:
             out_dir = f"{tempdir}/executorch"
             subprocess.run(
-                f"optimum-cli export executorch --model {model_id} --task {task} --recipe {recipe} --output_dir {out_dir}",
+                f"optimum-cli export executorch \
+                    --model {model_id} \
+                    --task {task} \
+                    --recipe {recipe} \
+                    --output_dir {tempdir}/executorch \
+                    --use_custom_sdpa \
+                    --qlinear \
+                    --qembedding",
                 shell=True,
                 check=True,
             )
@@ -62,14 +70,17 @@ def test_gemma2_export_to_executorch(self):
 
     @slow
     @pytest.mark.run_slow
-    def test_gemma2_text_generation_float16(self):
+    def test_gemma2_text_generation_with_custom_sdpa_8da4w_8we(self):
         # TODO: Switch to use google/gemma-2-2b once https://github.com/huggingface/optimum/issues/2127 is fixed
         # model_id = "google/gemma-2-2b"
         model_id = "unsloth/gemma-2-2b-it"
+        # ExecuTorch model + custom sdpa + 8da4w linear quantization + int8 embedding quantization
+        kwargs = {"qlinear": True, "qembedding": True}
         model = ExecuTorchModelForCausalLM.from_pretrained(
             model_id,
             recipe="xnnpack",
-            **{"dtype": "float16"},
+            attn_implementation="custom_sdpa",
+            **kwargs,
         )
         self.assertIsInstance(model, ExecuTorchModelForCausalLM)
         self.assertIsInstance(model.model, ExecuTorchModule)
diff --git a/tests/models/test_modeling_gemma3.py b/tests/models/test_modeling_gemma3.py
@@ -59,7 +59,14 @@ def test_gemma3_export_to_executorch(self):
         with tempfile.TemporaryDirectory() as tempdir:
             out_dir = f"{tempdir}/executorch"
             subprocess.run(
-                f"optimum-cli export executorch --model {model_id} --task {task} --recipe {recipe} --output_dir {out_dir}",
+                f"optimum-cli export executorch \
+                    --model {model_id} \
+                    --task {task} \
+                    --recipe {recipe} \
+                    --output_dir {tempdir}/executorch \
+                    --use_custom_sdpa \
+                    --qlinear \
+                    --qembedding",
                 shell=True,
                 check=True,
             )
@@ -176,14 +183,14 @@ def test_gemma3_text_generation_with_custom_sdpa_float16(self):
         parse(torchao.__version__) < parse("0.11.0.dev0"),
         reason="Only available on torchao >= 0.11.0.dev0",
     )
-    def test_gemma3_text_generation_with_custom_sdpa_8da4w(self):
+    def test_gemma3_text_generation_with_custom_sdpa_8da4w_8we(self):
         # TODO: Until https://github.com/huggingface/optimum/issues/2127 is fixed, have to use non-gated model on CI
         # model_id = "google/gemma-3-1b-it"
         model_id = "unsloth/gemma-3-1b-it"
         prompt = "Write a poem about a machine learning."
 
-        # ExecuTorch model + custom sdpa + 8da4w linear quantization
-        kwargs = {"qlinear": True}
+        # ExecuTorch model + custom sdpa + 8da4w linear quantization + int8 embedding quantization
+        kwargs = {"qlinear": True, "qembedding": True}
         model = ExecuTorchModelForCausalLM.from_pretrained(
             model_id,
             recipe="xnnpack",
diff --git a/tests/models/test_modeling_llama.py b/tests/models/test_modeling_llama.py
@@ -22,7 +22,9 @@
 import unittest
 
 import pytest
+import torchao
 from executorch.extension.pybindings.portable_lib import ExecuTorchModule
+from packaging.version import parse
 from transformers import AutoTokenizer
 from transformers.testing_utils import slow
 
@@ -34,6 +36,10 @@
 is_linux_ci = sys.platform.startswith("linux") and os.environ.get("GITHUB_ACTIONS") == "true"
 
 
+@pytest.mark.skipif(
+    parse(torchao.__version__) < parse("0.11.0.dev0"),
+    reason="Only available on torchao >= 0.11.0.dev0",
+)
 class ExecuTorchModelIntegrationTest(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -47,7 +53,14 @@ def test_llama3_2_1b_export_to_executorch(self):
         with tempfile.TemporaryDirectory() as tempdir:
             out_dir = f"{tempdir}/executorch"
             subprocess.run(
-                f"optimum-cli export executorch --model {model_id} --task {task} --recipe {recipe} --output_dir {out_dir}",
+                f"optimum-cli export executorch \
+                    --model {model_id} \
+                    --task {task} \
+                    --recipe {recipe} \
+                    --output_dir {tempdir}/executorch \
+                    --use_custom_sdpa \
+                    --qlinear \
+                    --qembedding",
                 shell=True,
                 check=True,
             )
@@ -88,13 +101,15 @@ def test_llama3_2_1b_text_generation(self):
 
     @slow
     @pytest.mark.run_slow
-    def test_llama_text_generation_with_custom_sdpa(self):
-        # ExecuTorch model + custom sdpa
+    def test_llama_text_generation_with_custom_sdpa_8da4w_8we(self):
+        # ExecuTorch model + custom sdpa + 8da4w linear quantization + int8 embedding quantization
         model_id = "NousResearch/Llama-3.2-1B"
+        kwargs = {"qlinear": True, "qembedding": True}
         model = ExecuTorchModelForCausalLM.from_pretrained(
             model_id,
             recipe="xnnpack",
             attn_implementation="custom_sdpa",
+            **kwargs,
         )
         self.assertIsInstance(model, ExecuTorchModelForCausalLM)
         self.assertIsInstance(model.model, ExecuTorchModule)
diff --git a/tests/models/test_modeling_olmo.py b/tests/models/test_modeling_olmo.py
@@ -17,11 +17,14 @@
 import logging
 import os
 import subprocess
+import sys
 import tempfile
 import unittest
 
 import pytest
+import torchao
 from executorch.extension.pybindings.portable_lib import ExecuTorchModule
+from packaging.version import parse
 from transformers import AutoTokenizer
 from transformers.testing_utils import slow
 
@@ -30,6 +33,13 @@
 from ..utils import check_causal_lm_output_quality
 
 
+is_linux_ci = sys.platform.startswith("linux") and os.environ.get("GITHUB_ACTIONS") == "true"
+
+
+@pytest.mark.skipif(
+    parse(torchao.__version__) < parse("0.11.0.dev0"),
+    reason="Only available on torchao >= 0.11.0.dev0",
+)
 class ExecuTorchModelIntegrationTest(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -42,14 +52,22 @@ def test_olmo_export_to_executorch(self):
         recipe = "xnnpack"
         with tempfile.TemporaryDirectory() as tempdir:
             subprocess.run(
-                f"optimum-cli export executorch --model {model_id} --task {task} --recipe {recipe} --output_dir {tempdir}/executorch",
+                f"optimum-cli export executorch \
+                    --model {model_id} \
+                    --task {task} \
+                    --recipe {recipe} \
+                    --output_dir {tempdir}/executorch \
+                    --use_custom_sdpa \
+                    --qlinear \
+                    --qembedding",
                 shell=True,
                 check=True,
             )
             self.assertTrue(os.path.exists(f"{tempdir}/executorch/model.pte"))
 
     @slow
     @pytest.mark.run_slow
+    @pytest.mark.skipif(is_linux_ci, reason="OOM on linux runner")
     def test_olmo_text_generation_with_xnnpack(self):
         model_id = "allenai/OLMo-1B-hf"
         model = ExecuTorchModelForCausalLM.from_pretrained(model_id, recipe="xnnpack")
@@ -74,13 +92,15 @@ def test_olmo_text_generation_with_xnnpack(self):
 
     @slow
     @pytest.mark.run_slow
-    def test_olmo_text_generation_with_custom_sdpa(self):
-        # ExecuTorch model + custom sdpa
+    def test_olmo_text_generation_with_custom_sdpa_8da4w_8we(self):
+        # ExecuTorch model + custom sdpa + 8da4w linear quantization + int8 embedding quantization
         model_id = "allenai/OLMo-1B-hf"
+        kwargs = {"qlinear": True, "qembedding": True}
         model = ExecuTorchModelForCausalLM.from_pretrained(
             model_id,
             recipe="xnnpack",
             attn_implementation="custom_sdpa",
+            **kwargs,
         )
         self.assertIsInstance(model, ExecuTorchModelForCausalLM)
         self.assertIsInstance(model.model, ExecuTorchModule)
diff --git a/tests/models/test_modeling_phi4.py b/tests/models/test_modeling_phi4.py
@@ -139,7 +139,8 @@ def test_phi4_text_generation_with_quantized_ckp(self):
         self.assertIsInstance(model, ExecuTorchModelForCausalLM)
         self.assertIsInstance(model.model, ExecuTorchModule)
 
-        tokenizer = AutoTokenizer.from_pretrained(model_id)
+        # Using "pytorch/Phi-4-mini-instruct-8da4w" will end up loading a wrong GPT2Tokenizer
+        tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-4-mini-instruct")
         generated_text = model.text_generation(
             tokenizer=tokenizer,
             prompt="My favourite condiment is ",
diff --git a/tests/models/test_modeling_qwen3.py b/tests/models/test_modeling_qwen3.py
@@ -41,13 +41,24 @@ def __init__(self, *args, **kwargs):
 
     @slow
     @pytest.mark.run_slow
+    @pytest.mark.skipif(
+        parse(torchao.__version__) < parse("0.11.0.dev0"),
+        reason="Only available on torchao >= 0.11.0.dev0",
+    )
     def test_qwen3_export_to_executorch(self):
         model_id = "Qwen/Qwen3-0.6B"
         task = "text-generation"
         recipe = "xnnpack"
         with tempfile.TemporaryDirectory() as tempdir:
             subprocess.run(
-                f"optimum-cli export executorch --model {model_id} --task {task} --recipe {recipe} --output_dir {tempdir}/executorch",
+                f"optimum-cli export executorch \
+                    --model {model_id} \
+                    --task {task} \
+                    --recipe {recipe} \
+                    --output_dir {tempdir}/executorch \
+                    --use_custom_sdpa \
+                    --qlinear \
+                    --qembedding",
                 shell=True,
                 check=True,
             )

Original file line number	Diff line number	Diff line change
`@@ -14,7 +14,7 @@`
`14`	`14`	`INSTALL_REQUIRE = [`
`15`	`15`	`"optimum~=1.24",`
`16`	`16`	`"executorch>=0.6.0",`
`17`		`- "transformers==4.51.0",`
	`17`	`+ "transformers==4.51.3",`
`18`	`18`	`]`
`19`	`19`
`20`	`20`	`TESTS_REQUIRE = [`