CLI testing (#366)

* CLI testing


Create a basic pytest CI for the CLI.

Update pre-commit to use python3.12

Some CLI logging for consistency

Author: KelvinChung2000

.github/workflows/CIL_testing.yml

@@ -0,0 +1,13 @@
+name: Test CLI with pytest
+
+on: [push, pull_request]
+
+
+jobs:
+  run_pytest_CLI:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/prepare_FABulous_container
+      - name: Run pytest for FABulous CLI
+        run: pytest -v

FABulous/FABulous_CLI/FABulous_CLI.py

@@ -404,7 +404,7 @@ def do_gen_config_mem(self, args):
             self.fabulousAPI.genConfigMem(
                 i, self.projectDir / f"Tile/{i}/{i}_ConfigMem.csv"
             )
-        logger.info("Generating configMem complete")
+        logger.info("ConfigMem generation complete")
 
     @with_category(CMD_FABRIC_FLOW)
     @with_argparser(tile_list_parser)
@@ -501,7 +501,7 @@ def do_gen_all_tile(self, *ignored):
         """Generates all tiles by calling 'do_gen_tile'."""
         logger.info("Generating all tiles")
         self.do_gen_tile(" ".join(self.allTile))
-        logger.info("Generated all tiles")
+        logger.info("All tiles generation complete")
 
     @with_category(CMD_FABRIC_FLOW)
     def do_gen_fabric(self, *ignored):
@@ -609,7 +609,7 @@ def do_gen_bitStream_spec(self, *ignored):
                 w.writerow([key1])
                 for key2, val in specObject["TileSpecs"][key1].items():
                     w.writerow([key2, val])
-        logger.info("Generated bitstream specification")
+        logger.info("Bitstream specification generation complete")
 
     @with_category(CMD_FABRIC_FLOW)
     def do_gen_top_wrapper(self, *ignored):
@@ -619,7 +619,7 @@ def do_gen_top_wrapper(self, *ignored):
             f"{self.projectDir}/Fabric/{self.fabulousAPI.fabric.name}_top.{self.extension}"
         )
         self.fabulousAPI.genTopWrapper()
-        logger.info("Generated top wrapper")
+        logger.info("Top wrapper generation complete")
 
     @with_category(CMD_FABRIC_FLOW)
     @allow_blank

FABulous/FABulous_CLI/cmd_synthesis.py

@@ -241,6 +241,8 @@ def do_synthesis(self, args):
     p: Path
     paths: list[Path] = []
     for p in args.files:
+        if not p.is_absolute():
+            p = self.projectDir / p
         resolvePath: Path = p.absolute()
         if resolvePath.exists():
             paths.append(resolvePath)

pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+testpaths = tests

requirements.txt

@@ -3,3 +3,5 @@ loguru>=0.7.0
 fasm>=0.0.2
 cmd2>=2,<3
 requests>=2.0.0
+pytest>=8.3.5
+pytest-mock

tests/CLI_test/conftest.py

@@ -0,0 +1,66 @@
+import os
+from pathlib import Path
+
+import pytest
+from _pytest.logging import LogCaptureFixture
+from loguru import logger
+
+from FABulous.FABulous_CLI.FABulous_CLI import FABulous_CLI
+from FABulous.FABulous_CLI.helper import create_project, setup_logger
+
+
+def normalize(block: str):
+    """Normalize a block of text to perform comparison.
+
+    Strip newlines from the very beginning and very end, then split into separate lines and strip trailing whitespace
+    from each line.
+    """
+    assert isinstance(block, str)
+    block = block.strip("\n")
+    return [line.rstrip() for line in block.splitlines()]
+
+
+def run_cmd(app, cmd):
+    """Clear stdout, stdin and stderr buffers, run the command, and return stdout and stderr"""
+    app.onecmd_plus_hooks(cmd)
+
+
+def normalize_and_check_for_errors(caplog_text: str):
+    """Normalize a block of text and check for errors."""
+    log = normalize(caplog_text)
+    assert not any("ERROR" in line for line in log), "Error found in log messages"
+    return log
+
+
+TILE = "LUT4AB"
+
+os.environ["FAB_ROOT"] = str(Path(__file__).resolve().parent.parent.parent / "FABulous")
+
+
+@pytest.fixture
+def cli(tmp_path):
+    projectDir = tmp_path / "test_project"
+    fabulousRoot = str(Path(__file__).resolve().parent.parent.parent / "FABulous")
+    os.environ["FAB_ROOT"] = fabulousRoot
+    os.environ["FAB_PROJ_DIR"] = str(projectDir)
+    create_project(projectDir)
+    setup_logger(0)
+    cli = FABulous_CLI(
+        writerType="verilog", projectDir=projectDir, enteringDir=tmp_path
+    )
+    cli.debug = True
+    run_cmd(cli, "load_fabric")
+    return cli
+
+
+@pytest.fixture
+def caplog(caplog: LogCaptureFixture):
+    handler_id = logger.add(
+        caplog.handler,
+        format="{message}",
+        level=0,
+        filter=lambda record: record["level"].no >= caplog.handler.level,
+        enqueue=False,  # Set to 'True' if your test is spawning child processes.
+    )
+    yield caplog
+    logger.remove(handler_id)

tests/CLI_test/test_CLI.py

@@ -0,0 +1,134 @@
+from pathlib import Path
+import pytest
+from tests.CLI_test.conftest import (
+    TILE,
+    normalize_and_check_for_errors,
+    run_cmd,
+)
+
+
+def test_load_fabric(cli, caplog):
+    """Test loading fabric from CSV file"""
+    run_cmd(cli, "load_fabric")
+    log = normalize_and_check_for_errors(caplog.text)
+    assert "Loading fabric" in log[0]
+    assert "Complete" in log[-1]
+
+
+def test_gen_config_mem(cli, caplog):
+    """Test generating configuration memory"""
+    run_cmd(cli, f"gen_config_mem {TILE}")
+    log = normalize_and_check_for_errors(caplog.text)
+    assert f"Generating Config Memory for {TILE}" in log[0]
+    assert "ConfigMem generation complete" in log[-1]
+
+
+def test_gen_switch_matrix(cli, caplog):
+    """Test generating switch matrix"""
+    run_cmd(cli, f"gen_switch_matrix {TILE}")
+    log = normalize_and_check_for_errors(caplog.text)
+    assert f"Generating switch matrix for {TILE}" in log[0]
+    assert "Switch matrix generation complete" in log[-1]
+
+
+def test_gen_tile(cli, caplog):
+    """Test generating tile"""
+    run_cmd(cli, f"gen_tile {TILE}")
+    log = normalize_and_check_for_errors(caplog.text)
+    assert f"Generating tile {TILE}" in log[0]
+    assert "Tile generation complete" in log[-1]
+
+
+def test_gen_all_tile(cli, caplog):
+    """Test generating all tiles"""
+    run_cmd(cli, "gen_all_tile")
+    log = normalize_and_check_for_errors(caplog.text)
+    assert "Generating all tiles" in log[0]
+    assert "All tiles generation complete" in log[-1]
+
+
+def test_gen_fabric(cli, caplog):
+    """Test generating fabric"""
+    run_cmd(cli, "gen_fabric")
+    log = normalize_and_check_for_errors(caplog.text)
+    assert "Generating fabric " in log[0]
+    assert "Fabric generation complete" in log[-1]
+
+
+def test_gen_geometry(cli, caplog):
+    """Test generating geometry"""
+
+    # Test with default padding
+    run_cmd(cli, "gen_geometry")
+    log = normalize_and_check_for_errors(caplog.text)
+    assert "Generating geometry" in log[0]
+    assert "geometry generation complete" in log[-2].lower()
+
+    # Test with custom padding
+    run_cmd(cli, "gen_geometry 16")
+    log = normalize_and_check_for_errors(caplog.text)
+    assert "Generating geometry" in log[0]
+    assert "can now be imported into fabulator" in log[-1].lower()
+
+
+def test_gen_top_wrapper(cli, caplog):
+    """Test generating top wrapper"""
+    run_cmd(cli, "gen_top_wrapper")
+    log = normalize_and_check_for_errors(caplog.text)
+    assert "Generating top wrapper" in log[0]
+    assert "Top wrapper generation complete" in log[-1]
+
+
+def test_run_FABulous_fabric(cli, caplog):
+    """Test running FABulous fabric flow"""
+    run_cmd(cli, "run_FABulous_fabric")
+    log = normalize_and_check_for_errors(caplog.text)
+    assert "Running FABulous" in log[0]
+    assert "FABulous fabric flow complete" in log[-1]
+
+
+def test_gen_model_npnr(cli, caplog):
+    """Test generating Nextpnr model"""
+    run_cmd(cli, "gen_model_npnr")
+    log = normalize_and_check_for_errors(caplog.text)
+    assert "Generating npnr model" in log[0]
+    assert "Generated npnr model" in log[-1]
+
+
+def test_run_FABulous_bitstream(cli, caplog, mocker):
+    """Test the run_FABulous_bitstream command"""
+    m = mocker.patch("subprocess.run", return_value=None)
+    run_cmd(cli, "run_FABulous_fabric")
+    Path(cli.projectDir / "user_design" / "sequential_16bit_en.json").touch()
+    Path(cli.projectDir / "user_design" / "sequential_16bit_en.fasm").touch()
+    run_cmd(cli, "run_FABulous_bitstream ./user_design/sequential_16bit_en.v")
+    log = normalize_and_check_for_errors(caplog.text)
+    assert "Bitstream generated" in log[-1]
+    assert m.call_count == 3
+
+
+def test_run_simulation(cli, caplog, mocker):
+    """Test running simulation"""
+    m = mocker.patch("subprocess.run", return_value=None)
+    run_cmd(cli, "run_FABulous_fabric")
+    Path(cli.projectDir / "user_design" / "sequential_16bit_en.json").touch()
+    Path(cli.projectDir / "user_design" / "sequential_16bit_en.fasm").touch()
+    Path(cli.projectDir / "user_design" / "sequential_16bit_en.bin").touch()
+    run_cmd(cli, "run_FABulous_bitstream ./user_design/sequential_16bit_en.v")
+    run_cmd(cli, "run_simulation fst ./user_design/sequential_16bit_en.bin")
+    log = normalize_and_check_for_errors(caplog.text)
+    assert "Simulation finished" in log[-1]
+    assert m.call_count == 5
+
+
+def test_run_tcl(cli, caplog, tmp_path):
+    """Test running a Tcl script"""
+    script_content = '# Dummy Tcl script\nputs "Text from tcl"'
+    tcl_script_path = tmp_path / "test_script.tcl"
+    with open(tcl_script_path, "w") as f:
+        f.write(script_content)
+
+    run_cmd(cli, f"run_tcl {str(tcl_script_path)}")
+    log = normalize_and_check_for_errors(caplog.text)
+    assert f"Execute TCL script {str(tcl_script_path)}" in log[0]
+    assert "TCL script executed" in log[-1]

tests/CLI_test/test_helper.py

@@ -0,0 +1,52 @@
+import pytest
+
+from FABulous.FABulous_CLI.helper import create_project
+
+
+def test_create_project(tmp_path):
+    # Test Verilog project creation
+    project_dir = tmp_path / "test_project_verilog"
+    create_project(project_dir)
+
+    # Check if directories exist
+    assert project_dir.exists()
+    assert (project_dir / ".FABulous").exists()
+
+    # Check if .env file exists and contains correct content
+    env_file = project_dir / ".FABulous" / ".env"
+    assert env_file.exists()
+    assert env_file.read_text() == "FAB_PROJ_LANG=verilog\n"
+
+    # Check if template files were copied
+    assert any(
+        project_dir.glob("**/*.v")
+    ), "No Verilog files found in project directory"
+
+
+def test_create_project_vhdl(tmp_path):
+    # Test VHDL project creation
+    project_dir = tmp_path / "test_project_vhdl"
+    create_project(project_dir, lang="vhdl")
+
+    # Check if directories exist
+    assert project_dir.exists()
+    assert (project_dir / ".FABulous").exists()
+
+    # Check if .env file exists and contains correct content
+    env_file = project_dir / ".FABulous" / ".env"
+    assert env_file.exists()
+    assert env_file.read_text() == "FAB_PROJ_LANG=vhdl\n"
+
+    # Check if template files were copied
+    assert any(
+        project_dir.glob("**/*.vhdl")
+    ), "No VHDL files found in project directory"
+
+
+def test_create_project_existing_dir(tmp_path):
+    # Test creating project in existing directory
+    project_dir = tmp_path / "existing_dir"
+    project_dir.mkdir()
+
+    with pytest.raises(SystemExit):
+        create_project(project_dir)