Introduce verilog -> cocotb simulation fud2 path (#1997)

* merge Fud2 example (#1965)

* Sketch an example fud2 op

* Fix typo

---------

Co-authored-by: Adrian Sampson <[email protected]>

* axi-generator.py takes yxi file as argument (run python3 axi-generator.py input.yxi)

* bash script for whole axi workflow

* add input.yxi to axi-generator.py call in sim.sh

* pass runt test?

* diff between generated-axi-with-vec-add and fudaxi/cat.futil

* to verilog should theoretically work

* pass diff

* axi.sh works on fixed-vec-add-w-imports

* add futil to yxi operation to fud2

* almost working axi-gen

* Update fud2 calyx -> axi-wrapped-calyx path.

Testing locally on vec add, the hard-coded cocotb tests pass, meaning
there is parity with the previous bash scripts

* fud2 formatting

* update fud2 snapshots

* cleanup of extra files

* formatting of axi-generator

* make dedicated test dir for yxi

* tidy up old comments

* first pass at dynamic cocotb harness

* Remove axi_test prints and start on fud2 cocotb

* new Makefile and test runners in yxi/axi-calyx

Makefile expects a DATA_PATH and VERILOG_SOURCE

* WIP: Need to find a relative/absolute path makefile

cocotb expects datapath and relatviepath to be relative to the actual
Makefile, not from the invocation

* Working fud2 cocotb executor

* remove copies of Makefile and harnesses from

* remove extra fixed-vec-add-w-imports.futil

* add runt tests for fud2 invocation

* add runt tests

* attempt at runt tests. still periodically failing

* fix up runt tests

* update some runt tests

* more runt fixes on calyx to wrapped-calyx to verilog test

* Trigger Build

* add a cargo clean before building to get fud2 up to date

* add --manifest-path argument to cargo clean

* remove clean and add --all in build for compiler test

* attempt to create a unique directory for each fud2 execution

* version checking for --quiet

* maybe runt tests finally work

* Revert "maybe runt tests finally work"

This reverts commit 09ac9c2.

* add new expects for old ninja 1.10 we have on CI

* divert runt fud2 stderr to dev/null for sake of CI

* remove unecessary import

* cleanup

* PR comments: Some renaming mainly

* revert --workspace to --all in CI workflow

* remove a 2> /dev/null which didn't seem to od anything and make shorter cocotb firsts in CI

* remove all-features from workflow in favor of --features yxi because I don't want to migrate to rhai rn

* pipe fud2 cocotb stderr to devnull

---------

Co-authored-by: Elise Song <[email protected]>
Co-authored-by: Adrian Sampson <[email protected]>
Co-authored-by: eys29 <[email protected]>

.github/workflows/rust.yml

@@ -174,24 +174,24 @@ jobs:
       uses: actions-rs/cargo@v1
       with:
         command: build
-        args: --all-features --manifest-path /home/calyx/Cargo.toml
+        args: --workspace --features yxi --manifest-path /home/calyx/Cargo.toml
 
 #    - name: Source code doc tests
 #      uses: actions-rs/cargo@v1
 #      with:
 #        command: test
 #        args: --manifest-path /home/calyx/calyx/Cargo.toml --doc lib
 
-    - name: Runt tests
-      working-directory: /home/calyx
-      run: |
-        runt -x 'cocotb' -d -o fail -j 1 --max-futures 5
-
     # Run these sequentially because they might fail intermittently
     - name: Cocotb tests
       working-directory: /home/calyx
       run: |
         runt -i 'cocotb' -d -o fail -j 1 --max-futures 1
+    
+    - name: Runt tests
+      working-directory: /home/calyx
+      run: |
+        runt -x 'cocotb' -d -o fail -j 1 --max-futures 5
 
     - name: Run Python Tests
       working-directory: /home/calyx

Dockerfile

@@ -65,7 +65,7 @@ WORKDIR /home
 ADD . calyx
 # Build the compiler
 WORKDIR /home/calyx
-RUN cargo build --all && \
+RUN cargo build --workspace && \
     cargo install vcdump && \
     cargo install runt --version 0.4.1
 

fud2/src/lib.rs

@@ -155,7 +155,7 @@ pub fn build_driver(bld: &mut DriverBuilder) {
     let verilog_refmem_noverify = bld.state("verilog-refmem-noverify", &["sv"]);
 
     // Icarus Verilog.
-    let verilog_noverify = bld.state("verilog-noverify", &["sv"]);
+    let verilog_noverify = bld.state("verilog-noverify", &["sv", "v"]);
     let icarus_setup = bld.setup("Icarus Verilog", |e| {
         e.var("iverilog", "iverilog")?;
         e.rule(
@@ -780,18 +780,19 @@ pub fn build_driver(bld: &mut DriverBuilder) {
         calyx,
         |e, input, output| {
             // Generate the YXI file.
-            //no extension
+            // no extension
             let file_name = input
                 .rsplit_once('/')
                 .unwrap()
                 .1
                 .rsplit_once('.')
                 .unwrap()
                 .0;
-            let tmp_yxi = format!("{}.yxi", file_name);
 
-            //Get yxi file from main compute program.
-            //TODO(nate): Can this use the `yxi` operation instead of hardcoding the build cmd calyx rule with arguments?
+            // Get yxi file from main compute program.
+            // TODO(nate): Eventually (#1952) This will be able to use the `yxi` operation
+            // instead of hardcoding the build cmd calyx rule with arguments
+            let tmp_yxi = format!("{}.yxi", file_name);
             e.build_cmd(&[&tmp_yxi], "calyx", &[input], &[])?;
             e.arg("backend", "yxi")?;
 
@@ -822,4 +823,60 @@ pub fn build_driver(bld: &mut DriverBuilder) {
             Ok(())
         },
     );
+
+    let cocotb_setup = bld.setup("cocotb", |e| {
+        e.config_var_or("cocotb-makefile-dir", "cocotb.makefile-dir", "$calyx-base/yxi/axi-calyx/cocotb")?;
+        // TODO (nate): this is duplicated from the sim_setup above. Can this be shared?
+        // The input data file. `sim.data` is required.
+        let data_name = e.config_val("sim.data")?;
+        let data_path = e.external_path(data_name.as_ref());
+        e.var("sim_data", data_path.as_str())?;
+
+        // Cocotb is wants files relative to the location of the makefile.
+        // This is annoying to calculate on the fly, so we just copy necessary files to the build directory
+        e.rule("copy", "cp $in $out")?;
+        e.rule("make-cocotb", "make DATA_PATH=$sim_data VERILOG_SOURCE=$in COCOTB_LOG_LEVEL=CRITICAL > $out")?;
+        // This cleans up the extra `make` cruft, leaving what is in between `{` and `}.`
+        e.rule("cleanup-cocotb", r"sed -n '/Output:/,/make\[1\]/{/Output:/d;/make\[1\]/d;p}' $in > $out")?;
+        Ok(())
+    });
+
+    let cocotb_axi = bld.state("cocotb-axi", &["dat"]);
+    // Example invocation: `fud2 <path to axi wrapped verilog> --from verilog-noverify --to cocotb-axi --set sim.data=<path to .data/json file>`
+    bld.op(
+        "calyx-to-cocotb-axi",
+        &[calyx_setup, cocotb_setup],
+        verilog_noverify,
+        cocotb_axi,
+        |e, input, output| {
+            e.build_cmd(
+                &["Makefile"],
+                "copy",
+                &["$cocotb-makefile-dir/Makefile"],
+                &[],
+            )?;
+            e.build_cmd(
+                &["axi_test.py"],
+                "copy",
+                &["$cocotb-makefile-dir/axi_test.py"],
+                &[],
+            )?;
+            e.build_cmd(
+                &["run_axi_test.py"],
+                "copy",
+                &["$cocotb-makefile-dir/run_axi_test.py"],
+                &[],
+            )?;
+            e.build_cmd(
+                &["tmp.dat"],
+                "make-cocotb",
+                &[input],
+                &["Makefile", "axi_test.py", "run_axi_test.py"],
+            )?;
+
+            e.build_cmd(&[output], "cleanup-cocotb", &["tmp.dat"], &[])?;
+
+            Ok(())
+        },
+    );
 }

runt.toml

@@ -525,19 +525,34 @@ fud e {} -s verilog.cycle_limit 500 \
          -s verilog.data {}.data \
          --to dat -q
 """
-##### Cayx AXI tests #####
+##### Calyx AXI tests #####
 [[tests]]
 name = "calyx-py AXI wrapper generation"
 paths = ["yxi/tests/*.yxi"]
 cmd = """
 python3 yxi/axi-calyx/axi-generator.py {}
 """
 
+#Ignore fud2 stderr for now due to ninja nondeterminism
 [[tests]]
-name = "fud2 axi-wrapper invocation"
-paths = ["yxi/tests/fud2/*.futil"]
+name = "fud2 AXI wrapper invocation"
+paths = ["yxi/tests/fud2/seq-mem-vec-add.futil"]
 cmd = """
-fud2 {} --to calyx --through axi-wrapped
+fud2 {} --to calyx --through axi-wrapped 2> /dev/null
+"""
+
+[[tests]]
+name = "fud2 AXI-wrapped to verilog"
+paths = ["yxi/tests/fud2/seq-mem-vec-add-axi-wrapped.futil"]
+cmd = """
+fud2 {} --from calyx --to verilog-noverify 2> /dev/null
+"""
+
+[[tests]]
+name = "fud2 cocotb execution"
+paths = ["yxi/tests/fud2/seq-mem-vec-add-verilog.v"]
+cmd = """
+fud2 {} --from verilog-noverify --to cocotb-axi --set sim.data=yxi/tests/fud2/vectorized-add.data 2> /dev/null
 """
 
 ##### Xilinx Tests ######

yxi/axi-calyx/cocotb/Makefile

@@ -3,22 +3,20 @@
 # defaults
 SIM ?= icarus
 TOPLEVEL_LANG ?= verilog
-vfile ?= ../outputs/generated-axi-with-vec-add.v
 
 
 #Needed to extract desired test from runt invocation
 
-VERILOG_SOURCES += $(PWD)/${vfile}
+VERILOG_SOURCES += $(VERILOG_SOURCE)
 
 #Defines build directory, if left to default only a single computation is run
-SIM_BUILD=sim_build/axi-combined
+SIM_BUILD=sim_build/
 
 # TOPLEVEL is the name of the toplevel module in your Verilog or VHDL file
 TOPLEVEL = wrapper
 
 # MODULE is the basename of the Python test file
-MODULE = axi-combined-tests
-
+MODULE = run_axi_test
 
 # include cocotb's make rules to take care of the simulator setup
 include $(shell cocotb-config --makefiles)/Makefile.sim

yxi/axi-calyx/cocotb/axi_test.py

@@ -0,0 +1,150 @@
+import json
+import cocotb
+from cocotb.clock import Clock
+from cocotbext.axi import AxiBus, AxiRam
+from cocotb.triggers import Timer, FallingEdge, with_timeout, RisingEdge, ClockCycles
+from typing import Literal, Mapping, Any, Union, List
+from pathlib import Path
+import os
+
+
+# NOTE (nathanielnrn) cocotb-bus 0.2.1 has a bug that does not recognize optional
+# signals such as WSTRB when it is capitalized. Install directly from the cocotb-bus
+# github repo to fix
+class KernelTB:
+    def __init__(self, toplevel, data_path: Path):
+        self.toplevel = toplevel
+        self.data_path = data_path
+        assert os.path.isfile(
+            self.data_path
+        ), "data_path must be a data path to a valid file"
+
+    #Go through each mem, create an AxiRam, write data to it
+    async def setup_rams(self, data: Mapping[str, Any]):
+        # Create cocotb AxiRams
+        rams = {}
+        for mem in data.keys():
+            assert not isinstance(data[mem]["data"][0], list)
+            size = mem_size_in_bytes(mem, data)
+            width = data_width_in_bytes(mem, data)
+
+            # From_prefix assumes signals of form toplevel.<prefix>_<signal>
+            # i.e m0_axi_RDATA.
+            # These prefixes have to match verilog code. See kernel.xml <args>
+            # and ports assigned within that for guidance.
+            # In general, the index of `m<idx>_axi` just
+            # increments by 1 in fud axi generation
+            #print(f"mem is: {mem}")
+            rams[mem] = AxiRam(
+                AxiBus.from_prefix(self.toplevel, f"{mem}"),
+                self.toplevel.clk,
+                reset = self.toplevel.reset,
+                # self.toplevel.ap_rst_n,
+                size=size,
+            )
+
+            # NOTE: This defaults to little endian to match AxiRam defaults
+            data_in_bytes = encode(data[mem]["data"], width, byteorder="little", signed = bool(data[mem]["format"]["is_signed"]))
+            addr = 0x0000
+            rams[mem].write(addr, data_in_bytes)
+
+        self.rams = rams
+
+    def get_rams(self):
+        return self.rams
+
+    async def init_toplevel(self):
+        await Timer(50, "ns")
+        self.toplevel.reset.value = 1
+        await ClockCycles(self.toplevel.clk, 5)
+        self.toplevel.reset.value = 0
+        self.toplevel.go.value = 1
+
+
+async def run_kernel_test(toplevel, data_path: str):
+    tb = KernelTB(toplevel, Path(data_path))
+    data_map = None
+    with open(data_path) as f:
+        data_map = json.load(f)
+        f.close()
+    assert data_map is not None
+    await tb.setup_rams(data_map)
+    #print(data_map)
+
+
+    # set up clock of 2ns period, simulator default timestep is 1ps
+    cocotb.start_soon(Clock(toplevel.clk, 2, units="ns").start())
+    await tb.init_toplevel()
+    await Timer(100, "ns")
+    await FallingEdge(toplevel.clk)
+
+
+    # Finish when ap_done is high or 100 us of simulation have passed.
+    timeout = 5000
+    await with_timeout(RisingEdge(toplevel.done), timeout, "us")
+
+
+    # Get data from ram
+    mems: list[str] = list(data_map.keys())
+    rams = tb.get_rams()
+    post = {}
+    for mem in mems:
+        addr = 0x000
+        size = mem_size_in_bytes(mem, data_map)
+        post_execution = rams[mem].read(addr, size)
+        width = data_width_in_bytes(mem, data_map)
+        post_execution = decode(post_execution, width)
+        post.update({mem:{"data" : post_execution}})
+        post[mem]["format"] = data_map[mem]["format"]
+    # post = {"memories": post}
+
+    print("Output:\n" + json.dumps(post, indent = 2))
+
+
+def mem_size_in_bytes(mem: str, data):
+    """Returns size of memory within data in bytes"""
+    width = data_width_in_bytes(mem, data)
+    length = len(data[mem]["data"]) * width
+    return length
+
+
+def data_width_in_bytes(mem: str, data):
+    """Returns data width of mem in bytes"""
+    assert mem in data, "mem must be a key in data"
+    width = data[mem]["format"]["width"] // 8
+    if data[mem]["format"]["width"] % 8 != 0:
+        width += 1
+    return width
+
+
+# AxiRam assumes little bytorder, hence the defaults
+def decode(
+    b: bytes,
+    width: int,
+    byteorder: Union[Literal["little"], Literal["big"]] = "little",
+    signed=False,
+):
+    """Return the list of `ints` corresponding to value in `b` based on
+    encoding of `width` bytes
+    For example, `decode('b\x00\x00\x00\04', 4)` returns `[4]`
+    """
+    assert len(b) % width == 0, "Mismatch between bytes length and width"
+    to_return = []
+    for i in range(len(b) // width):
+        start = i * width
+        end = start + width
+        to_return.append(
+            int.from_bytes(b[start:end], byteorder=byteorder, signed=signed)
+        )
+    return to_return
+
+
+def encode(
+    lst: List[int],
+    width,
+    byteorder: Union[Literal["little"], Literal["big"]] = "little",
+    signed: bool = False
+) -> bytes:
+
+    """Return the `width`-wide byte representation of lst with byteorder"""
+    return b''.join(i.to_bytes(width, byteorder, signed=signed) for i in lst)

yxi/axi-calyx/cocotb/run_axi_test.py

@@ -0,0 +1,24 @@
+import cocotb
+import os
+
+#Idea is to have
+
+#MAKE file -> calls cocotb, runs a single function from here
+#this function looks for datapath
+
+#Makefile sets datapath and verilog interested in testing
+
+@cocotb.test()
+async def run(toplevel):
+    from axi_test import run_kernel_test
+
+    data_path = os.environ.get("DATA_PATH")
+    if not os.path.isabs(data_path):
+        data_path = os.getcwd() + "/" + data_path
+    assert data_path is not None and os.path.isfile(data_path), "DATA_PATH must be set and must be a valid file."
+
+    await run_kernel_test(toplevel, data_path)
+
+
+
+