Queues: tidy queue call, oracles, data gen, FIFOs (#2163)

* Tidy imports

* Comment

* Inline more wires

* Simplify wires section further

* Tweak

* Some thrashing

* Silly fix

* Get back the keepgoing option

* Set the basic queues to keep going

* Give python oracles a keepgoing option

* New script to keep going more often

* Do away with error mode. It is now always n on

* Make queue_call lower err

* New data and expect files that don't useuse the no-err flag

* Inline wires in FIFO

* More niceties in FIFO

* A hair more refactoring in FIFO

* Raise err in case of invalid command

* Keepgoing flag in calyx-level generators too!

calyx-py/calyx/builder.py

@@ -132,7 +132,12 @@ def output(
         """
         return self._port_with_attributes(name, size, False, attribute_literals)
 
-    def output_with_attributes(self, name: str, size: int, attribute_literals: List[Union[str, Tuple[str, int]]]) -> ExprBuilder:
+    def output_with_attributes(
+        self,
+        name: str,
+        size: int,
+        attribute_literals: List[Union[str, Tuple[str, int]]],
+    ) -> ExprBuilder:
         """Declare an output port on the component with attributes.
 
         Returns an expression builder for the port.
@@ -555,7 +560,7 @@ def const_mult(
         name = name or self.generate_name("const_mult")
         self.prog.import_("primitives/binary_operators.futil")
         return self.cell(name, ast.Stdlib.const_mult(size, const))
-        
+
     def pad(self, in_width: int, out_width: int, name: str = None) -> CellBuilder:
         """Generate a StdPad cell."""
         name = name or self.generate_name("pad")
@@ -890,15 +895,7 @@ def mem_load_to_mem(self, mem, i, ans, j, groupname):
             load_grp.done = ans.done
         return load_grp
 
-    def op_store_in_reg(
-        self,
-        op_cell,
-        left,
-        right,
-        cellname,
-        width,
-        ans_reg=None
-    ):
+    def op_store_in_reg(self, op_cell, left, right, cellname, width, ans_reg=None):
         """Inserts wiring into `self` to perform `reg := left op right`,
         where `op_cell`, a Cell that performs some `op`, is provided.
         """
@@ -974,35 +971,30 @@ def neq_store_in_reg(
         cell = self.neq(width, cellname, signed)
         return self.op_store_in_reg(cell, left, right, cell.name, 1, ans_reg)
 
+    def le_store_in_reg(
+        self, left, right, ans_reg=None, cellname=None, width=None, signed=False
+    ):
+        """Inserts wiring into `self` to perform `reg := left <= right`."""
+        width = width or self.try_infer_width(width, left, right)
+        cell = self.le(width, cellname, signed)
+        return self.op_store_in_reg(cell, left, right, cell.name, 1, ans_reg)
+
     def mult_store_in_reg(
-        self,
-        left,
-        right,
-        ans_reg=None,
-        cellname=None,
-        width=None,
-        signed=False
+        self, left, right, ans_reg=None, cellname=None, width=None, signed=False
     ):
         """Inserts wiring into `self` to perform `reg := left * right`."""
         width = width or self.try_infer_width(width, left, right)
         cell = self.mult_pipe(width, cellname, signed)
         return self.op_store_in_reg(cell, left, right, cell.name, width, ans_reg)
 
     def div_store_in_reg(
-        self,
-        left,
-        right,
-        ans_reg=None,
-        cellname=None,
-        width=None,
-        signed=False
+        self, left, right, ans_reg=None, cellname=None, width=None, signed=False
     ):
         """Inserts wiring into `self` to perform `reg := left / right`."""
         width = width or self.try_infer_width(width, left, right)
         cell = self.div_pipe(width, cellname, signed)
         return self.op_store_in_reg(cell, left, right, cell.name, width, ans_reg)
 
-
     def infer_width(self, expr) -> int:
         """Infer the width of an expression."""
         if isinstance(expr, int):  # We can't infer the width of an integer.

calyx-py/calyx/fifo_oracle.py

@@ -6,7 +6,8 @@
 
 if __name__ == "__main__":
     max_cmds, len = int(sys.argv[1]), int(sys.argv[2])
+    keepgoing = "--keepgoing" in sys.argv
     commands, values = queue_util.parse_json()
     fifo = queues.Fifo(len)
-    ans = queues.operate_queue(commands, values, fifo, max_cmds)
+    ans = queues.operate_queue(commands, values, fifo, max_cmds, keepgoing=keepgoing)
     queue_util.dump_json(commands, values, ans)

calyx-py/calyx/gen_queue_data_expect.sh

@@ -3,7 +3,7 @@
 # For SDN, we use piezo mode when making the data file and
 # use pifotree_oracle to generate the expected output
 python3 queue_data_gen.py 20000 --no-err 16 > ../test/correctness/queues/sdn.data
-cat ../test/correctness/queues/sdn.data | python3 pifo_tree_oracle.py 20000 16 > ../test/correctness/queues/sdn.expect
+cat ../test/correctness/queues/sdn.data | python3 pifo_tree_oracle.py 20000 16 --keepgoing > ../test/correctness/queues/sdn.expect
 
 # For the others, we drop piezo mode for data gen, and we use the appropriate
 # oracle, which is one of the following:
@@ -12,6 +12,6 @@ cat ../test/correctness/queues/sdn.data | python3 pifo_tree_oracle.py 20000 16 >
 # - pifo_tree_oracle.py
 
 for queue_kind in fifo pifo pifo_tree; do
-    python3 queue_data_gen.py 20000 --no-err 16 > ../test/correctness/queues/$queue_kind.data
-    cat ../test/correctness/queues/$queue_kind.data | python3 ${queue_kind}_oracle.py 20000 16 > ../test/correctness/queues/$queue_kind.expect
+    python3 queue_data_gen.py 20000 > ../test/correctness/queues/$queue_kind.data
+    cat ../test/correctness/queues/$queue_kind.data | python3 ${queue_kind}_oracle.py 20000 16 --keepgoing > ../test/correctness/queues/$queue_kind.expect
 done

calyx-py/calyx/pifo_oracle.py

@@ -6,10 +6,11 @@
 
 if __name__ == "__main__":
     max_cmds, len = int(sys.argv[1]), int(sys.argv[2])
+    keepgoing = "--keepgoing" in sys.argv
     commands, values = queue_util.parse_json()
 
     # Our PIFO is simple: it just orchestrates two FIFOs. The boundary is 200.
     pifo = queues.Pifo(queues.Fifo(len), queues.Fifo(len), 200, len)
 
-    ans = queues.operate_queue(commands, values, pifo, max_cmds)
+    ans = queues.operate_queue(commands, values, pifo, max_cmds, keepgoing=keepgoing)
     queue_util.dump_json(commands, values, ans)

calyx-py/calyx/pifo_tree_oracle.py

@@ -8,6 +8,7 @@
 if __name__ == "__main__":
 
     max_cmds, len = int(sys.argv[1]), int(sys.argv[2])
+    keepgoing = "--keepgoing" in sys.argv
     commands, values = queue_util.parse_json()
 
     # Our PIFO is a little complicated: it is a tree of queues.
@@ -27,5 +28,5 @@
         len,
     )
 
-    ans = queues.operate_queue(commands, values, pifo, max_cmds)
+    ans = queues.operate_queue(commands, values, pifo, max_cmds, keepgoing=keepgoing)
     queue_util.dump_json(commands, values, ans)

calyx-py/calyx/queue_call.py

@@ -1,14 +1,31 @@
 # pylint: disable=import-error
-from . import py_ast as ast
-from calyx import queue_util
+from .py_ast import Empty
 import calyx.builder as cb
 
 
 def insert_runner(prog, queue, name, num_cmds, stats_component=None):
     """Inserts the component `name` into the program.
-    This will be used to `invoke` the component `queue` and feed it one command.
+    This will be used to `invoke` the component `queue` and feed it _one command_.
     This component is designed to be invoked by some other component, and does not
     directly interface with external memories.
+
+    The user-facing interface of this component is captured by a number
+    of items that are passed to this component by reference.
+    #
+    - 1: `commands`, a list of commands.
+       Where each command is a 2-bit unsigned integer with the following format:
+       `0`: pop
+       `1`: peek
+       `2`: push
+    - 2: `values`, a list of values.
+       Where each value is a 32-bit unsigned integer.
+       The value at `i` is pushed if the command at `i` is `2`.
+    - 3: `has_ans`, a 1-bit unsigned integer.
+       We raise/lower this to indicate whether the queue had a reply to the command.
+    - 4: `component_ans`, a 32-bit unsigned integer.
+       We put in this register the answer, if any.
+    - 5: `component_err`, a 1-bit unsigned integer.
+       We raise/lower it to indicate whether an error occurred.
     """
     assert (
         name != "main"
@@ -18,31 +35,14 @@ def insert_runner(prog, queue, name, num_cmds, stats_component=None):
 
     # We take a stats component by reference,
     # but all we'll really do with it is pass it to the queue component.
-    if stats_component:
-        stats = runner.cell("stats_runner", stats_component, is_ref=True)
+    stats_cell = (
+        runner.cell("stats_runner", stats_component, is_ref=True)
+        if stats_component
+        else None
+    )
 
     # We'll invoke the queue component.
     queue = runner.cell("myqueue", queue)
-
-    # The user-facing interface of this component is captured by a number
-    # of items that are passed to this component by reference.
-    #
-    # - 1: `commands`, a list of commands.
-    #    Where each command is a 2-bit unsigned integer with the following format:
-    #    `0`: pop
-    #    `1`: peek
-    #    `2`: push
-    # - 2: `values`, a list of values.
-    #    Where each value is a 32-bit unsigned integer.
-    #    The value at `i` is pushed if the command at `i` is `2`.
-    # - 3: `has_ans`, a 1-bit unsigned integer.
-    #    We raise/lower this to indicate whether the queue had a reply to the command.
-    # - 4: `component_ans`, a 32-bit unsigned integer.
-    #    We put in this register the answer to the command, if any.
-    # - 5: `component_err`, a 1-bit unsigned integer.
-    #    We raise/lower it to indicates whether an error occurred
-    #    and the queue should no longer be invoked.
-    #
     # The user-facing interface of the `queue` component is assumed to be:
     # - input `cmd`
     #    where each command is a 2-bit unsigned integer, with the following format:
@@ -65,35 +65,20 @@ def insert_runner(prog, queue, name, num_cmds, stats_component=None):
     cmd = runner.reg(2)  # The command we're currently processing
     value = runner.reg(32)  # The value we're currently processing
 
-    incr_i = runner.incr(i)  # i++
-    cmd_le_1 = runner.le_use(cmd.out, 1)  # cmd <= 1, meaning cmd is pop or peek
-
-    # Wiring to perform `cmd := commands[i]` and `value := values[i]`.
-    write_cmd_to_reg = runner.mem_load_d1(commands, i.out, cmd, "write_cmd")
-    write_value_to_reg = runner.mem_load_d1(values, i.out, value, "write_value")
-
-    # Wiring to raise/lower flags and compute a negation.
-    raise_has_ans = runner.reg_store(has_ans, 1, "raise_has_ans")
-    lower_has_ans = runner.reg_store(has_ans, 0, "lower_has_ans")
-    not_err = runner.not_use(err.out)
-
-    # Wiring that raises `err` iff `i = num_cmds`.
-    check_if_out_of_cmds, _ = runner.eq_store_in_reg(i.out, num_cmds, err)
-
     runner.control += [
-        write_cmd_to_reg,  # `cmd := commands[i]`
-        write_value_to_reg,  # `value := values[i]`
+        runner.mem_load_d1(commands, i.out, cmd, "write_cmd"),  # `cmd := commands[i]`
+        runner.mem_load_d1(values, i.out, value, "write_value"),  # `value := values[i]`
         (
-            cb.invoke(  # Invoke the queue.
+            cb.invoke(  # Invoke the queue with a stats component.
                 queue,
                 in_cmd=cmd.out,
                 in_value=value.out,
                 ref_ans=ans,
                 ref_err=err,
-                ref_stats=stats,
+                ref_stats=stats_cell,
             )
             if stats_component
-            else cb.invoke(  # Invoke the queue.
+            else cb.invoke(  # Invoke the queue without a stats component.
                 queue,
                 in_cmd=cmd.out,
                 in_value=value.out,
@@ -103,23 +88,28 @@ def insert_runner(prog, queue, name, num_cmds, stats_component=None):
         ),
         # We're back from the invoke, and it's time for some post-mortem analysis.
         cb.if_with(
-            not_err,  # If there was no error
+            runner.not_use(err.out),  # If there was no error
             [
-                cb.if_with(
-                    cmd_le_1,  # If the command was a pop or peek
-                    raise_has_ans,  # then raise the `has_ans` flag
-                    lower_has_ans,  # else lower the `has_ans` flag
-                ),
+                # If cmd <= 1, meaning cmd is pop or peek, raise the `has_ans` flag.
+                # Otherwise, lower the `has_ans` flag.
+                runner.le_store_in_reg(cmd.out, 1, has_ans)[0]
             ],
         ),
-        incr_i,  # Increment the command index
-        check_if_out_of_cmds,  # If we're out of commands, raise `err`
+        runner.incr(i),  # i++
+        runner.reg_store(err, 0, "lower_err"),  # Lower the error flag.
     ]
 
     return runner
 
 
-def insert_main(prog, queue, num_cmds, controller=None, stats_component=None):
+def insert_main(
+    prog,
+    queue,
+    num_cmds,
+    keepgoing=False,
+    controller=None,
+    stats_component=None,
+):
     """Inserts the component `main` into the program.
     It triggers the dataplane and controller components.
     """
@@ -138,26 +128,32 @@ def insert_main(prog, queue, num_cmds, controller=None, stats_component=None):
     commands = main.seq_mem_d1("commands", 2, num_cmds, 32, is_external=True)
     values = main.seq_mem_d1("values", 32, num_cmds, 32, is_external=True)
     ans_mem = main.seq_mem_d1("ans_mem", 32, num_cmds, 32, is_external=True)
-
-    ans_neq_0 = main.neq_use(dataplane_ans.out, 0)  # ans != 0
-
+    i = main.reg(32)  # A counter for how many times we have invoked the dataplane.
     j = main.reg(32)  # The index on the answer-list we'll write to
-    incr_j = main.incr(j)  # j++
-    write_ans = main.mem_store_d1(ans_mem, j.out, dataplane_ans.out, "write_ans")
-    # ans_mem[j] = dataplane_ans
-    lower_has_ans = main.reg_store(has_ans, 0, "lower_has_ans")  # has_ans := 0
-
-    not_err = main.not_use(dataplane_err.out)
+    keep_looping = main.and_(1)  # If this is high, we keep going. Otherwise, we stop.
+    lt = main.lt(32)
+    not_err = main.not_(1)
+
+    with main.comb_group("Compute_keep_looping") as compute_keep_looping:
+        # The condition to keep looping is:
+        # The index `i` is less than the number of commands `num_cmds`
+        # AND
+        # If the `keepgoing` flag is _not_ set, the dataplane error is not high.
+        lt.left = i.out
+        lt.right = num_cmds
+        not_err.in_ = dataplane_err.out
+        keep_looping.left = lt.out
+        keep_looping.right = cb.HI if keepgoing else not_err.out
 
     main.control += cb.while_with(
         # We will run the dataplane and controller components in sequence,
-        # in a while loop. The loop will terminate when the dataplane component
-        # raises `dataplane_err`.
-        not_err,  # While the dataplane component has not errored out.
+        # in a while loop. The loop will terminate when `break_` has a value of `1`.
+        cb.CellAndGroup(keep_looping, compute_keep_looping),
         [
-            lower_has_ans,  # Lower the has-ans flag.
+            main.reg_store(has_ans, 0, "lower_has_ans"),  # Lower the has-ans flag.
             (
-                cb.invoke(  # Invoke the dataplane component.
+                cb.invoke(
+                    # Invoke the dataplane component with a stats component.
                     dataplane,
                     ref_commands=commands,
                     ref_values=values,
@@ -167,7 +163,8 @@ def insert_main(prog, queue, num_cmds, controller=None, stats_component=None):
                     ref_stats_runner=stats,
                 )
                 if stats_component
-                else cb.invoke(  # Invoke the dataplane component.
+                else cb.invoke(
+                    # Invoke the dataplane component without a stats component.
                     dataplane,
                     ref_commands=commands,
                     ref_values=values,
@@ -180,15 +177,24 @@ def insert_main(prog, queue, num_cmds, controller=None, stats_component=None):
             # write it to the answer-list and increment the index `j`.
             cb.if_(
                 has_ans.out,
-                cb.if_with(ans_neq_0, [write_ans, incr_j]),
+                cb.if_with(
+                    main.neq_use(dataplane_ans.out, 0),
+                    [
+                        main.mem_store_d1(
+                            ans_mem, j.out, dataplane_ans.out, "write_ans"
+                        ),
+                        main.incr(j),
+                    ],
+                ),
             ),
             (
                 cb.invoke(  # Invoke the controller component.
                     controller,
                     ref_stats_controller=stats,
                 )
                 if controller
-                else ast.Empty
+                else Empty
             ),
+            main.incr(i),  # i++
         ],
     )