From f5f421fbf1334f68655764632701d7101631a24a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20R=C3=B6nninger?= Date: Tue, 26 Jan 2021 11:50:46 +0100 Subject: [PATCH] axi_xbar: Update inline documentation --- src/axi_xbar.sv | 244 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 203 insertions(+), 41 deletions(-) diff --git a/src/axi_xbar.sv b/src/axi_xbar.sv index 44ee6a070..f055a98a7 100644 --- a/src/axi_xbar.sv +++ b/src/axi_xbar.sv @@ -15,20 +15,152 @@ `include "axi/typedef.svh" -// axi_xbar: Fully-connected AXI4+ATOP crossbar with an arbitrary number of slave and master ports. -// See `doc/axi_xbar.md` for the documentation, including the definition of parameters and ports. +/// # AXI4+ATOP Fully-Connected Crossbar +/// +/// `axi_xbar` is a fully-connected crossbar that implements the full AXI4 specification plus atomic +/// operations (ATOPs) from AXI5 (E1.1). +/// +/// +/// ## Design Overview +/// +/// `axi_xbar` is a fully-connected crossbar, which means that each master module that is connected +/// to a *slave port* for of the crossbar has direct wires to all slave modules that are connected +/// to the *master ports* of the crossbar. +/// A block-diagram of the crossbar is shown below: +/// +/// ![Block-diagram showing the design of the full AXI4 Crossbar.](doc/axi_xbar.png +/// "Block-diagram showing the design of the full AXI4 Crossbar.") +/// +/// The crossbar has a configurable number of slave and master ports. +/// +/// The ID width of the master ports is **wider** than that of the slave ports. The additional ID +/// bits are used by the internal multiplexers to route responses. The ID width of the master ports +/// must be `AxiIdWidthSlvPorts + $clog_2(NoSlvPorts)`. +/// +/// +/// ## Address Map +/// +/// One address map is shared by all master ports. The *address map* contains an arbitrary number +/// of rules (but at least one). Each *rule* maps one address range to one master port. Multiple +/// rules can map to the same master port. The address ranges of two rules may overlap: in case two +/// address ranges overlap, the rule at the higher (more significant) position in the address map +/// prevails. +/// +/// Each address range includes the start address but does **not** include the end address. +/// That is, an address *matches* an address range if and only if +/// ``` +/// addr >= start_addr && addr < end_addr +/// ``` +/// The start address must be less than or equal to the end address. +/// +/// The address map can be defined and changed at run time (it is an input signal to the crossbar). +/// However, the address map must not be changed while any AW or AR channel of any slave port is +/// valid. +/// +/// [`addr_decode`](https://github.com/pulp-platform/common_cells/blob/master/src/addr_decode.sv) +/// module is used for decoding the address map. +/// +/// +/// ## Decode Errors and Default Slave Port +/// +/// Each slave port has its own internal *decode error slave* module. If the address of a +/// transaction does not match any rule, the transaction is routed to that decode error slave +/// module. That module absorbs each transaction and responds with a decode error (with the proper +/// number of beats). The data of each read response beat is `32'hBADCAB1E` (zero-extended or +/// truncated to match the data width). +/// +/// Each slave port can have a default master port. If the default master port is enabled for a +/// slave port, any address on that slave port that does not match any rule is routed to the default +/// master port instead of the decode error slave. The default master port can be enabled and +/// changed at run time (it is an input signal to the crossbar), and the same restrictions as for +/// the address map apply. +/// +/// +/// ## Ordering and Stalls +/// +/// When one slave port receives two transactions with the same ID and direction (i.e., both read or +/// both write) but targeting two different master ports, it will not accept the second transaction +/// until the first has completed. During this time, the crossbar stalls the AR or AW channel of +/// that slave port. To determine whether two transactions have the same ID, the +/// `AxiIdUsedSlvPorts` least-significant bits are compared. That parameter can be set to the full +/// `AxiIdWidthSlvPorts` to avoid false ID conflicts, or it can be set to a lower value to reduce +/// area and delay at the cost of more false conflicts. +/// +/// The reason for this ordering constraint is that AXI transactions with the same ID and direction +/// must remain ordered. If this crossbar would forward both transactions described above, the +/// second master port could get a response before the first one, and the crossbar would have to +/// reorder the responses before returning them on the master port. However, for efficiency +/// reasons, this crossbar does not have reorder buffers. +/// +/// +/// ## Verification Methodology +/// +/// This module has been verified with a directed random verification testbench, described and +/// implemented in [`tb_axi_xbar`](module.tb_axi_xbar) and +/// [`tb_axi_xbar_pkg`](package.tb_axi_xbar_pkg). +/// +/// +/// ## Design Rationale for No Pipelining Inside Crossbar +/// +/// Inserting spill registers between [`axi_demux`](module.axi_demux) and +/// [`axi_mux`](module.axi_mux) seems attractive to further reduce the length of combinatorial paths +/// in the crossbar. However, this can lead to deadlocks in the W channel where two different +/// [`axi_mux`](module.axi_mux) at the master ports would circular wait on two different +/// [`axi_demux`](module.axi_demux). In fact, spill registers between the switching modules causes +/// all four deadlock criteria to be met. Recall that the criteria are: +/// +/// 1. Mutual Exclusion +/// 2. Hold and Wait +/// 3. No Preemption +/// 4. Circular Wait +/// +/// The first criterion is given by the nature of a multiplexer on the W channel: all W beats have +/// to arrive in the same order as the AW beats regardless of the ID at the slave module. Thus, the +/// different master ports of the multiplexer exclude each other because the order is given by the +/// arbitration tree of the AW channel. +/// +/// The second and third criterion are inherent to the AXI protocol: For (2), the valid signal has +/// to be held high until the ready signal goes high. For (3), AXI does not allow interleaving of +/// W beats and requires W bursts to be in the same order as AW beats. +/// +/// The fourth criterion is thus the only one that can be broken to prevent deadlocks. However, +/// inserting a spill register between a master port of the [`axi_demux`](module.axi_demux) and a +/// slave port of the [[`axi_mux`](module.axi_mux) can lead to a circular dependency inside the +/// W FIFOs. This comes from the particular way the round robin arbiter from the AW channel in the +/// multiplexer defines its priorities. It is constructed in a way by giving each of its slave +/// ports an increasing priority and then comparing pairwise down till a winner is chosen. When the +/// winner gets transferred, the priority state is advanced by one position, preventing starvation. +/// +/// The problem can be shown with an example. Assume an arbitration tree with 10 inputs. Two +/// requests want to be served in the same clock cycle. The one with the higher priority wins and +/// the priority state advances. In the next cycle again the same two inputs have a request +/// waiting. Again it is possible that the same port as last time wins as the priority shifted only +/// one position further. This can lead in conjunction with the other arbitration trees in the +/// other muxes of the crossbar to the circular dependencies inside the FIFOs. Removing the spill +/// register between the demultiplexer and multiplexer forces the switching decision into the +/// W FIFOs in the same clock cycle. This leads to a strict ordering of the switching decision, +/// thus preventing the circular wait. +/// module axi_xbar #( - /// Number of slave ports of the crossbar. - /// This many master modules are connected to it. + /// The number of AXI slave ports of the crossbar. + /// (In other words, how many AXI master modules can be attached). parameter int unsigned NumSlvPorts = 32'd0, - /// Number of master ports of the crossbar. - /// This many slave modules are connected to it. + /// The number of AXI master ports of the crossbar. + /// (In other words, how many AXI slave modules can be attached). parameter int unsigned NumMstPorts = 32'd0, - /// AXI ID width of the slave ports. The ID width of the master ports is determined - /// Automatically. See `axi_mux` for details. + /// AXI ID width of the slave ports. + /// + /// This parameter also determines the corresponding value for `MstPortIdWidth` . + /// Routing of responses is done by extending the ID by the index of the slave port witch accepted + /// the transaction. See [`axi_mux`](module.axi_mux) for details. parameter int unsigned SlvPortIdWidth = 32'd0, - /// The used ID portion to determine if a different salve is used for the same ID. - /// See `axi_demux` for details. + /// The number of slave port ID bits (starting at the least significant) the crossbar uses to + /// determine the uniqueness of an AXI ID (see section *Ordering and Stalls* above). + /// + /// This value *must* follow `SlvPortIdWidth >= SlvPortIdWidthUsed && SlvPortIdWidthUsed > 0`. + /// + /// Setting this to small values leads to less area, however on an increased stalling rate + /// due to ID collisions. parameter int unsigned SlvPortIdWidthUsed = 32'd0, /// AXI4+ATOP address field width. parameter int unsigned AddrWidth = 32'd0, @@ -36,44 +168,48 @@ module axi_xbar #( parameter int unsigned DataWidth = 32'd0, /// AXI4+ATOP user field width. parameter int unsigned UserWidth = 32'd0, - /// Maximum number of open transactions each master connected to the crossbar can have in - /// flight at the same time. + /// Maximum number of open transactions each master connected to the crossbar can have + /// [in flight](doc/#in-flight) at the same time. parameter int unsigned SlvPortMaxTxns = 32'd0, - /// Maximum number of open transactions each slave connected to the crossbar can have in - /// flight at the same time. + /// Maximum number of open transactions each slave connected to the crossbar can have + /// [in flight](../doc#in-flight) per ID at the same time. parameter int unsigned MstPortMaxTxns = 32'd0, - /// Determine if the internal FIFOs of the crossbar are instantiated in fallthrough mode. + /// Routing decisions on the AW channel fall through to the W channel. Enabling this allows the + /// crossbar to accept a W beat in the same cycle as the corresponding AW beat, but it increases + /// the combinatorial path of the W channel with logic from the AW channel. + /// + /// Setting this to `0` prevents logic on the AW channel from extending into the W channel. + /// /// 0: No fallthrough /// 1: Fallthrough parameter bit FallThrough = 32'd1, - /// The Latency mode of the xbar. This determines if the channels on the ports have - /// a spill register instantiated. - /// Example configurations are provided with the enum `xbar_latency_e`. + /// The `LatencyMode` parameter allows to insert spill registers after each channel + /// (AW, W, B, AR, and R) of each master port (i.e., each [`axi_mux`](module.axi_mux)) and before + /// each channel of each slave port (i.e., each [`axi_demux`](module.axi_demux)). + /// Spill registers cut combinatorial paths, so this parameter reduces the length of combinatorial + /// paths through the crossbar. + /// + /// Some common configurations are given in the [`xbar_latency_e` `enum`](package.axi_pkg). + /// The recommended configuration (`axi_pkg::CUT_ALL_AX`) is to have a latency of 2 on the AW and + /// AR channels because these channels have the most combinatorial logic on them. + /// Additionally, `FallThrough` should be set to `0` to prevent logic on the AW channel from + /// extending combinatorial paths on the W channel. However, it is possible to run the crossbar + /// in a fully combinatorial configuration by setting `LatencyMode` to `NO_LATENCY` and + /// `FallThrough` to `1`. + /// + /// If two crossbars are connected in both directions, meaning both have one of their master ports + /// connected to a slave port of the other, the `LatencyMode` of both crossbars must be set to + /// either `CUT_SLV_PORTS`, `CUT_MST_PORTS`, or `CUT_ALL_PORTS`. Any other latency mode will lead + /// to timing loops on the uncut channels between the two crossbars. The latency mode of the two + /// crossbars does not have to be identical. parameter axi_pkg::xbar_latency_e LatencyMode = axi_pkg::CUT_ALL_AX, /// The number of address rules defined for routing of the transactions. - /// Each master port can have multiple rules, should have however at least one. + /// Each master port can have multiple rules, should have however at least one or be the + /// *default master port* of at least one slave port. /// If a transaction can not be routed the xbar will answer with an `axi_pkg::RESP_DECERR`. parameter int unsigned NumAddrRules = 32'd0, /// Enable atomic operations support. parameter bit EnableAtops = 1'b1, - // /// AXI4+ATOP AW channel struct type for the slave ports. - // parameter type slv_aw_chan_t = logic, - // /// AXI4+ATOP AW channel struct type for the master ports. - // parameter type mst_aw_chan_t = logic, - // /// AXI4+ATOP W channel struct type for all ports. - // parameter type w_chan_t = logic, - // /// AXI4+ATOP B channel struct type for the slave ports. - // parameter type slv_b_chan_t = logic, - // /// AXI4+ATOP B channel struct type for the master ports. - // parameter type mst_b_chan_t = logic, - // /// AXI4+ATOP AR channel struct type for the slave ports. - // parameter type slv_ar_chan_t = logic, - // /// AXI4+ATOP AR channel struct type for the master ports. - // parameter type mst_ar_chan_t = logic, - // /// AXI4+ATOP R channel struct type for the slave ports. - // parameter type slv_r_chan_t = logic, - // /// AXI4+ATOP R channel struct type for the master ports. - // parameter type mst_r_chan_t = logic, /// AXI4+ATOP request struct type for a single slave port. parameter type slv_port_axi_req_t = logic, /// AXI4+ATOP response struct type for a single slave port. @@ -84,7 +220,7 @@ module axi_xbar #( parameter type mst_port_axi_rsp_t = logic, /// Address rule type for the address decoders from `common_cells:addr_decode`. /// - /// Example types are provided in `axi_pkg`. + /// Example types are provided in [`axi_pkg`](package.axi_pkg). /// /// Required struct fields: /// @@ -102,6 +238,8 @@ module axi_xbar #( parameter type default_mst_port_idx_t = logic [DefaultMstPortIdxWidth-1:0] ) ( /// Clock, positive edge triggered. + /// + /// All other signals (except `rst_ni`) are synchronous to this signal. input logic clk_i, /// Asynchronous reset, active low. input logic rst_ni, @@ -121,6 +259,7 @@ module axi_xbar #( input rule_t [NumAddrRules-1:0] addr_map_i, /// Enables a default master port for each slave port. When this is enabled unmapped /// transactions get issued at the master port given by `default_mst_port_i`. + /// Each bit index corresponds to the index of a master port and is ordered little-endian (downto). /// /// When not used, tie to `'0`. input logic [NumSlvPorts-1:0] en_default_mst_port_i, @@ -343,37 +482,60 @@ endmodule `include "axi/assign.svh" -/// This is the interface wrapper for `axi_xbar`. Ports and parameters are analog to `axxi_xbar`. +/// This is the interface wrapper for `axi_xbar`. Ports and parameters are analog to `axi_xbar`, +/// see [`axi_xbar` documentation](module.axi_xbar). /// The AXI4+ATOP master and slave ports are structured here as interfaces. -/// Indexing of the interface is big-endian. This module does the little-endian indexing -/// for the port structs. +/// +/// The indexing of the master and slave port interface arrays is big-endian. module axi_xbar_intf #( + /// See [`axi_xbar`](module.axi_xbar). parameter int unsigned NumSlvPorts = 32'd0, + /// See [`axi_xbar`](module.axi_xbar). parameter int unsigned NumMstPorts = 32'd0, + /// See [`axi_xbar`](module.axi_xbar). parameter int unsigned SlvPortIdWidth = 32'd0, + /// See [`axi_xbar`](module.axi_xbar). parameter int unsigned SlvPortIdWidthUsed = 32'd0, + /// See [`axi_xbar`](module.axi_xbar). parameter int unsigned AddrWidth = 32'd0, + /// See [`axi_xbar`](module.axi_xbar). parameter int unsigned DataWidth = 32'd0, + /// See [`axi_xbar`](module.axi_xbar). parameter int unsigned UserWidth = 32'd0, + /// See [`axi_xbar`](module.axi_xbar). parameter int unsigned SlvPortMaxTxns = 32'd0, + /// See [`axi_xbar`](module.axi_xbar). parameter int unsigned MstPortMaxTxns = 32'd0, + /// See [`axi_xbar`](module.axi_xbar). parameter bit FallThrough = 32'd1, + /// See [`axi_xbar`](module.axi_xbar). parameter axi_pkg::xbar_latency_e LatencyMode = axi_pkg::CUT_ALL_AX, + /// See [`axi_xbar`](module.axi_xbar). parameter int unsigned NumAddrRules = 32'd0, + /// See [`axi_xbar`](module.axi_xbar). parameter bit EnableAtops = 1'b1, + /// See [`axi_xbar`](module.axi_xbar). parameter type rule_t = axi_pkg::xbar_rule_64_t, /// Dependent parameter, do **not** override! parameter int unsigned DefaultMstPortIdxWidth = cf_math_pkg::idx_width(NumMstPorts), /// Dependent parameter, do **not**parameter override! parameter type default_mst_port_idx_t = logic [DefaultMstPortIdxWidth-1:0] ) ( + /// See [`axi_xbar`](module.axi_xbar). input logic clk_i, + /// See [`axi_xbar`](module.axi_xbar). input logic rst_ni, + /// See [`axi_xbar`](module.axi_xbar). input logic test_i, + /// Unpacked, big-endian (upto) array of slave port interfaces. AXI_BUS.Slave slv_ports[NumSlvPorts], + /// Unpacked, big-endian (upto) array of master port interfaces. AXI_BUS.Master mst_ports[NumMstPorts], + /// See [`axi_xbar`](module.axi_xbar). input rule_t [NumAddrRules-1:0] addr_map_i, + /// See [`axi_xbar`](module.axi_xbar). input logic [NumSlvPorts -1:0] en_default_mst_port_i, + /// See [`axi_xbar`](module.axi_xbar). input default_mst_port_idx_t [NumSlvPorts -1:0] default_mst_ports_i );