axi4: simplify Fragmenter by using user bits

src/main/scala/uncore/axi4/Fragmenter.scala

@@ -10,8 +10,7 @@ import diplomacy._
 import scala.math.{min,max}
 import uncore.tilelink2.{leftOR, rightOR, UIntToOH1, OH1ToOH}
 
-// lite: masters all use only one ID => reads will not be interleaved
+class AXI4Fragmenter()(implicit p: Parameters) extends LazyModule
-class AXI4Fragmenter(lite: Boolean = false, maxInFlight: => Int = 32, combinational: Boolean = true)(implicit p: Parameters) extends LazyModule
 {
   val maxBeats = 1 << AXI4Parameters.lenBits
   def expandTransfer(x: TransferSizes, beatBytes: Int, alignment: BigInt) =
@@ -19,11 +18,11 @@ class AXI4Fragmenter(lite: Boolean = false, maxInFlight: => Int = 32, combinatio
   def mapSlave(s: AXI4SlaveParameters, beatBytes: Int) = s.copy(
     supportsWrite = expandTransfer(s.supportsWrite, beatBytes, s.minAlignment),
     supportsRead  = expandTransfer(s.supportsRead,  beatBytes, s.minAlignment),
-    interleavedId = if (lite) Some(0) else s.interleavedId) // see AXI4FragmenterSideband for !lite case
+    interleavedId = None) // this breaks interleaving guarantees
   def mapMaster(m: AXI4MasterParameters) = m.copy(aligned = true)
 
   val node = AXI4AdapterNode(
-    masterFn = { mp => mp.copy(masters = mp.masters.map(m => mapMaster(m))) },
+    masterFn = { mp => mp.copy(masters = mp.masters.map(m => mapMaster(m)), userBits = mp.userBits + 1) },
     slaveFn  = { sp => sp.copy(slaves  = sp.slaves .map(s => mapSlave(s, sp.beatBytes))) })
 
   lazy val module = new LazyModuleImp(this) {
@@ -40,9 +39,6 @@ class AXI4Fragmenter(lite: Boolean = false, maxInFlight: => Int = 32, combinatio
       val master    = edgeIn.master
       val masters   = master.masters
 
-      // If the user claimed this was a lite interface, then there must be only one Id
-      require (!lite || master.endId == 1)
-
       // We don't support fragmenting to sub-beat accesses
       slaves.foreach { s =>
         require (!s.supportsRead  || s.supportsRead.contains(beatBytes))
@@ -139,154 +135,73 @@ class AXI4Fragmenter(lite: Boolean = false, maxInFlight: => Int = 32, combinatio
       val readSizes1  = slaves.map(s => s.supportsRead .max/beatBytes-1)
       val writeSizes1 = slaves.map(s => s.supportsWrite.max/beatBytes-1)
 
-      // Indirection variables for inputs and outputs; makes transformation application easier
+      // Irrevocable queues in front because we want to accept the request before responses come back
       val (in_ar, ar_last, _)       = fragment(Queue.irrevocable(in.ar, 1, flow=true), readSizes1)
       val (in_aw, aw_last, w_beats) = fragment(Queue.irrevocable(in.aw, 1, flow=true), writeSizes1)
-      val in_w = in.w
-      val in_r = in.r
-      val in_b = in.b
-      val out_ar = Wire(out.ar)
-      val out_aw = out.aw
-      val out_w = out.w
-      val out_r = Wire(out.r)
-      val out_b = Wire(out.b)
 
-      val depth = if (combinational) 1 else 2
+      // AR flow control; super easy
-      // In case a slave ties arready := rready, we need a queue to break the combinational loop
+      out.ar <> in_ar
-      // between the two branches (in_ar => {out_ar => out_r, sideband} => in_r).
+      out.ar.bits.user.get := Cat(in_ar.bits.user.toList ++ Seq(ar_last))
-      if (in.ar.bits.getWidth < in.r.bits.getWidth) {
-        out.ar <> Queue(out_ar, depth, flow=combinational)
-        out_r <> out.r
-      } else {
-        out.ar <> out_ar
-        out_r <> Queue(out.r, depth, flow=combinational)
-      }
-      // In case a slave ties awready := bready or wready := bready, we need this queue
-      out_b <> Queue(out.b, depth, flow=combinational)
-
-      // Sideband to track which transfers were the last fragment
-      def sideband() = if (lite) {
-        Module(new Queue(Bool(), maxInFlight, flow=combinational)).io
-      } else {
-        Module(new AXI4FragmenterSideband(maxInFlight, flow=combinational)).io
-      }
-      val sideband_ar_r = sideband()
-      val sideband_aw_b = sideband()
-
-      // AR flow control
-      out_ar.valid := in_ar.valid && sideband_ar_r.enq.ready
-      in_ar.ready := sideband_ar_r.enq.ready && out_ar.ready
-      sideband_ar_r.enq.valid := in_ar.valid && out_ar.ready
-      out_ar.bits := in_ar.bits
-      sideband_ar_r.enq.bits := ar_last
 
       // When does W channel start counting a new transfer
       val wbeats_latched = RegInit(Bool(false))
       val wbeats_ready = Wire(Bool())
       val wbeats_valid = Wire(Bool())
       when (wbeats_valid && wbeats_ready) { wbeats_latched := Bool(true) }
-      when (out_aw.fire()) { wbeats_latched := Bool(false) }
+      when (out.aw.fire()) { wbeats_latched := Bool(false) }
 
       // AW flow control
-      out_aw.valid := in_aw.valid && sideband_aw_b.enq.ready && (wbeats_ready || wbeats_latched)
+      out.aw.valid := in_aw.valid && (wbeats_ready || wbeats_latched)
-      in_aw.ready := sideband_aw_b.enq.ready && out_aw.ready && (wbeats_ready || wbeats_latched)
+      in_aw.ready := out.aw.ready && (wbeats_ready || wbeats_latched)
-      sideband_aw_b.enq.valid := in_aw.valid && out_aw.ready && (wbeats_ready || wbeats_latched)
       wbeats_valid := in_aw.valid && !wbeats_latched
-      out_aw.bits := in_aw.bits
+      out.aw.bits := in_aw.bits
-      sideband_aw_b.enq.bits := aw_last
+      out.aw.bits.user.get := Cat(in_aw.bits.user.toList ++ Seq(aw_last))
 
       // We need to inject 'last' into the W channel fragments, count!
       val w_counter = RegInit(UInt(0, width = AXI4Parameters.lenBits+1))
       val w_idle = w_counter === UInt(0)
       val w_todo = Mux(w_idle, Mux(wbeats_valid, w_beats, UInt(0)), w_counter)
       val w_last = w_todo === UInt(1)
-      w_counter := w_todo - out_w.fire()
+      w_counter := w_todo - out.w.fire()
-      assert (!out_w.fire() || w_todo =/= UInt(0)) // underflow impossible
+      assert (!out.w.fire() || w_todo =/= UInt(0)) // underflow impossible
 
       // W flow control
       wbeats_ready := w_idle
-      out_w.valid := in_w.valid && (!wbeats_ready || wbeats_valid)
+      out.w.valid := in.w.valid && (!wbeats_ready || wbeats_valid)
-      in_w.ready := out_w.ready && (!wbeats_ready || wbeats_valid)
+      in.w.ready := out.w.ready && (!wbeats_ready || wbeats_valid)
-      out_w.bits := in_w.bits
+      out.w.bits := in.w.bits
-      out_w.bits.last := w_last
+      out.w.bits.last := w_last
       // We should also recreate the last last
-      assert (!out_w.valid || !in_w.bits.last || w_last)
+      assert (!out.w.valid || !in.w.bits.last || w_last)
 
       // R flow control
-      val r_last = out_r.bits.last
+      val r_last = out.r.bits.user.get(0)
-      in_r.valid := out_r.valid && (!r_last || sideband_ar_r.deq.valid)
+      in.r <> out.r
-      out_r.ready := in_r.ready && (!r_last || sideband_ar_r.deq.valid)
+      in.r.bits.last := out.r.bits.last && r_last
-      sideband_ar_r.deq.ready := r_last && out_r.valid && in_r.ready
+      in.r.bits.user.foreach { _ := out.r.bits.user.get >> 1 }
-      in_r.bits := out_r.bits
-      in_r.bits.last := r_last && sideband_ar_r.deq.bits
 
       // B flow control
-      val b_last = sideband_aw_b.deq.bits
+      val b_last = out.b.bits.user.get(0)
-      in_b.valid := out_b.valid && sideband_aw_b.deq.valid && b_last
+      in.b <> out.b
-      out_b.ready := sideband_aw_b.deq.valid && (!b_last || in_b.ready)
+      in.b.valid := out.b.valid && b_last
-      sideband_aw_b.deq.ready := out_b.valid && (!b_last || in_b.ready)
+      out.b.ready := in.b.ready || !b_last
-      in_b.bits := out_b.bits
+      in.b.bits.user.foreach { _ := out.b.bits.user.get >> 1 }
 
       // Merge errors from dropped B responses
-      val r_resp = RegInit(UInt(0, width = AXI4Parameters.respBits))
+      val error = RegInit(Vec.fill(edgeIn.master.endId) { UInt(0, width = AXI4Parameters.respBits)})
-      val resp = out_b.bits.resp | r_resp
+      in.b.bits.resp := out.b.bits.resp | error(out.b.bits.id)
-      when (out_b.fire()) { r_resp := Mux(b_last, UInt(0), resp) }
+      (error zip UIntToOH(out.b.bits.id, edgeIn.master.endId).toBools) foreach { case (reg, sel) =>
-      in_b.bits.resp := resp
+        when (sel && out.b.fire()) { reg := Mux(b_last, UInt(0), reg | out.b.bits.resp) }
-    }
-  }
-
-  /* We want to put barriers between the fragments of a fragmented transfer and all other transfers.
-   * This lets us use very little state to reassemble the fragments (else we need one FIFO per ID).
-   * Furthermore, because all the fragments share the same AXI ID, they come back contiguously.
-   * This guarantees that no other R responses might get mixed between fragments, ensuring that the
-   * interleavedId for the slaves remains unaffected by the fragmentation transformation.
-   * Of course, if you need to fragment, this means there is a potentially hefty serialization cost.
-   * However, this design allows full concurrency in the common no-fragmentation-needed scenario.
-   */
-  class AXI4FragmenterSideband(maxInFlight: Int, flow: Boolean = false) extends Module
-  {
-    val io = new QueueIO(Bool(), maxInFlight)
-    io.count := UInt(0)
-
-    val PASS = UInt(2, width = 2) // allow 'last=1' bits to enque, on 'last=0' if count>0 block else accept+FIND
-    val FIND = UInt(0, width = 2) // allow 'last=0' bits to enque, accept 'last=1' and switch to WAIT
-    val WAIT = UInt(1, width = 2) // block all access till count=0
-
-    val state = RegInit(PASS)
-    val count = RegInit(UInt(0, width = log2Up(maxInFlight)))
-    val full  = count === UInt(maxInFlight-1)
-    val empty = count === UInt(0)
-    val last  = count === UInt(1)
-
-    io.deq.bits := state(1) || (last && state(0)) // PASS || (last && WAIT)
-    io.deq.valid := !empty
-
-    io.enq.ready := !full && (empty || (state === FIND) || (state === PASS && io.enq.bits))
-
-    // WAIT => count > 0
-    assert (state =/= WAIT || count =/= UInt(0))
-
-    if (flow) {
-      when (io.enq.valid) {
-        io.deq.valid := Bool(true)
-        when (empty) { io.deq.bits := io.enq.bits }
       }
     }
-
-    count := count + io.enq.fire() - io.deq.fire()
-    switch (state) {
-      is(PASS) { when (io.enq.valid && !io.enq.bits && empty) { state := FIND } }
-      is(FIND) { when (io.enq.valid &&  io.enq.bits && !full) { state := Mux(empty, PASS, WAIT) } }
-      is(WAIT) { when (last && io.deq.ready)                  { state := PASS } }
-    }
   }
 }
 
 object AXI4Fragmenter
 {
   // applied to the AXI4 source node; y.node := AXI4Fragmenter()(x.node)
-  def apply(lite: Boolean = false, maxInFlight: => Int = 32, combinational: Boolean = true)(x: AXI4OutwardNode)(implicit p: Parameters, sourceInfo: SourceInfo): AXI4OutwardNode = {
+  def apply()(x: AXI4OutwardNode)(implicit p: Parameters, sourceInfo: SourceInfo): AXI4OutwardNode = {
-    val fragmenter = LazyModule(new AXI4Fragmenter(lite, maxInFlight, combinational))
+    val fragmenter = LazyModule(new AXI4Fragmenter)
     fragmenter.node := x
     fragmenter.node
   }