ahb: rewrote TLToAHB to avoid retracting requests on stall

src/main/scala/uncore/tilelink2/ToAHB.scala

@@ -25,12 +25,29 @@ case class TLToAHBNode() extends MixedAdapterNode(TLImp, AHBImp)(
         nodePath           = s.nodePath,
         supportsGet        = s.supportsRead,
         supportsPutFull    = s.supportsWrite, // but not PutPartial
-        fifoId             = Some(0)) // a common FIFO domain
+        fifoId             = Some(0))
     }
     TLManagerPortParameters(managers, beatBytes, 1, 1)
   })
 
-class TLToAHB(val combinational: Boolean = true)(implicit p: Parameters) extends LazyModule
+class AHBControlBundle(params: TLEdge) extends util.GenericParameterizedBundle(params)
+{
+  val full   = Bool()
+  val send   = Bool() // => full+data
+  val first  = Bool()
+  val last   = Bool()
+  val write  = Bool()
+  val size   = UInt(width = params.bundle.sizeBits)
+  val source = UInt(width = params.bundle.sourceBits)
+  val hsize  = UInt(width = AHBParameters.sizeBits)
+  val hburst = UInt(width = AHBParameters.burstBits)
+  val addr   = UInt(width = params.bundle.addressBits)
+  val data   = UInt(width = params.bundle.dataBits)
+}
+
+// The input side has either a flow queue (a_pipe=false) or a pipe queue (a_pipe=true)
+// The output side always has a flow queue
+class TLToAHB(val a_pipe: Boolean = true)(implicit p: Parameters) extends LazyModule
 {
   val node = TLToAHBNode()
 
@@ -46,82 +63,125 @@ class TLToAHB(val combinational: Boolean = true)(implicit p: Parameters) extends
       val lgMax = log2Ceil(maxTransfer)
       val lgBytes = log2Ceil(beatBytes)
 
-      // AHB has no cache coherence
+      // Initial FSM state
-      in.b.valid := Bool(false)
+      val resetState = Wire(new AHBControlBundle(edgeIn))
-      in.c.ready := Bool(true)
+      resetState.full  := Bool(false)
-      in.e.ready := Bool(true)
+      resetState.send  := Bool(false)
+      resetState.first := Bool(true)
+
+      // The stages of the combinational pipeline
+      val reg  = RegInit(resetState)
+      val send = Wire(init = reg)
+      val step = Wire(init = send)
+      val next = Wire(init = step)
+      reg := next
+
+      // Advance the FSM based on the result of this AHB beat
+      when (send.send && !out.hreadyout) /* retry AHB */ {
+        step.full  := Bool(true)
+        step.send  := Bool(true)
+      } .elsewhen (send.full && !send.send) /* retry beat */ {
+        step.full  := Bool(true)
+        step.send  := Bool(false)
+      } .elsewhen (send.full && !send.last) /* continue burst */ {
+        step.full  := Bool(true)
+        step.send  := Bool(false) // => looks like a retry to injector
+        step.first := Bool(false)
+        step.last  := (if (lgBytes + 1 >= lgMax) Bool(true) else
+                       !((UIntToOH1(send.size, lgMax) & ~send.addr) >> (lgBytes + 1)).orR())
+        step.addr  := Cat(send.addr(edgeIn.bundle.addressBits-1, lgMax), send.addr(lgMax-1, 0) + UInt(beatBytes))
+      } .otherwise /* new burst */ {
+        step.full  := Bool(false)
+        step.send  := Bool(false)
+        step.first := Bool(true)
+      }
+
+      val d_block = Wire(Bool())
+      val pre  = if (a_pipe) step else reg
+      val post = if (a_pipe) next else send
+
+      // Transform TL size into AHB hsize+hburst
+      val a_sizeDelta = Cat(UInt(0, width = 1), in.a.bits.size) - UInt(lgBytes+1)
+      val a_singleBeat = Bool(lgBytes >= lgMax) || a_sizeDelta(edgeIn.bundle.sizeBits)
+      val a_logBeats1 = a_sizeDelta(edgeIn.bundle.sizeBits-1, 0)
+
+      // Pulse this every time we commit to sending an AHB request
+      val a_commit = Wire(Bool())
+
+      // Inject A channel into FSM
+      when (pre.send) /* busy */ {
+        a_commit := Bool(false)
+        in.a.ready := Bool(false)
+      } .elsewhen (pre.full) /* retry beat (or continue burst) */ {
+        post.send  := !d_block && (!pre.write || in.a.valid)
+        post.data  := in.a.bits.data
+        a_commit   := !d_block && !pre.write // only read beats commit to a D beat answer
+        in.a.ready := !d_block && pre.write
+      } .otherwise /* new burst */ {
+        a_commit := in.a.fire() // every first beat commits to a D beat answer
+        in.a.ready := !d_block
+        when (in.a.fire()) {
+          post.full  := Bool(true)
+          post.send  := Bool(true)
+          post.last  := a_singleBeat
+          post.write := edgeIn.hasData(in.a.bits)
+          post.size  := in.a.bits.size
+          post.source:= in.a.bits.source
+          post.hsize := Mux(a_singleBeat, in.a.bits.size, UInt(lgBytes))
+          post.hburst:= Mux(a_singleBeat, BURST_SINGLE, (a_logBeats1<<1) | UInt(1))
+          post.addr  := in.a.bits.address
+          post.data  := in.a.bits.data
+        }
+      }
+
+      out.hmastlock := Bool(false) // for now
+      out.htrans    := Mux(send.send, Mux(send.first, TRANS_NONSEQ, TRANS_SEQ), Mux(send.first, TRANS_IDLE, TRANS_BUSY))
+      out.hsel      := send.send || !send.first
+      out.hready    := out.hreadyout
+      out.hwrite    := send.write
+      out.haddr     := send.addr
+      out.hsize     := send.hsize
+      out.hburst    := send.hburst
+      out.hprot     := PROT_DEFAULT
+      out.hwdata    := RegEnable(send.data, out.hreadyout)
 
       // We need a skidpad to capture D output:
       // We cannot know if the D response will be accepted until we have
       // presented it on D as valid.  We also can't back-pressure AHB in the
-      // data phase.  Therefore, we must have enough space to save the data
+      // data phase.  Therefore, we must have enough space to save the all
-      // phase result.  Whenever we have a queued response, we can not allow
+      // commited AHB requests (A+D phases = 2). To decouple d_ready from
-      // AHB to present new responses, so we must quash the address phase.
+      // a_ready and htrans, we add another entry for a_pipe=true.
+      val depth = if (a_pipe) 3 else 2
       val d = Wire(in.d)
-      in.d <> Queue(d, 1, flow = true)
+      in.d <> Queue(d, depth, flow=true)
-      val a_quash = in.d.valid && !in.d.ready
+      assert (!d.valid || d.ready)
+
+      val d_flight = RegInit(UInt(0, width = 2))
+      assert (d_flight <= UInt(depth))
+      d_flight := d_flight + a_commit.asUInt - in.d.fire().asUInt
+      d_block := d_flight >= UInt(depth)
 
-      // Record what is coming out in d_phase
       val d_valid   = RegInit(Bool(false))
-      val d_hasData = Reg(Bool())
       val d_error   = Reg(Bool())
-      val d_addr_lo = Reg(UInt(width = lgBytes))
+      val d_write   = RegEnable(send.write,  out.hreadyout)
-      val d_source  = Reg(UInt())
+      val d_source  = RegEnable(send.source, out.hreadyout)
-      val d_size    = Reg(UInt())
+      val d_addr    = RegEnable(send.addr,   out.hreadyout)
+      val d_size    = RegEnable(send.size,   out.hreadyout)
 
-      when (out.hreadyout) { d_error := d_error || out.hresp }
+      when (out.hreadyout) {
-      when (d.fire()) { d_valid := Bool(false) }
+        d_valid := send.send && (send.last || !send.write)
+        when (out.hresp)  { d_error := d_write }
+        when (send.first) { d_error := Bool(false) }
+      }
 
       d.valid := d_valid && out.hreadyout
-      d.bits  := edgeIn.AccessAck(d_addr_lo, UInt(0), d_source, d_size, out.hrdata, out.hresp || d_error)
+      d.bits  := edgeIn.AccessAck(d_addr, UInt(0), d_source, d_size, out.hrdata, out.hresp || d_error)
-      d.bits.opcode := Mux(d_hasData, TLMessages.AccessAckData, TLMessages.AccessAck)
+      d.bits.opcode := Mux(d_write, TLMessages.AccessAck, TLMessages.AccessAckData)
 
-      // We need an irrevocable input for AHB to stall on read bursts
+      // AHB has no cache coherence
-      // We also need the values to NOT change when valid goes low => 1 entry only
+      in.b.valid := Bool(false)
-      val a = Queue(in.a, 1, flow = combinational, pipe = !combinational)
+      in.c.ready := Bool(true)
-      val a_valid = a.valid && !a_quash
+      in.e.ready := Bool(true)
-
-      // This is lot like TLEdge.firstlast, but counts beats also for single-beat TL types
-      val a_size = edgeIn.size(a.bits)
-      val a_beats1 = UIntToOH1(a_size, lgMax) >> lgBytes
-      val a_counter = RegInit(UInt(0, width = log2Up(maxTransfer/beatBytes)))
-      val a_counter1 = a_counter - UInt(1)
-      val a_first = a_counter === UInt(0)
-      val a_last = a_counter === UInt(1) || a_beats1 === UInt(0)
-      val a_offset = (a_beats1 & ~a_counter1) << lgBytes
-      val a_hasData = edgeIn.hasData(a.bits)
-
-      // Expand no-data A-channel requests into multiple beats
-      a.ready := (a_hasData || a_last) && out.hreadyout && !a_quash
-      when (a_valid && out.hreadyout) {
-        a_counter := Mux(a_first, a_beats1, a_counter1)
-        d_valid := !a_hasData || a_last
-        // Record what will be in the data phase
-        when (a_first) {
-          d_hasData := !a_hasData
-          d_error   := Bool(false)
-          d_addr_lo := a.bits.address
-          d_source  := a.bits.source
-          d_size    := a.bits.size
-        }
-      }
-
-      // Transform TL size into AHB hsize+hburst
-      val a_size_bits = a_size.getWidth
-      val a_sizeDelta = Cat(UInt(0, width = 1), a_size) - UInt(lgBytes+1)
-      val a_singleBeat = a_sizeDelta(a_size_bits)
-      val a_logBeats1 = a_sizeDelta(a_size_bits-1, 0)
-
-      out.hmastlock := Bool(false) // for now
-      out.htrans    := Mux(a_valid, Mux(a_first, TRANS_NONSEQ, TRANS_SEQ), Mux(a_first, TRANS_IDLE, TRANS_BUSY))
-      out.hsel      := a_valid || !a_first
-      out.hready    := out.hreadyout
-      out.hwrite    := a_hasData
-      out.haddr     := a.bits.address | a_offset
-      out.hsize     := Mux(a_singleBeat, a.bits.size, UInt(lgBytes))
-      out.hburst    := Mux(a_singleBeat, BURST_SINGLE, (a_logBeats1<<1) | UInt(1))
-      out.hprot     := PROT_DEFAULT
-      out.hwdata    := RegEnable(a.bits.data, a.fire())
     }
   }
 }
@@ -129,8 +189,8 @@ class TLToAHB(val combinational: Boolean = true)(implicit p: Parameters) extends
 object TLToAHB
 {
   // applied to the TL source node; y.node := TLToAHB()(x.node)
-  def apply(combinational: Boolean = true)(x: TLOutwardNode)(implicit p: Parameters, sourceInfo: SourceInfo): AHBOutwardNode = {
+  def apply(a_pipe: Boolean = true)(x: TLOutwardNode)(implicit p: Parameters, sourceInfo: SourceInfo): AHBOutwardNode = {
-    val ahb = LazyModule(new TLToAHB(combinational))
+    val ahb = LazyModule(new TLToAHB(a_pipe))
     ahb.node := x
     ahb.node
   }