Merge pull request #396 from ucb-bar/decoupled

TL2 Decoupled

src/main/scala/regmapper/RegMapper.scala

@@ -31,7 +31,7 @@ object RegMapper
   // Create a generic register-based device
   def apply(bytes: Int, concurrency: Int, undefZero: Boolean, in: DecoupledIO[RegMapperInput], mapping: RegField.Map*) = {
     val bytemap = mapping.toList
-    // Don't be an asshole...
+    // Negative addresses are bad
     bytemap.foreach { byte => require (byte._1 >= 0) }
 
     // Transform all fields into bit offsets Seq[(bit, field)]

src/main/scala/uncore/tilelink2/Arbiter.scala

@@ -3,18 +3,17 @@
 package uncore.tilelink2
 
 import Chisel._
-import chisel3.util.IrrevocableIO
 import diplomacy._
 
 object TLArbiter
 {
-  // (valids, idle) => readys
+  // (valids, granted) => readys
   type Policy = (Seq[Bool], Bool) => Seq[Bool]
 
-  val lowestIndexFirst: Policy = (valids, idle) =>
+  val lowestIndexFirst: Policy = (valids, granted) =>
     valids.scanLeft(Bool(true))(_ && !_).init
 
-  def apply[T <: Data](policy: Policy)(sink: IrrevocableIO[T], sources: (UInt, IrrevocableIO[T])*) {
+  def apply[T <: Data](policy: Policy)(sink: DecoupledIO[T], sources: (UInt, DecoupledIO[T])*) {
     if (sources.isEmpty) {
       sink.valid := Bool(false)
     } else {
@@ -25,38 +24,33 @@ object TLArbiter
       // The number of beats which remain to be sent
       val beatsLeft = RegInit(UInt(0))
       val idle = beatsLeft === UInt(0)
+      val latch = idle && sink.ready // winner (if any) claims sink
 
       // Who wants access to the sink?
       val valids = sourcesIn.map(_.valid)
       // Arbitrate amongst the requests
-      val readys = Vec(policy(valids, idle))
+      val readys = Vec(policy(valids, latch))
       // Which request wins arbitration?
-      val winners = Vec((readys zip valids) map { case (r,v) => r&&v })
+      val winner = Vec((readys zip valids) map { case (r,v) => r&&v })
 
       // Confirm the policy works properly
       require (readys.size == valids.size)
-      // Never two winners
+      // Never two winner
-      val prefixOR = winners.scanLeft(Bool(false))(_||_).init
+      val prefixOR = winner.scanLeft(Bool(false))(_||_).init
-      assert((prefixOR zip winners) map { case (p,w) => !p || !w } reduce {_ && _})
+      assert((prefixOR zip winner) map { case (p,w) => !p || !w } reduce {_ && _})
       // If there was any request, there is a winner
-      assert (!valids.reduce(_||_) || winners.reduce(_||_))
+      assert (!valids.reduce(_||_) || winner.reduce(_||_))
 
       // Track remaining beats
-      val maskedBeats = (winners zip beatsIn) map { case (w,b) => Mux(w, b, UInt(0)) }
+      val maskedBeats = (winner zip beatsIn) map { case (w,b) => Mux(w, b, UInt(0)) }
       val initBeats = maskedBeats.reduce(_ | _) // no winner => 0 beats
-      val todoBeats = Mux(idle, initBeats, beatsLeft)
+      beatsLeft := Mux(latch, initBeats, beatsLeft - sink.fire())
-      beatsLeft := todoBeats - sink.fire()
-      assert (!sink.fire() || todoBeats =/= UInt(0)) // underflow is impoosible
 
       // The one-hot source granted access in the previous cycle
       val state = RegInit(Vec.fill(sources.size)(Bool(false)))
-      val muxState = Mux(idle, winners, state)
+      val muxState = Mux(idle, winner, state)
       state := muxState
 
-      val ones = Vec.fill(sources.size)(Bool(true))
-      val picked = Mux(idle, ones, state)
-      sink.valid := Mux1H(picked, valids)
-
       if (sources.size > 1) {
         val allowed = Mux(idle, readys, state)
         (sourcesIn zip allowed) foreach { case (s, r) =>
@@ -66,6 +60,7 @@ object TLArbiter
         sourcesIn(0).ready := sink.ready
       }
 
+      sink.valid := Mux(idle, valids.reduce(_||_), Mux1H(state, valids))
       sink.bits := Mux1H(muxState, sourcesIn.map(_.bits))
     }
   }

src/main/scala/uncore/tilelink2/AtomicAutomata.scala

@@ -175,7 +175,7 @@ class TLAtomicAutomata(logical: Boolean = true, arithmetic: Boolean = true, conc
       source_c.bits := edgeOut.Put(a_cam_a.bits.source, edgeIn.address(a_cam_a.bits), a_cam_a.bits.size, amo_data)._2
 
       // Finishing an AMO from the CAM has highest priority
-      TLArbiter(TLArbiter.lowestIndexFirst)(out.a, (UInt(1), source_c), (edgeOut.numBeats(in.a.bits), source_i))
+      TLArbiter(TLArbiter.lowestIndexFirst)(out.a, (UInt(0), source_c), (edgeOut.numBeats1(in.a.bits), source_i))
 
       // Capture the A state into the CAM
       when (source_i.fire() && !a_isSupported) {

src/main/scala/uncore/tilelink2/Buffer.scala

@@ -5,7 +5,7 @@ package uncore.tilelink2
 import Chisel._
 import chisel3.internal.sourceinfo.SourceInfo
 import diplomacy._
-import scala.math.max
+import scala.math.{min,max}
 
 // pipe is only used if a queue has depth = 1
 class TLBuffer(a: Int = 2, b: Int = 2, c: Int = 2, d: Int = 2, e: Int = 2, pipe: Boolean = true) extends LazyModule
@@ -17,8 +17,8 @@ class TLBuffer(a: Int = 2, b: Int = 2, c: Int = 2, d: Int = 2, e: Int = 2, pipe:
   require (e >= 0)
 
   val node = TLAdapterNode(
-    clientFn  = { seq => seq(0).copy(minLatency = seq(0).minLatency + max(1,b) + max(1,c)) },
+    clientFn  = { seq => seq(0).copy(minLatency = seq(0).minLatency + min(1,b) + min(1,c)) },
-    managerFn = { seq => seq(0).copy(minLatency = seq(0).minLatency + max(1,a) + max(1,d)) })
+    managerFn = { seq => seq(0).copy(minLatency = seq(0).minLatency + min(1,a) + min(1,d)) })
 
   lazy val module = new LazyModuleImp(this) {
     val io = new Bundle {

src/main/scala/uncore/tilelink2/Bundles.scala

@@ -3,7 +3,7 @@
 package uncore.tilelink2
 
 import Chisel._
-import chisel3.util.{Irrevocable, IrrevocableIO, ReadyValidIO}
+import chisel3.util.{ReadyValidIO}
 import diplomacy._
 import util.{AsyncQueueSource, AsyncQueueSink, GenericParameterizedBundle}
 
@@ -165,11 +165,11 @@ final class TLBundleE(params: TLBundleParameters)
 
 class TLBundle(params: TLBundleParameters) extends TLBundleBase(params)
 {
-  val a = Irrevocable(new TLBundleA(params))
+  val a = Decoupled(new TLBundleA(params))
-  val b = Irrevocable(new TLBundleB(params)).flip
+  val b = Decoupled(new TLBundleB(params)).flip
-  val c = Irrevocable(new TLBundleC(params))
+  val c = Decoupled(new TLBundleC(params))
-  val d = Irrevocable(new TLBundleD(params)).flip
+  val d = Decoupled(new TLBundleD(params)).flip
-  val e = Irrevocable(new TLBundleE(params))
+  val e = Decoupled(new TLBundleE(params))
 }
 
 object TLBundle
@@ -177,20 +177,20 @@ object TLBundle
   def apply(params: TLBundleParameters) = new TLBundle(params)
 }
 
-final class IrrevocableSnoop[+T <: Data](gen: T) extends Bundle
+final class DecoupledSnoop[+T <: Data](gen: T) extends Bundle
 {
   val ready = Bool()
   val valid = Bool()
   val bits = gen.asOutput
 
   def fire(dummy: Int = 0) = ready && valid
-  override def cloneType: this.type = new IrrevocableSnoop(gen).asInstanceOf[this.type]
+  override def cloneType: this.type = new DecoupledSnoop(gen).asInstanceOf[this.type]
 }
 
-object IrrevocableSnoop
+object DecoupledSnoop
 {
-  def apply[T <: Data](i: IrrevocableIO[T]) = {
+  def apply[T <: Data](i: DecoupledIO[T]) = {
-    val out = Wire(new IrrevocableSnoop(i.bits))
+    val out = Wire(new DecoupledSnoop(i.bits))
     out.ready := i.ready
     out.valid := i.valid
     out.bits  := i.bits
@@ -200,22 +200,22 @@ object IrrevocableSnoop
 
 class TLBundleSnoop(params: TLBundleParameters) extends TLBundleBase(params)
 {
-  val a = new IrrevocableSnoop(new TLBundleA(params))
+  val a = new DecoupledSnoop(new TLBundleA(params))
-  val b = new IrrevocableSnoop(new TLBundleB(params))
+  val b = new DecoupledSnoop(new TLBundleB(params))
-  val c = new IrrevocableSnoop(new TLBundleC(params))
+  val c = new DecoupledSnoop(new TLBundleC(params))
-  val d = new IrrevocableSnoop(new TLBundleD(params))
+  val d = new DecoupledSnoop(new TLBundleD(params))
-  val e = new IrrevocableSnoop(new TLBundleE(params))
+  val e = new DecoupledSnoop(new TLBundleE(params))
 }
 
 object TLBundleSnoop
 {
   def apply(x: TLBundle) = {
     val out = Wire(new TLBundleSnoop(x.params))
-    out.a <> IrrevocableSnoop(x.a)
+    out.a <> DecoupledSnoop(x.a)
-    out.b <> IrrevocableSnoop(x.b)
+    out.b <> DecoupledSnoop(x.b)
-    out.c <> IrrevocableSnoop(x.c)
+    out.c <> DecoupledSnoop(x.c)
-    out.d <> IrrevocableSnoop(x.d)
+    out.d <> DecoupledSnoop(x.d)
-    out.e <> IrrevocableSnoop(x.e)
+    out.e <> DecoupledSnoop(x.e)
     out
   }
 }
@@ -234,14 +234,14 @@ final class AsyncBundle[T <: Data](val depth: Int, gen: T) extends Bundle
 
 object FromAsyncBundle
 {
-  def apply[T <: Data](x: AsyncBundle[T], sync: Int = 3): IrrevocableIO[T] = {
+  def apply[T <: Data](x: AsyncBundle[T], sync: Int = 3): DecoupledIO[T] = {
     val sink = Module(new AsyncQueueSink(x.mem(0), x.depth, sync))
     x.ridx := sink.io.ridx
     sink.io.widx := x.widx
     sink.io.mem  := x.mem
     sink.io.source_reset_n := x.source_reset_n
     x.sink_reset_n := !sink.reset
-    val out = Wire(Irrevocable(x.mem(0)))
+    val out = Wire(Decoupled(x.mem(0)))
     out.valid := sink.io.deq.valid
     out.bits := sink.io.deq.bits
     sink.io.deq.ready := out.ready

src/main/scala/uncore/tilelink2/Edges.scala

@@ -4,7 +4,6 @@ package uncore.tilelink2
 
 import Chisel._
 import chisel3.internal.sourceinfo.SourceInfo
-import chisel3.util.IrrevocableIO
 import diplomacy._
 
 class TLEdge(
@@ -232,7 +231,7 @@ class TLEdge(
     (first, last, beats1 & ~counter1)
   }
 
-  def firstlast(x: IrrevocableIO[TLChannel]): (Bool, Bool, UInt) = firstlast(x.bits, x.fire())
+  def firstlast(x: DecoupledIO[TLChannel]): (Bool, Bool, UInt) = firstlast(x.bits, x.fire())
 }
 
 class TLEdgeOut(

src/main/scala/uncore/tilelink2/Fragmenter.scala

@@ -39,10 +39,9 @@ class TLFragmenter(minSize: Int, maxSize: Int, alwaysMin: Boolean = false) exten
   def mapClient(c: TLClientParameters) = c.copy(
     sourceId = IdRange(c.sourceId.start << fragmentBits, c.sourceId.end << fragmentBits))
 
-  // Because the Fragmenter stalls inner A while serving outer, it can wipe away inner latency
   val node = TLAdapterNode(
     clientFn  = { case Seq(c) => c.copy(clients = c.clients.map(mapClient)) },
-    managerFn = { case Seq(m) => m.copy(managers = m.managers.map(mapManager), minLatency = 0) })
+    managerFn = { case Seq(m) => m.copy(managers = m.managers.map(mapManager)) })
 
   lazy val module = new LazyModuleImp(this) {
     val io = new Bundle {
@@ -190,8 +189,13 @@ class TLFragmenter(minSize: Int, maxSize: Int, alwaysMin: Boolean = false) exten
     val maxLgPutPartials = maxPutPartials.map(m => if (m == 0) lgMinSize else UInt(log2Ceil(m)))
     val maxLgHints       = maxHints      .map(m => if (m == 0) lgMinSize else UInt(log2Ceil(m)))
 
+    // Make the request repeatable
+    val repeater = Module(new Repeater(in.a.bits))
+    repeater.io.enq <> in.a
+    val in_a = repeater.io.deq
+
     // If this is infront of a single manager, these become constants
-    val find = manager.findFast(edgeIn.address(in.a.bits))
+    val find = manager.findFast(edgeIn.address(in_a.bits))
     val maxLgArithmetic  = Mux1H(find, maxLgArithmetics)
     val maxLgLogical     = Mux1H(find, maxLgLogicals)
     val maxLgGet         = Mux1H(find, maxLgGets)
@@ -200,7 +204,7 @@ class TLFragmenter(minSize: Int, maxSize: Int, alwaysMin: Boolean = false) exten
     val maxLgHint        = Mux1H(find, maxLgHints)
 
     val limit = if (alwaysMin) lgMinSize else 
-      MuxLookup(in.a.bits.opcode, lgMinSize, Array(
+      MuxLookup(in_a.bits.opcode, lgMinSize, Array(
         TLMessages.PutFullData    -> maxLgPutFull,
         TLMessages.PutPartialData -> maxLgPutPartial,
         TLMessages.ArithmeticData -> maxLgArithmetic,
@@ -208,11 +212,11 @@ class TLFragmenter(minSize: Int, maxSize: Int, alwaysMin: Boolean = false) exten
         TLMessages.Get            -> maxLgGet,
         TLMessages.Hint           -> maxLgHint))
 
-    val aOrig = in.a.bits.size
+    val aOrig = in_a.bits.size
     val aFrag = Mux(aOrig > limit, limit, aOrig)
     val aOrigOH1 = UIntToOH1(aOrig, log2Ceil(maxSize))
     val aFragOH1 = UIntToOH1(aFrag, log2Up(maxDownSize))
-    val aHasData = node.edgesIn(0).hasData(in.a.bits)
+    val aHasData = node.edgesIn(0).hasData(in_a.bits)
     val aMask = Mux(aHasData, UInt(0), aFragOH1)
 
     val gennum = RegInit(UInt(0, width = counterBits))
@@ -223,14 +227,19 @@ class TLFragmenter(minSize: Int, maxSize: Int, alwaysMin: Boolean = false) exten
 
     when (out.a.fire()) { gennum := new_gennum }
 
-    val delay = !aHasData && aFragnum =/= UInt(0)
+    repeater.io.repeat := !aHasData && aFragnum =/= UInt(0)
-    out.a.valid := in.a.valid
+    out.a <> in_a
-    in.a.ready := out.a.ready && !delay
+    out.a.bits.addr_hi := in_a.bits.addr_hi | (~aFragnum << log2Ceil(minSize/beatBytes) & aOrigOH1 >> log2Ceil(beatBytes))
-    out.a.bits := in.a.bits
+    out.a.bits.source := Cat(in_a.bits.source, aFragnum)
-    out.a.bits.addr_hi := in.a.bits.addr_hi | (~aFragnum << log2Ceil(minSize/beatBytes) & aOrigOH1 >> log2Ceil(beatBytes))
-    out.a.bits.source := Cat(in.a.bits.source, aFragnum)
     out.a.bits.size := aFrag
 
+    // Optimize away some of the Repeater's registers
+    assert (!repeater.io.full || !aHasData)
+    out.a.bits.data := in.a.bits.data
+    val fullMask = UInt((BigInt(1) << beatBytes) - 1)
+    assert (!repeater.io.full || in_a.bits.mask === fullMask)
+    out.a.bits.mask := Mux(repeater.io.full, fullMask, in.a.bits.mask)
+
     // Tie off unused channels
     in.b.valid := Bool(false)
     in.c.ready := Bool(true)

src/main/scala/uncore/tilelink2/HintHandler.scala

@@ -46,7 +46,7 @@ class TLHintHandler(supportManagers: Boolean = true, supportClients: Boolean = f
       hint.bits := edgeIn.HintAck(in.a.bits, edgeOut.manager.findIdStartFast(address))
       out.a.bits := in.a.bits
 
-      TLArbiter(TLArbiter.lowestIndexFirst)(in.d, (edgeOut.numBeats(out.d.bits), out.d), (UInt(1), hint))
+      TLArbiter(TLArbiter.lowestIndexFirst)(in.d, (edgeOut.numBeats1(out.d.bits), out.d), (UInt(0), hint))
     } else {
       out.a.valid := in.a.valid
       in.a.ready := out.a.ready
@@ -69,7 +69,7 @@ class TLHintHandler(supportManagers: Boolean = true, supportClients: Boolean = f
       hint.bits := edgeOut.HintAck(out.b.bits)
       in.b.bits := out.b.bits
 
-      TLArbiter(TLArbiter.lowestIndexFirst)(out.c, (edgeIn.numBeats(in.c.bits), in.c), (UInt(1), hint))
+      TLArbiter(TLArbiter.lowestIndexFirst)(out.c, (edgeIn.numBeats1(in.c.bits), in.c), (UInt(0), hint))
     } else if (bce) {
       in.b.valid := out.b.valid
       out.b.ready := in.b.ready

src/main/scala/uncore/tilelink2/Monitor.scala

@@ -281,7 +281,7 @@ class TLMonitor(gen: () => TLBundleSnoop, edge: () => TLEdge, sourceInfo: Source
     when (bundle.e.valid) { legalizeFormatE(bundle.e.bits, edge) }
   }
 
-  def legalizeMultibeatA(a: IrrevocableSnoop[TLBundleA], edge: TLEdge)(implicit sourceInfo: SourceInfo) {
+  def legalizeMultibeatA(a: DecoupledSnoop[TLBundleA], edge: TLEdge)(implicit sourceInfo: SourceInfo) {
     val (a_first, _, _) = edge.firstlast(a.bits, a.fire())
     val opcode  = Reg(UInt())
     val param   = Reg(UInt())
@@ -304,7 +304,7 @@ class TLMonitor(gen: () => TLBundleSnoop, edge: () => TLEdge, sourceInfo: Source
     }
   }
 
-  def legalizeMultibeatB(b: IrrevocableSnoop[TLBundleB], edge: TLEdge)(implicit sourceInfo: SourceInfo) {
+  def legalizeMultibeatB(b: DecoupledSnoop[TLBundleB], edge: TLEdge)(implicit sourceInfo: SourceInfo) {
     val (b_first, _, _) = edge.firstlast(b.bits, b.fire())
     val opcode  = Reg(UInt())
     val param   = Reg(UInt())
@@ -327,7 +327,7 @@ class TLMonitor(gen: () => TLBundleSnoop, edge: () => TLEdge, sourceInfo: Source
     }
   }
 
-  def legalizeMultibeatC(c: IrrevocableSnoop[TLBundleC], edge: TLEdge)(implicit sourceInfo: SourceInfo) {
+  def legalizeMultibeatC(c: DecoupledSnoop[TLBundleC], edge: TLEdge)(implicit sourceInfo: SourceInfo) {
     val (c_first, _, _) = edge.firstlast(c.bits, c.fire())
     val opcode  = Reg(UInt())
     val param   = Reg(UInt())
@@ -353,7 +353,7 @@ class TLMonitor(gen: () => TLBundleSnoop, edge: () => TLEdge, sourceInfo: Source
     }
   }
 
-  def legalizeMultibeatD(d: IrrevocableSnoop[TLBundleD], edge: TLEdge)(implicit sourceInfo: SourceInfo) {
+  def legalizeMultibeatD(d: DecoupledSnoop[TLBundleD], edge: TLEdge)(implicit sourceInfo: SourceInfo) {
     val (d_first, _, _) = edge.firstlast(d.bits, d.fire())
     val opcode  = Reg(UInt())
     val param   = Reg(UInt())
@@ -386,50 +386,26 @@ class TLMonitor(gen: () => TLBundleSnoop, edge: () => TLEdge, sourceInfo: Source
     legalizeMultibeatD(bundle.d, edge)
   }
 
-  def legalizeIrrevocable(irr: IrrevocableSnoop[TLChannel], edge: TLEdge)(implicit sourceInfo: SourceInfo) {
-    val last_v = RegNext(irr.valid, Bool(false))
-    val last_r = RegNext(irr.ready, Bool(false))
-    val last_b = RegNext(irr.bits)
-    val bits_changed = irr.bits.asUInt === last_b.asUInt
-
-    when (last_v && !last_r) {
-      assert(irr.valid,    s"${irr.bits.channelName} had contents that were revoked by the supplier (valid lowered)" + extra)
-      assert(bits_changed, s"${irr.bits.channelName} had contents that were revoked by the supplier (contents changed)" + extra)
-    }
-  }
-
-  def legalizeIrrevocable(bundle: TLBundleSnoop, edge: TLEdge)(implicit sourceInfo: SourceInfo) {
-    legalizeIrrevocable(bundle.a, edge)
-    legalizeIrrevocable(bundle.b, edge)
-    legalizeIrrevocable(bundle.c, edge)
-    legalizeIrrevocable(bundle.d, edge)
-    legalizeIrrevocable(bundle.e, edge)
-  }
-
   def legalizeSourceUnique(bundle: TLBundleSnoop, edge: TLEdge)(implicit sourceInfo: SourceInfo) {
     val inflight = RegInit(UInt(0, width = edge.client.endSourceId))
 
     val (_, a_last, _) = edge.firstlast(bundle.a.bits, bundle.a.fire())
     val (_, d_last, _) = edge.firstlast(bundle.d.bits, bundle.d.fire())
 
-    val bypass = bundle.a.bits.source === bundle.d.bits.source
-    val a_bypass = bypass && bundle.d.valid && d_last
-    val d_bypass = bypass && bundle.a.valid && a_last
-
     if (edge.manager.minLatency > 0) {
-      assert(!bypass || !bundle.a.valid || !bundle.d.valid, s"'A' and 'D' concurrent, despite minlatency ${edge.manager.minLatency}" + extra)
+      assert(bundle.a.bits.source =/= bundle.d.bits.source || !bundle.a.valid || !bundle.d.valid, s"'A' and 'D' concurrent, despite minlatency ${edge.manager.minLatency}" + extra)
     }
 
     val a_set = Wire(init = UInt(0, width = edge.client.endSourceId))
     when (bundle.a.fire()) {
       when (a_last) { a_set := UIntToOH(bundle.a.bits.source) }
-      assert(a_bypass || !inflight(bundle.a.bits.source), "'A' channel re-used a source ID" + extra)
+      assert(!inflight(bundle.a.bits.source), "'A' channel re-used a source ID" + extra)
     }
 
     val d_clr = Wire(init = UInt(0, width = edge.client.endSourceId))
     when (bundle.d.fire() && bundle.d.bits.opcode =/= TLMessages.ReleaseAck) {
       when (d_last) { d_clr := UIntToOH(bundle.d.bits.source) }
-      assert(d_bypass || inflight(bundle.d.bits.source), "'D' channel acknowledged for nothing inflight" + extra)
+      assert((a_set | inflight)(bundle.d.bits.source), "'D' channel acknowledged for nothing inflight" + extra)
     }
 
     inflight := (inflight | a_set) & ~d_clr
@@ -438,7 +414,6 @@ class TLMonitor(gen: () => TLBundleSnoop, edge: () => TLEdge, sourceInfo: Source
   def legalize(bundle: TLBundleSnoop, edge: TLEdge)(implicit sourceInfo: SourceInfo) {
     legalizeFormat     (bundle, edge)
     legalizeMultibeat  (bundle, edge)
-    legalizeIrrevocable(bundle, edge)
     legalizeSourceUnique(bundle, edge)
   }
 

src/main/scala/uncore/tilelink2/Nodes.scala

@@ -21,6 +21,9 @@ object TLImp extends NodeImp[TLClientPortParameters, TLManagerPortParameters, TL
   }
 
   var emitMonitors = true
+  var stressTestDecoupled = false
+  var combinationalCheck = false
+
   def colour = "#000000" // black
   def connect(bo: => TLBundle, bi: => TLBundle, ei: => TLEdgeIn)(implicit sourceInfo: SourceInfo): (Option[LazyModule], () => Unit) = {
     val monitor = if (emitMonitors) {
@@ -31,6 +34,40 @@ object TLImp extends NodeImp[TLClientPortParameters, TLManagerPortParameters, TL
     (monitor, () => {
       bi <> bo
       monitor.foreach { _.module.io.in := TLBundleSnoop(bo) }
+      if (combinationalCheck) {
+        // It is forbidden for valid to depend on ready in TL2
+        // If someone did that, then this will create a detectable combinational loop
+        bo.a.ready := bi.a.ready && bo.a.valid
+        bi.b.ready := bo.b.ready && bi.b.valid
+        bo.c.ready := bi.c.ready && bo.c.valid
+        bi.d.ready := bo.d.ready && bi.d.valid
+        bo.e.ready := bi.e.ready && bo.e.valid
+      }
+      if (stressTestDecoupled) {
+        // Randomly stall the transfers
+        val allow = LFSRNoiseMaker(5)
+        bi.a.valid := bo.a.valid && allow(0)
+        bo.a.ready := bi.a.ready && allow(0)
+        bo.b.valid := bi.b.valid && allow(1)
+        bi.b.ready := bo.b.ready && allow(1)
+        bi.c.valid := bo.c.valid && allow(2)
+        bo.c.ready := bi.c.ready && allow(2)
+        bo.d.valid := bi.d.valid && allow(3)
+        bi.d.ready := bo.d.ready && allow(3)
+        bi.e.valid := bo.e.valid && allow(4)
+        bo.e.ready := bi.e.ready && allow(4)
+        // Inject garbage whenever not valid
+        val bits_a = bo.a.bits.fromBits(LFSRNoiseMaker(bo.a.bits.asUInt.getWidth))
+        val bits_b = bi.b.bits.fromBits(LFSRNoiseMaker(bi.b.bits.asUInt.getWidth))
+        val bits_c = bo.c.bits.fromBits(LFSRNoiseMaker(bo.c.bits.asUInt.getWidth))
+        val bits_d = bi.d.bits.fromBits(LFSRNoiseMaker(bi.d.bits.asUInt.getWidth))
+        val bits_e = bo.e.bits.fromBits(LFSRNoiseMaker(bo.e.bits.asUInt.getWidth))
+        when (!bi.a.valid) { bi.a.bits := bits_a }
+        when (!bo.b.valid) { bo.b.bits := bits_b }
+        when (!bi.c.valid) { bi.c.bits := bits_c }
+        when (!bo.d.valid) { bo.d.bits := bits_d }
+        when (!bi.e.valid) { bi.e.bits := bits_e }
+      }
     })
   }
 

src/main/scala/uncore/tilelink2/RAMModel.scala

@@ -103,7 +103,7 @@ class TLRAMModel(log: String = "") extends LazyModule
     a_flight.size   := edge.size(in.a.bits)
     a_flight.opcode := in.a.bits.opcode
 
-    flight(in.a.bits.source) := a_flight
+    when (in.a.fire()) { flight(in.a.bits.source) := a_flight }
     val bypass = if (edge.manager.minLatency > 0) Bool(false) else in.a.valid && in.a.bits.source === out.d.bits.source
     val d_flight = RegNext(Mux(bypass, a_flight, flight(out.d.bits.source)))
 

src/main/scala/uncore/tilelink2/RegisterRouter.scala

@@ -41,10 +41,8 @@ class TLRegisterNode(address: AddressSet, concurrency: Int = 0, beatBytes: Int =
     in.bits.mask  := a.bits.mask
     in.bits.extra := Cat(edge.addr_lo(a.bits), a.bits.source, a.bits.size)
 
-    // Invoke the register map builder and make it Irrevocable
+    // Invoke the register map builder
-    val out = Queue.irrevocable(
+    val out = RegMapper(beatBytes, concurrency, undefZero, in, mapping:_*)
-      RegMapper(beatBytes, concurrency, undefZero, in, mapping:_*),
-      entries = 1, flow = true)
 
     // No flow control needed
     in.valid  := a.valid

src/main/scala/uncore/tilelink2/Repeater.scala

@@ -0,0 +1,39 @@
+// See LICENSE for license details.
+
+package uncore.tilelink2
+
+import Chisel._
+
+// A Repeater passes it's input to it's output, unless repeat is asserted.
+// When repeat is asserted, the Repeater copies the input and repeats it next cycle.
+class Repeater[T <: Data](gen: T) extends Module
+{
+  val io = new Bundle {
+    val repeat = Bool(INPUT)
+    val full = Bool(OUTPUT)
+    val enq = Decoupled(gen).flip
+    val deq = Decoupled(gen)
+  }
+
+  val full = RegInit(Bool(false))
+  val saved = Reg(gen)
+
+  // When !full, a repeater is pass-through
+  io.deq.valid := io.enq.valid || full
+  io.enq.ready := io.deq.ready && !full
+  io.deq.bits := Mux(full, saved, io.enq.bits)
+  io.full := full
+
+  when (io.enq.fire() &&  io.repeat) { full := Bool(true); saved := io.enq.bits }
+  when (io.deq.fire() && !io.repeat) { full := Bool(false) }
+}
+
+object Repeater
+{
+  def apply[T <: Data](enq: DecoupledIO[T], repeat: Bool): DecoupledIO[T] = {
+    val repeater = Module(new Repeater(enq.bits))
+    repeater.io.repeat := repeat
+    repeater.io.enq := enq
+    repeater.io.deq
+  }
+}

src/main/scala/uncore/tilelink2/ToAXI4.scala

@@ -221,12 +221,9 @@ class TLToAXI4(idBits: Int, combinational: Boolean = true) extends LazyModule
     // Give R higher priority than B
     val r_wins = out.r.valid || r_holds_d
 
-    val in_d = Wire(in.d)
+    out.r.ready := in.d.ready
-    in.d <> Queue.irrevocable(in_d, entries=1, flow=combinational)
+    out_b.ready := in.d.ready && !r_wins
-
+    in.d.valid := Mux(r_wins, out.r.valid, out_b.valid)
-    out.r.ready := in_d.ready
-    out_b.ready := in_d.ready && !r_wins
-    in_d.valid := Mux(r_wins, out.r.valid, out_b.valid)
 
     val r_error = out.r.bits.resp =/= AXI4Parameters.RESP_OKAY
     val b_error = out_b.bits.resp =/= AXI4Parameters.RESP_OKAY
@@ -234,8 +231,8 @@ class TLToAXI4(idBits: Int, combinational: Boolean = true) extends LazyModule
     val r_d = edgeIn.AccessAck(r_addr_lo, r_sink, r_source, r_size, UInt(0), r_error)
     val b_d = edgeIn.AccessAck(b_addr_lo, b_sink, b_source, b_size, b_error)
 
-    in_d.bits := Mux(r_wins, r_d, b_d)
+    in.d.bits := Mux(r_wins, r_d, b_d)
-    in_d.bits.data := out.r.bits.data // avoid a costly Mux
+    in.d.bits.data := out.r.bits.data // avoid a costly Mux
 
     // Tie off unused channels
     in.b.valid := Bool(false)

src/main/scala/uncore/tilelink2/WidthWidget.scala

@@ -4,17 +4,15 @@ package uncore.tilelink2
 
 import Chisel._
 import chisel3.internal.sourceinfo.SourceInfo
-import chisel3.util.{Irrevocable, IrrevocableIO}
 import diplomacy._
 import scala.math.{min,max}
 
 // innBeatBytes => the new client-facing bus width
 class TLWidthWidget(innerBeatBytes: Int) extends LazyModule
 {
-  // Because we stall the request while sending beats, atomics can overlap => minLatency=0
   val node = TLAdapterNode(
-    clientFn  = { case Seq(c) => c.copy(minLatency = 0) },
+    clientFn  = { case Seq(c) => c },
-    managerFn = { case Seq(m) => m.copy(minLatency = 0, beatBytes = innerBeatBytes) })
+    managerFn = { case Seq(m) => m.copy(beatBytes = innerBeatBytes) })
 
   lazy val module = new LazyModuleImp(this) {
     val io = new Bundle {
@@ -22,7 +20,7 @@ class TLWidthWidget(innerBeatBytes: Int) extends LazyModule
       val out = node.bundleOut
     }
 
-    def merge[T <: TLDataChannel](edgeIn: TLEdge, in: IrrevocableIO[T], edgeOut: TLEdge, out: IrrevocableIO[T]) = {
+    def merge[T <: TLDataChannel](edgeIn: TLEdge, in: DecoupledIO[T], edgeOut: TLEdge, out: DecoupledIO[T]) = {
       val inBytes = edgeIn.manager.beatBytes
       val outBytes = edgeOut.manager.beatBytes
       val ratio = outBytes / inBytes
@@ -86,7 +84,7 @@ class TLWidthWidget(innerBeatBytes: Int) extends LazyModule
       }
     }
 
-    def split[T <: TLDataChannel](edgeIn: TLEdge, in: IrrevocableIO[T], edgeOut: TLEdge, out: IrrevocableIO[T]) = {
+    def split[T <: TLDataChannel](edgeIn: TLEdge, in: DecoupledIO[T], edgeOut: TLEdge, out: DecoupledIO[T]) = {
       val inBytes = edgeIn.manager.beatBytes
       val outBytes = edgeOut.manager.beatBytes
       val ratio = inBytes / outBytes
@@ -121,9 +119,7 @@ class TLWidthWidget(innerBeatBytes: Int) extends LazyModule
       val dataOut = if (edgeIn.staticHasData(in.bits) == Some(false)) UInt(0) else Mux1H(select, dataSlices)
       val maskOut = Mux1H(select, maskSlices)
 
-      in.ready := out.ready && last
+      out <> in
-      out.valid := in.valid
-      out.bits := in.bits
       edgeOut.data(out.bits) := dataOut
 
       out.bits match {
@@ -134,15 +130,19 @@ class TLWidthWidget(innerBeatBytes: Int) extends LazyModule
       }
 
       // addr_lo gets truncated automagically
+
+      // Repeat the input if we're not last
+      !last
     }
     
-    def splice[T <: TLDataChannel](edgeIn: TLEdge, in: IrrevocableIO[T], edgeOut: TLEdge, out: IrrevocableIO[T]) = {
+    def splice[T <: TLDataChannel](edgeIn: TLEdge, in: DecoupledIO[T], edgeOut: TLEdge, out: DecoupledIO[T]) = {
       if (edgeIn.manager.beatBytes == edgeOut.manager.beatBytes) {
         // nothing to do; pass it through
         out <> in
       } else if (edgeIn.manager.beatBytes > edgeOut.manager.beatBytes) {
         // split input to output
-        split(edgeIn, in, edgeOut, out)
+        val repeat = Wire(Bool())
+        repeat := split(edgeIn, Repeater(in, repeat), edgeOut, out)
       } else {
         // merge input to output
         merge(edgeIn, in, edgeOut, out)

src/main/scala/uncore/tilelink2/Xbar.scala

@@ -3,7 +3,6 @@
 package uncore.tilelink2
 
 import Chisel._
-import chisel3.util.IrrevocableIO
 import diplomacy._
 
 class TLXbar(policy: TLArbiter.Policy = TLArbiter.lowestIndexFirst) extends LazyModule
@@ -130,18 +129,18 @@ class TLXbar(policy: TLArbiter.Policy = TLArbiter.lowestIndexFirst) extends Lazy
     val requestDOI = Vec(out.map { o => Vec(inputIdRanges.map  { i => i.contains(o.d.bits.source) }) })
     val requestEIO = Vec(in.map  { i => Vec(outputIdRanges.map { o => o.contains(i.e.bits.sink)   }) })
 
-    val beatsAI = Vec((in  zip node.edgesIn)  map { case (i, e) => e.numBeats(i.a.bits) })
+    val beatsAI = Vec((in  zip node.edgesIn)  map { case (i, e) => e.numBeats1(i.a.bits) })
-    val beatsBO = Vec((out zip node.edgesOut) map { case (o, e) => e.numBeats(o.b.bits) })
+    val beatsBO = Vec((out zip node.edgesOut) map { case (o, e) => e.numBeats1(o.b.bits) })
-    val beatsCI = Vec((in  zip node.edgesIn)  map { case (i, e) => e.numBeats(i.c.bits) })
+    val beatsCI = Vec((in  zip node.edgesIn)  map { case (i, e) => e.numBeats1(i.c.bits) })
-    val beatsDO = Vec((out zip node.edgesOut) map { case (o, e) => e.numBeats(o.d.bits) })
+    val beatsDO = Vec((out zip node.edgesOut) map { case (o, e) => e.numBeats1(o.d.bits) })
-    val beatsEI = Vec((in  zip node.edgesIn)  map { case (i, e) => e.numBeats(i.e.bits) })
+    val beatsEI = Vec((in  zip node.edgesIn)  map { case (i, e) => e.numBeats1(i.e.bits) })
 
     // Which pairs support support transfers
     def transpose[T](x: Seq[Seq[T]]) = Seq.tabulate(x(0).size) { i => Seq.tabulate(x.size) { j => x(j)(i) } }
     def filter[T](data: Seq[T], mask: Seq[Boolean]) = (data zip mask).filter(_._2).map(_._1)
 
     // Replicate an input port to each output port
-    def fanout[T <: TLChannel](input: IrrevocableIO[T], select: Seq[Bool]) = {
+    def fanout[T <: TLChannel](input: DecoupledIO[T], select: Seq[Bool]) = {
       val filtered = Wire(Vec(select.size, input))
       for (i <- 0 until select.size) {
         filtered(i).bits := input.bits