Merge pull request #387 from ucb-bar/safer-crossings

Safer crossings

src/main/scala/diplomacy/Nodes.scala

@@ -50,8 +50,21 @@ abstract class BaseNode
   protected[diplomacy] def colour:  String
 }
 
-trait InwardNode[DI, UI, BI <: Data] extends BaseNode
+case class NodeHandle[DI, UI, BI <: Data, DO, UO, BO <: Data]
+  (inward: InwardNode[DI, UI, BI], outward: OutwardNode[DO, UO, BO])
+  extends Object with InwardNodeHandle[DI, UI, BI] with OutwardNodeHandle[DO, UO, BO]
+
+trait InwardNodeHandle[DI, UI, BI <: Data]
 {
+  val inward: InwardNode[DI, UI, BI]
+  def := (h: OutwardNodeHandle[DI, UI, BI])(implicit sourceInfo: SourceInfo): Option[LazyModule] =
+    inward.:=(h)(sourceInfo)
+}
+
+trait InwardNode[DI, UI, BI <: Data] extends BaseNode with InwardNodeHandle[DI, UI, BI]
+{
+  val inward = this
+
   protected[diplomacy] val numPI: Range.Inclusive
   require (!numPI.isEmpty, s"No number of inputs would be acceptable to ${name}${lazyModule.line}")
   require (numPI.start >= 0, s"${name} accepts a negative number of inputs${lazyModule.line}")
@@ -75,8 +88,15 @@ trait InwardNode[DI, UI, BI <: Data] extends BaseNode
   protected[diplomacy] def iConnect: Vec[BI]
 }
 
-trait OutwardNode[DO, UO, BO <: Data] extends BaseNode
+trait OutwardNodeHandle[DO, UO, BO <: Data]
 {
+  val outward: OutwardNode[DO, UO, BO]
+}
+
+trait OutwardNode[DO, UO, BO <: Data] extends BaseNode with OutwardNodeHandle[DO, UO, BO]
+{
+  val outward = this
+
   protected[diplomacy] val numPO: Range.Inclusive
   require (!numPO.isEmpty, s"No number of outputs would be acceptable to ${name}${lazyModule.line}")
   require (numPO.start >= 0, s"${name} accepts a negative number of outputs${lazyModule.line}")
@@ -136,8 +156,9 @@ class MixedNode[DI, UI, EI, BI <: Data, DO, UO, EO, BO <: Data](
   def iConnect = bundleIn
 
   // connects the outward part of a node with the inward part of this node
-  def := (y: OutwardNode[DI, UI, BI])(implicit sourceInfo: SourceInfo): Option[LazyModule] = {
+  override def := (h: OutwardNodeHandle[DI, UI, BI])(implicit sourceInfo: SourceInfo): Option[LazyModule] = {
     val x = this // x := y
+    val y = h.outward
     val info = sourceLine(sourceInfo, " at ", "")
     require (!LazyModule.stack.isEmpty, s"${y.name} cannot be connected to ${x.name} outside of LazyModule scope" + info)
     val i = x.iPushed

src/main/scala/regmapper/RegisterCrossing.scala

@@ -7,47 +7,79 @@ import chisel3.util.{Irrevocable}
 import util.{AsyncQueue,AsyncResetRegVec}
 
 // A very simple flow control state machine, run in the specified clock domain
-class BusyRegisterCrossing(clock: Clock, reset: Bool)
+class BusyRegisterCrossing extends Module {
-    extends Module(_clock = clock, _reset = reset) {
   val io = new Bundle {
-    val progress       = Bool(INPUT)
+    val bypass                  = Bool(INPUT)
-    val request_valid  = Bool(INPUT)
+    val master_request_valid    = Bool(INPUT)
-    val response_ready = Bool(INPUT)
+    val master_request_ready    = Bool(OUTPUT)
-    val busy           = Bool(OUTPUT)
+    val master_response_valid   = Bool(OUTPUT)
+    val master_response_ready   = Bool(INPUT)
+    val crossing_request_valid  = Bool(OUTPUT)
+    val crossing_request_ready  = Bool(INPUT)
+    // ... no crossing_response_ready; we are always ready
   }
 
-  val busy = RegInit(Bool(false))
+  val busy   = RegInit(Bool(false))
-  when (io.progress) {
+  val bypass = Reg(Bool())
-    busy := Mux(busy, !io.response_ready, io.request_valid)
+
+  when (io.crossing_request_ready || Mux(busy, bypass, io.bypass)) {
+    busy := Mux(busy, !io.master_response_ready, io.master_request_valid)
   }
-  io.busy := busy
+
+  when (io.master_request_valid && io.master_request_ready) {
+    bypass := io.bypass
+  }
+
+  io.crossing_request_valid := io.master_request_valid && !io.bypass && !busy
+  io.master_request_ready   := (io.bypass || io.crossing_request_ready) && !busy
+  io.master_response_valid  := (bypass    || io.crossing_request_ready) &&  busy
+}
+
+class RegisterCrossingAssertion extends Module {
+  val io = new Bundle {
+    val master_bypass = Bool(INPUT)
+    val slave_reset = Bool(INPUT)
+  }
+
+  assert (io.master_bypass || !io.slave_reset)
 }
 
 // RegField should support connecting to one of these
 class RegisterWriteIO[T <: Data](gen: T) extends Bundle {
-  val request  = Decoupled(gen).flip()
+  val request  = Decoupled(gen).flip
   val response = Irrevocable(Bool()) // ignore .bits
 }
 
-// To turn on/off a domain:
+// To turn off=>on a domain:
-//  1. lower allow on the other side
+//  A. To turn disable the master domain
-//  2. wait for inflight traffic to resolve
+//    1. wait for all inflight traffic to resolve
-//  3. assert reset in the domain
+//    2. assert master reset
-//  4. turn off the domain
+//    3. (optional) stop the master clock
-//  5. turn on the domain
+//    --- YOU MAY NOT TURN OFF POWER ---
-//  6. deassert reset in the domain
+//    4. re-enable the clock
-//  7. raise allow on the other side
+//    5. deassert reset
+//  B. To turn off the slave domain
+//    1. assert bypass
+//    2. wait for inflight traffic to resolve
+//    3. assert slave reset
+//    4. (optional) stop the slave clock
+//    --- YOU MAY NOT TURN OFF POWER ---
+//    5. re-enable the clock
+//    6. deassert reset
+//    7. deassert bypass
+//
+// If you need to cut power, use something that support isolation gates.
 
 class RegisterWriteCrossingIO[T <: Data](gen: T) extends Bundle {
   // Master clock domain
   val master_clock   = Clock(INPUT)
   val master_reset   = Bool(INPUT)
-  val master_allow   = Bool(INPUT) // actually wait for the slave
   val master_port    = new RegisterWriteIO(gen)
+  // Bypass requests from the master to be noops
+  val master_bypass  = Bool(INPUT)
   // Slave clock domain
   val slave_clock    = Clock(INPUT)
   val slave_reset    = Bool(INPUT)
-  val slave_allow    = Bool(INPUT) // honour requests from the master
   val slave_register = gen.asOutput
   val slave_valid    = Bool(OUTPUT) // is high on 1st cycle slave_register has a new value
 }
@@ -55,37 +87,40 @@ class RegisterWriteCrossingIO[T <: Data](gen: T) extends Bundle {
 class RegisterWriteCrossing[T <: Data](gen: T, sync: Int = 3) extends Module {
   val io = new RegisterWriteCrossingIO(gen)
   // The crossing must only allow one item inflight at a time
+  val control = Module(new BusyRegisterCrossing)
   val crossing = Module(new AsyncQueue(gen, 1, sync))
 
-  // We can just randomly reset one-side of a single entry AsyncQueue.
+  control.clock := io.master_clock
-  // If the enq side is reset, at worst deq.bits is reassigned from mem(0), which stays fixed.
+  control.reset := io.master_reset
-  // If the deq side is reset, at worst the master rewrites mem(0) once, deq.bits stays fixed.
   crossing.io.enq_clock := io.master_clock
   crossing.io.enq_reset := io.master_reset
   crossing.io.deq_clock := io.slave_clock
   crossing.io.deq_reset := io.slave_reset
 
+  control.io.bypass := io.master_bypass
+  control.io.master_request_valid  := io.master_port.request.valid
+  control.io.master_response_ready := io.master_port.response.ready
+  io.master_port.request.ready  := control.io.master_request_ready
+  io.master_port.response.valid := control.io.master_response_valid
+
+  control.io.crossing_request_ready := crossing.io.enq.ready
+  crossing.io.enq.valid := control.io.crossing_request_valid
   crossing.io.enq.bits := io.master_port.request.bits
-  io.slave_register := crossing.io.deq.bits
-  io.slave_valid    := crossing.io.deq.valid
-
-  // If the slave is not operational, just drop the write.
-  val progress = crossing.io.enq.ready || !io.master_allow
-
-  val control = Module(new BusyRegisterCrossing(io.master_clock, io.master_reset))
-  control.io.progress := progress
-  control.io.request_valid  := io.master_port.request.valid
-  control.io.response_ready := io.master_port.response.ready
 
   crossing.io.deq.ready := Bool(true)
-  crossing.io.enq.valid := io.master_port.request.valid && !control.io.busy
+  io.slave_valid := crossing.io.deq.valid
-  io.master_port.request.ready  := progress && !control.io.busy
+  io.slave_register := crossing.io.deq.bits
-  io.master_port.response.valid := progress && control.io.busy
+
+  val assertion = Module(new RegisterCrossingAssertion)
+  assertion.clock := io.master_clock
+  assertion.reset := io.master_reset
+  assertion.io.master_bypass := io.master_bypass
+  assertion.io.slave_reset := io.slave_reset
 }
 
 // RegField should support connecting to one of these
 class RegisterReadIO[T <: Data](gen: T) extends Bundle {
-  val request  = Decoupled(Bool()).flip() // ignore .bits
+  val request  = Decoupled(Bool()).flip // ignore .bits
   val response = Irrevocable(gen)
 }
 
@@ -93,43 +128,46 @@ class RegisterReadCrossingIO[T <: Data](gen: T) extends Bundle {
   // Master clock domain
   val master_clock   = Clock(INPUT)
   val master_reset   = Bool(INPUT)
-  val master_allow   = Bool(INPUT) // actually wait for the slave
   val master_port    = new RegisterReadIO(gen)
+  // Bypass requests from the master to be noops
+  val master_bypass  = Bool(INPUT)
   // Slave clock domain
   val slave_clock    = Clock(INPUT)
   val slave_reset    = Bool(INPUT)
-  val slave_allow    = Bool(INPUT) // honour requests from the master
   val slave_register = gen.asInput
 }
 
 class RegisterReadCrossing[T <: Data](gen: T, sync: Int = 3) extends Module {
   val io = new RegisterReadCrossingIO(gen)
   // The crossing must only allow one item inflight at a time
+  val control = Module(new BusyRegisterCrossing)
   val crossing = Module(new AsyncQueue(gen, 1, sync))
 
-  // We can just randomly reset one-side of a single entry AsyncQueue.
+  control.clock := io.master_clock
-  // If the enq side is reset, at worst deq.bits is reassigned from mem(0), which stays fixed.
+  control.reset := io.master_reset
-  // If the deq side is reset, at worst the slave rewrites mem(0) once, deq.bits stays fixed.
-  crossing.io.enq_clock := io.slave_clock
-  crossing.io.enq_reset := io.slave_reset
   crossing.io.deq_clock := io.master_clock
   crossing.io.deq_reset := io.master_reset
+  crossing.io.enq_clock := io.slave_clock
+  crossing.io.enq_reset := io.slave_reset
 
-  crossing.io.enq.bits := io.slave_register
+  control.io.bypass := io.master_bypass
+  control.io.master_request_valid  := io.master_port.request.valid
+  control.io.master_response_ready := io.master_port.response.ready
+  io.master_port.request.ready  := control.io.master_request_ready
+  io.master_port.response.valid := control.io.master_response_valid
+
+  control.io.crossing_request_ready := crossing.io.deq.valid
+  crossing.io.deq.ready := control.io.crossing_request_valid
   io.master_port.response.bits := crossing.io.deq.bits
 
-  // If the slave is not operational, just repeat the last value we saw.
-  val progress = crossing.io.deq.valid || !io.master_allow
-
-  val control = Module(new BusyRegisterCrossing(io.master_clock, io.master_reset))
-  control.io.progress := progress
-  control.io.request_valid  := io.master_port.request.valid
-  control.io.response_ready := io.master_port.response.ready
-
-  io.master_port.response.valid := progress && control.io.busy
-  io.master_port.request.ready  := progress && !control.io.busy
-  crossing.io.deq.ready := io.master_port.request.valid && !control.io.busy
   crossing.io.enq.valid := Bool(true)
+  crossing.io.enq.bits := io.slave_register
+
+  val assertion = Module(new RegisterCrossingAssertion)
+  assertion.clock := io.master_clock
+  assertion.reset := io.master_reset
+  assertion.io.master_bypass := io.master_bypass
+  assertion.io.slave_reset := io.slave_reset
 }
 
 /** Wrapper to create an
@@ -151,8 +189,7 @@ object AsyncRWSlaveRegField {
     width:  Int,
     init: Int,
     name: Option[String] = None,
-    master_allow: Bool = Bool(true),
+    master_bypass: Bool = Bool(true)
-    slave_allow: Bool = Bool(true)
   ): (UInt, RegField) = {
 
     val async_slave_reg = Module(new AsyncResetRegVec(width, init))
@@ -163,12 +200,11 @@ object AsyncRWSlaveRegField {
     val wr_crossing = Module (new RegisterWriteCrossing(UInt(width = width)))
     name.foreach(n => wr_crossing.suggestName(s"${n}_wcrossing"))
 
-    wr_crossing.io.master_clock := master_clock
+    wr_crossing.io.master_clock  := master_clock
-    wr_crossing.io.master_reset := master_reset
+    wr_crossing.io.master_reset  := master_reset
-    wr_crossing.io.master_allow := master_allow
+    wr_crossing.io.master_bypass := master_bypass
-    wr_crossing.io.slave_clock  := slave_clock
+    wr_crossing.io.slave_clock   := slave_clock
-    wr_crossing.io.slave_reset  := slave_reset
+    wr_crossing.io.slave_reset   := slave_reset
-    wr_crossing.io.slave_allow  := slave_allow
 
     async_slave_reg.io.en := wr_crossing.io.slave_valid
     async_slave_reg.io.d  := wr_crossing.io.slave_register
@@ -176,12 +212,11 @@ object AsyncRWSlaveRegField {
     val rd_crossing = Module (new RegisterReadCrossing(UInt(width = width )))
     name.foreach(n => rd_crossing.suggestName(s"${n}_rcrossing"))
 
-    rd_crossing.io.master_clock := master_clock
+    rd_crossing.io.master_clock  := master_clock
-    rd_crossing.io.master_reset := master_reset
+    rd_crossing.io.master_reset  := master_reset
-    rd_crossing.io.master_allow := master_allow
+    rd_crossing.io.master_bypass := master_bypass
-    rd_crossing.io.slave_clock  := slave_clock
+    rd_crossing.io.slave_clock   := slave_clock
-    rd_crossing.io.slave_reset  := slave_reset
+    rd_crossing.io.slave_reset   := slave_reset
-    rd_crossing.io.slave_allow  := slave_allow
 
     rd_crossing.io.slave_register := async_slave_reg.io.q
 

src/main/scala/uncore/axi4/package.scala

@@ -5,5 +5,5 @@ import diplomacy._
 
 package object axi4
 {
-  type AXI4OutwardNode = OutwardNode[AXI4MasterPortParameters, AXI4SlavePortParameters, AXI4Bundle]
+  type AXI4OutwardNode = OutwardNodeHandle[AXI4MasterPortParameters, AXI4SlavePortParameters, AXI4Bundle]
 }

src/main/scala/uncore/tilelink2/Bundles.scala

@@ -226,6 +226,9 @@ final class AsyncBundle[T <: Data](val depth: Int, gen: T) extends Bundle
   val ridx = UInt(width = log2Up(depth)+1).flip
   val widx = UInt(width = log2Up(depth)+1)
   val mem  = Vec(depth, gen)
+  val source_reset_n = Bool()
+  val sink_reset_n = Bool().flip
+
   override def cloneType: this.type = new AsyncBundle(depth, gen).asInstanceOf[this.type]
 }
 
@@ -236,6 +239,8 @@ object FromAsyncBundle
     x.ridx := sink.io.ridx
     sink.io.widx := x.widx
     sink.io.mem  := x.mem
+    sink.io.source_reset_n := x.source_reset_n
+    x.sink_reset_n := !sink.reset
     val out = Wire(Irrevocable(x.mem(0)))
     out.valid := sink.io.deq.valid
     out.bits := sink.io.deq.bits
@@ -255,6 +260,8 @@ object ToAsyncBundle
     source.io.ridx := out.ridx
     out.mem := source.io.mem
     out.widx := source.io.widx
+    source.io.sink_reset_n := out.sink_reset_n
+    out.source_reset_n := !source.reset
     out
   }
 }

src/main/scala/uncore/tilelink2/Crossing.scala

@@ -18,16 +18,22 @@ class TLAsyncCrossingSource(sync: Int = 3) extends LazyModule
     }
 
     ((io.in zip io.out) zip (node.edgesIn zip node.edgesOut)) foreach { case ((in, out), (edgeIn, edgeOut)) =>
+      val sink_reset_n = out.a.sink_reset_n
       val bce = edgeIn.manager.anySupportAcquire && edgeIn.client.anySupportProbe
       val depth = edgeOut.manager.depth
 
       out.a <> ToAsyncBundle(in.a, depth, sync)
       in.d <> FromAsyncBundle(out.d, sync)
 
+      assert (!in.a.valid || sink_reset_n, "A channel request sent to a missing manager")
+
       if (bce) {
         in.b <> FromAsyncBundle(out.b, sync)
         out.c <> ToAsyncBundle(in.c, depth, sync)
         out.e <> ToAsyncBundle(in.e, depth, sync)
+
+        assert (!in.c.valid || sink_reset_n, "C channel response sent to a missing manager")
+        assert (!in.e.valid || sink_reset_n, "E channel response sent to a missing manager")
       } else {
         in.b.valid := Bool(false)
         in.c.ready := Bool(true)
@@ -51,15 +57,20 @@ class TLAsyncCrossingSink(depth: Int = 8, sync: Int = 3) extends LazyModule
     }
 
     ((io.in zip io.out) zip (node.edgesIn zip node.edgesOut)) foreach { case ((in, out), (edgeIn, edgeOut)) =>
+      val source_reset_n = in.a.source_reset_n
       val bce = edgeOut.manager.anySupportAcquire && edgeOut.client.anySupportProbe
 
       out.a <> FromAsyncBundle(in.a, sync)
       in.d <> ToAsyncBundle(out.d, depth, sync)
 
+      assert (!out.d.valid || source_reset_n, "D channel respose sent to missing client")
+
       if (bce) {
         in.b <> ToAsyncBundle(out.b, depth, sync)
         out.c <> FromAsyncBundle(in.c, sync)
         out.e <> FromAsyncBundle(in.e, sync)
+
+        assert (!out.b.valid || source_reset_n, "B channel request sent to missing client")
       } else {
         in.b.widx := UInt(0)
         in.c.ridx := UInt(0)
@@ -96,6 +107,7 @@ class TLAsyncCrossing(depth: Int = 8, sync: Int = 3) extends LazyModule
 {
   val nodeIn = TLInputNode()
   val nodeOut = TLOutputNode()
+  val node = NodeHandle(nodeIn, nodeOut)
 
   val source = LazyModule(new TLAsyncCrossingSource(sync))
   val sink = LazyModule(new TLAsyncCrossingSink(depth, sync))
@@ -140,8 +152,8 @@ class TLRAMCrossing extends LazyModule {
   val cross = LazyModule(new TLAsyncCrossing)
 
   model.node := fuzz.node
-  cross.nodeIn := TLFragmenter(4, 256)(model.node)
+  cross.node := TLFragmenter(4, 256)(model.node)
-  val monitor = (ram.node := cross.nodeOut)
+  val monitor = (ram.node := cross.node)
 
   lazy val module = new LazyModuleImp(this) with HasUnitTestIO {
     io.finished := fuzz.module.io.finished

src/main/scala/uncore/tilelink2/Edges.scala

@@ -4,6 +4,7 @@ package uncore.tilelink2
 
 import Chisel._
 import chisel3.internal.sourceinfo.SourceInfo
+import chisel3.util.IrrevocableIO
 import diplomacy._
 
 class TLEdge(
@@ -218,6 +219,20 @@ class TLEdge(
       }
     }
   }
+
+  def firstlast(bits: TLChannel, fire: Bool): (Bool, Bool, UInt) = {
+    val beats1   = numBeats1(bits)
+    val counter  = RegInit(UInt(0, width = log2Up(maxTransfer / manager.beatBytes)))
+    val counter1 = counter - UInt(1)
+    val first = counter === UInt(0)
+    val last  = counter === UInt(1) || beats1 === UInt(0)
+    when (fire) {
+      counter := Mux(first, beats1, counter1)
+    }
+    (first, last, beats1 & ~counter1)
+  }
+
+  def firstlast(x: IrrevocableIO[TLChannel]): (Bool, Bool, UInt) = firstlast(x.bits, x.fire())
 }
 
 class TLEdgeOut(

src/main/scala/uncore/tilelink2/Fuzzer.scala

@@ -113,19 +113,11 @@ class TLFuzzer(
 
     // Progress within each operation
     val a = out.a.bits
-    val a_beats1 = edge.numBeats1(a)
+    val (a_first, a_last, _) = edge.firstlast(out.a)
-    val a_counter = RegInit(UInt(0, width = maxLgBeats))
-    val a_counter1 = a_counter - UInt(1)
-    val a_first = a_counter === UInt(0)
-    val a_last  = a_counter === UInt(1) || a_beats1 === UInt(0)
     val req_done = out.a.fire() && a_last
 
     val d = out.d.bits
-    val d_beats1 = edge.numBeats1(d)
+    val (d_first, d_last, _) = edge.firstlast(out.d)
-    val d_counter = RegInit(UInt(0, width = maxLgBeats))
-    val d_counter1 = d_counter - UInt(1)
-    val d_first = d_counter === UInt(0)
-    val d_last  = d_counter === UInt(1) || d_beats1 === UInt(0)
     val resp_done = out.d.fire() && d_last
 
     // Source ID generation
@@ -199,14 +191,12 @@ class TLFuzzer(
     inc := !legal || req_done
     inc_beat := !legal || out.a.fire()
 
-    when (out.a.fire()) {
+    when (out.a.fire() && a_last) {
-      a_counter := Mux(a_first, a_beats1, a_counter1)
+      num_reqs := num_reqs - UInt(1)
-      when(a_last) { num_reqs := num_reqs - UInt(1) }
     }
 
-    when (out.d.fire()) {
+    when (out.d.fire() && d_last) {
-      d_counter := Mux(d_first, d_beats1, d_counter1)
+      num_resps := num_resps - UInt(1)
-      when(d_last) { num_resps := num_resps - UInt(1) }
     }
   }
 }
@@ -229,8 +219,8 @@ class TLFuzzRAM extends LazyModule
   xbar2.node := TLAtomicAutomata()(model.node)
   ram2.node := TLFragmenter(16, 256)(xbar2.node)
   xbar.node := TLWidthWidget(16)(TLHintHandler()(xbar2.node))
-  cross.nodeIn := TLFragmenter(4, 256)(TLBuffer()(xbar.node))
+  cross.node := TLFragmenter(4, 256)(TLBuffer()(xbar.node))
-  val monitor = (ram.node := cross.nodeOut)
+  val monitor = (ram.node := cross.node)
   gpio.node := TLFragmenter(4, 32)(TLBuffer()(xbar.node))
 
   lazy val module = new LazyModuleImp(this) with HasUnitTestIO {

src/main/scala/uncore/tilelink2/HintHandler.scala

@@ -33,38 +33,20 @@ class TLHintHandler(supportManagers: Boolean = true, supportClients: Boolean = f
     val smartClients = edgeIn.client.clients.map(_.supportsHint.max == edgeIn.client.maxTransfer).reduce(_&&_)
     val smartManagers = edgeOut.manager.managers.map(_.supportsHint.max == edgeOut.manager.maxTransfer).reduce(_&&_)
 
-    if (supportManagers && !smartManagers) {
+    if (supportManagers && !(passthrough && smartManagers)) {
-      // State of the Hint bypass
-      val counter = RegInit(UInt(0, width = log2Up(edgeOut.manager.maxTransfer/edgeOut.manager.beatBytes)))
-      val hintHoldsD = RegInit(Bool(false))
-      val outerHoldsD = counter =/= UInt(0)
-      // Only one of them can hold it
-      assert (!hintHoldsD || !outerHoldsD)
-
-      // Count outer D beats
-      val beats1 = edgeOut.numBeats1(out.d.bits)
-      when (out.d.fire()) { counter := Mux(outerHoldsD, counter - UInt(1), beats1) }
-
-      // Who wants what?
       val address = edgeIn.address(in.a.bits)
       val handleA = if (passthrough) !edgeOut.manager.supportsHintFast(address, edgeIn.size(in.a.bits)) else Bool(true)
       val hintBitsAtA = handleA && in.a.bits.opcode === TLMessages.Hint
-      val hintWantsD = in.a.valid && hintBitsAtA
+      val hint = Wire(out.d)
-      val outerWantsD = out.d.valid
 
-      // Prioritize existing D traffic over HintAck (and finish multibeat xfers)
+      hint.valid  := in.a.valid &&  hintBitsAtA
-      val hintWinsD = hintHoldsD || (!outerHoldsD && !outerWantsD)
-      hintHoldsD := hintWantsD && hintWinsD && !in.d.ready
-      // Hint can only hold D b/c it still wants it from last cycle
-      assert (!hintHoldsD || hintWantsD)
-
-      in.d.valid  := Mux(hintWinsD, hintWantsD, outerWantsD)
-      in.d.bits   := Mux(hintWinsD, edgeIn.HintAck(in.a.bits, edgeOut.manager.findIdStartFast(address)), out.d.bits)
-      out.d.ready := in.d.ready && !hintHoldsD
-
-      in.a.ready  := Mux(hintBitsAtA, hintWinsD && in.d.ready, out.a.ready)
       out.a.valid := in.a.valid && !hintBitsAtA
-      out.a.bits  := in.a.bits
+      in.a.ready := Mux(hintBitsAtA, hint.ready, out.a.ready)
+
+      hint.bits := edgeIn.HintAck(in.a.bits, edgeOut.manager.findIdStartFast(address))
+      out.a.bits := in.a.bits
+
+      TLArbiter(TLArbiter.lowestIndexFirst)(in.d, (edgeOut.numBeats(out.d.bits), out.d), (UInt(1), hint))
     } else {
       out.a.valid := in.a.valid
       in.a.ready := out.a.ready
@@ -75,37 +57,19 @@ class TLHintHandler(supportManagers: Boolean = true, supportClients: Boolean = f
       in.d.bits := out.d.bits
     }
 
-    if (supportClients && !smartClients) {
+    if (supportClients && !(passthrough && smartClients)) {
-      // State of the Hint bypass
-      val counter = RegInit(UInt(0, width = log2Up(edgeIn.client.maxTransfer/edgeIn.manager.beatBytes)))
-      val hintHoldsC = RegInit(Bool(false))
-      val innerHoldsC = counter =/= UInt(0)
-      // Only one of them can hold it
-      assert (!hintHoldsC || !innerHoldsC)
-
-      // Count inner C beats
-      val beats1 = edgeIn.numBeats1(in.c.bits)
-      when (in.c.fire()) { counter := Mux(innerHoldsC, counter - UInt(1), beats1) }
-
-      // Who wants what?
       val handleB = if (passthrough) !edgeIn.client.supportsHint(out.b.bits.source, edgeOut.size(out.b.bits)) else Bool(true)
       val hintBitsAtB = handleB && out.b.bits.opcode === TLMessages.Hint
-      val hintWantsC = out.b.valid && hintBitsAtB
+      val hint = Wire(in.c)
-      val innerWantsC = in.c.valid
 
-      // Prioritize existing C traffic over HintAck (and finish multibeat xfers)
+      hint.valid := out.b.valid &&  hintBitsAtB
-      val hintWinsC = hintHoldsC || (!innerHoldsC && !innerWantsC)
+      in.b.valid := out.b.valid && !hintBitsAtB
-      hintHoldsC := hintWantsC && hintWinsC && !out.c.ready
+      out.b.ready := Mux(hintBitsAtB, hint.ready, in.b.ready)
-      // Hint can only hold C b/c it still wants it from last cycle
-      assert (!hintHoldsC || hintWantsC)
 
-      out.c.valid := Mux(hintWinsC, hintWantsC, innerWantsC)
+      hint.bits := edgeOut.HintAck(out.b.bits)
-      out.c.bits  := Mux(hintWinsC, edgeOut.HintAck(out.b.bits), in.c.bits)
+      in.b.bits := out.b.bits
-      in.c.ready  := out.c.ready && !hintHoldsC
 
-      out.b.ready := Mux(hintBitsAtB, hintWinsC && out.c.ready, in.b.ready)
+      TLArbiter(TLArbiter.lowestIndexFirst)(out.c, (edgeIn.numBeats(in.c.bits), in.c), (UInt(1), hint))
-      in.b.valid  := out.b.valid && !hintBitsAtB
-      in.b.bits   := out.b.bits
     } else if (bce) {
       in.b.valid := out.b.valid
       out.b.ready := in.b.ready

src/main/scala/uncore/tilelink2/Isolation.scala

@@ -6,6 +6,7 @@ import Chisel._
 import chisel3.internal.sourceinfo.SourceInfo
 import diplomacy._
 
+// READ the comments in the TLIsolation object before you instantiate this module
 class TLIsolation(fOut: (Bool, UInt) => UInt, fIn: (Bool, UInt) => UInt) extends LazyModule
 {
   val node = TLAsyncIdentityNode()
@@ -30,6 +31,11 @@ class TLIsolation(fOut: (Bool, UInt) => UInt, fIn: (Bool, UInt) => UInt) extends
       in .d.widx := ISOi(out.d.widx)
       in .d.mem  := ISOi(out.d.mem)
 
+      out.a.source_reset_n := ISOo(in .a.source_reset_n)
+      in .a.sink_reset_n   := ISOi(out.a.sink_reset_n)
+      out.d.sink_reset_n   := ISOo(in .d.sink_reset_n)
+      in .d.source_reset_n := ISOi(out.d.source_reset_n)
+
       if (edgeOut.manager.base.anySupportAcquire && edgeOut.client.base.anySupportProbe) {
         in .b.widx := ISOi(out.b.widx)
         in .c.ridx := ISOi(out.c.ridx)
@@ -40,6 +46,13 @@ class TLIsolation(fOut: (Bool, UInt) => UInt, fIn: (Bool, UInt) => UInt) extends
         in .b.mem  := ISOi(out.b.mem)
         out.c.mem  := ISOo(in .c.mem)
         out.e.mem  := ISOo(in .e.mem)
+
+        out.b.sink_reset_n   := ISOo(in .b.sink_reset_n)
+        in .b.source_reset_n := ISOi(out.b.source_reset_n)
+        out.c.source_reset_n := ISOo(in .c.source_reset_n)
+        in .c.sink_reset_n   := ISOi(out.c.sink_reset_n)
+        out.e.source_reset_n := ISOo(in .e.source_reset_n)
+        in .e.sink_reset_n   := ISOi(out.e.sink_reset_n)
       } else {
         in .b.widx := UInt(0)
         in .c.ridx := UInt(0)
@@ -55,9 +68,10 @@ class TLIsolation(fOut: (Bool, UInt) => UInt, fIn: (Bool, UInt) => UInt) extends
 object TLIsolation
 {
   // applied to the TL source node; y.node := TLIsolation(fOut, fIn)(x.node)
-  // f should insert an isolation gate between the input UInt and its result
+  // f* should insert an isolation gate between the input UInt and its result
-  // fOut is applied for data flowing from client to manager
+  // fOut is applied to data flowing from client to manager
-  // fIn  is applied for data flowing from manager to client
+  // fIn  is applied to data flowing from manager to client
+  // **** WARNING: the isolation functions must bring the values to 0 ****
   def apply(fOut: (Bool, UInt) => UInt, fIn: (Bool, UInt) => UInt)(x: TLAsyncOutwardNode)(implicit sourceInfo: SourceInfo): (TLAsyncOutwardNode, () => (Bool, Bool)) = {
     val iso = LazyModule(new TLIsolation(fOut, fIn))
     iso.node := x

src/main/scala/uncore/tilelink2/Monitor.scala

@@ -282,68 +282,60 @@ class TLMonitor(gen: () => TLBundleSnoop, edge: () => TLEdge, sourceInfo: Source
   }
 
   def legalizeMultibeatA(a: IrrevocableSnoop[TLBundleA], edge: TLEdge)(implicit sourceInfo: SourceInfo) {
-    val counter = RegInit(UInt(0, width = log2Up(edge.maxTransfer)))
+    val (a_first, _, _) = edge.firstlast(a.bits, a.fire())
     val opcode  = Reg(UInt())
     val param   = Reg(UInt())
     val size    = Reg(UInt())
     val source  = Reg(UInt())
     val addr_hi = Reg(UInt())
-    when (a.valid && counter =/= UInt(0)) {
+    when (a.valid && !a_first) {
       assert (a.bits.opcode === opcode, "'A' channel opcode changed within multibeat operation" + extra)
       assert (a.bits.param  === param,  "'A' channel param changed within multibeat operation" + extra)
       assert (a.bits.size   === size,   "'A' channel size changed within multibeat operation" + extra)
       assert (a.bits.source === source, "'A' channel source changed within multibeat operation" + extra)
       assert (a.bits.addr_hi=== addr_hi,"'A' channel addr_hi changed with multibeat operation" + extra)
     }
-    when (a.fire()) {
+    when (a.fire() && a_first) {
-      counter := counter - UInt(1)
+      opcode  := a.bits.opcode
-      when (counter === UInt(0)) {
+      param   := a.bits.param
-        counter := edge.numBeats(a.bits) - UInt(1)
+      size    := a.bits.size
-        opcode  := a.bits.opcode
+      source  := a.bits.source
-        param   := a.bits.param
+      addr_hi := a.bits.addr_hi
-        size    := a.bits.size
-        source  := a.bits.source
-        addr_hi := a.bits.addr_hi
-      }
     }
   }
 
   def legalizeMultibeatB(b: IrrevocableSnoop[TLBundleB], edge: TLEdge)(implicit sourceInfo: SourceInfo) {
-    val counter = RegInit(UInt(0, width = log2Up(edge.maxTransfer)))
+    val (b_first, _, _) = edge.firstlast(b.bits, b.fire())
     val opcode  = Reg(UInt())
     val param   = Reg(UInt())
     val size    = Reg(UInt())
     val source  = Reg(UInt())
     val addr_hi = Reg(UInt())
-    when (b.valid && counter =/= UInt(0)) {
+    when (b.valid && !b_first) {
       assert (b.bits.opcode === opcode, "'B' channel opcode changed within multibeat operation" + extra)
       assert (b.bits.param  === param,  "'B' channel param changed within multibeat operation" + extra)
       assert (b.bits.size   === size,   "'B' channel size changed within multibeat operation" + extra)
       assert (b.bits.source === source, "'B' channel source changed within multibeat operation" + extra)
       assert (b.bits.addr_hi=== addr_hi,"'B' channel addr_hi changed with multibeat operation" + extra)
     }
-    when (b.fire()) {
+    when (b.fire() && b_first) {
-      counter := counter - UInt(1)
+      opcode  := b.bits.opcode
-      when (counter === UInt(0)) {
+      param   := b.bits.param
-        counter := edge.numBeats(b.bits) - UInt(1)
+      size    := b.bits.size
-        opcode  := b.bits.opcode
+      source  := b.bits.source
-        param   := b.bits.param
+      addr_hi := b.bits.addr_hi
-        size    := b.bits.size
-        source  := b.bits.source
-        addr_hi := b.bits.addr_hi
-      }
     }
   }
 
   def legalizeMultibeatC(c: IrrevocableSnoop[TLBundleC], edge: TLEdge)(implicit sourceInfo: SourceInfo) {
-    val counter = RegInit(UInt(0, width = log2Up(edge.maxTransfer)))
+    val (c_first, _, _) = edge.firstlast(c.bits, c.fire())
     val opcode  = Reg(UInt())
     val param   = Reg(UInt())
     val size    = Reg(UInt())
     val source  = Reg(UInt())
     val addr_hi = Reg(UInt())
     val addr_lo = Reg(UInt())
-    when (c.valid && counter =/= UInt(0)) {
+    when (c.valid && !c_first) {
       assert (c.bits.opcode === opcode, "'C' channel opcode changed within multibeat operation" + extra)
       assert (c.bits.param  === param,  "'C' channel param changed within multibeat operation" + extra)
       assert (c.bits.size   === size,   "'C' channel size changed within multibeat operation" + extra)
@@ -351,29 +343,25 @@ class TLMonitor(gen: () => TLBundleSnoop, edge: () => TLEdge, sourceInfo: Source
       assert (c.bits.addr_hi=== addr_hi,"'C' channel addr_hi changed with multibeat operation" + extra)
       assert (c.bits.addr_lo=== addr_lo,"'C' channel addr_lo changed with multibeat operation" + extra)
     }
-    when (c.fire()) {
+    when (c.fire() && c_first) {
-      counter := counter - UInt(1)
+      opcode  := c.bits.opcode
-      when (counter === UInt(0)) {
+      param   := c.bits.param
-        counter := edge.numBeats(c.bits) - UInt(1)
+      size    := c.bits.size
-        opcode  := c.bits.opcode
+      source  := c.bits.source
-        param   := c.bits.param
+      addr_hi := c.bits.addr_hi
-        size    := c.bits.size
+      addr_lo := c.bits.addr_lo
-        source  := c.bits.source
-        addr_hi := c.bits.addr_hi
-        addr_lo := c.bits.addr_lo
-      }
     }
   }
 
   def legalizeMultibeatD(d: IrrevocableSnoop[TLBundleD], edge: TLEdge)(implicit sourceInfo: SourceInfo) {
-    val counter = RegInit(UInt(0, width = log2Up(edge.maxTransfer)))
+    val (d_first, _, _) = edge.firstlast(d.bits, d.fire())
     val opcode  = Reg(UInt())
     val param   = Reg(UInt())
     val size    = Reg(UInt())
     val source  = Reg(UInt())
     val sink    = Reg(UInt())
     val addr_lo = Reg(UInt())
-    when (d.valid && counter =/= UInt(0)) {
+    when (d.valid && !d_first) {
       assert (d.bits.opcode === opcode, "'D' channel opcode changed within multibeat operation" + extra)
       assert (d.bits.param  === param,  "'D' channel param changed within multibeat operation" + extra)
       assert (d.bits.size   === size,   "'D' channel size changed within multibeat operation" + extra)
@@ -381,17 +369,13 @@ class TLMonitor(gen: () => TLBundleSnoop, edge: () => TLEdge, sourceInfo: Source
       assert (d.bits.sink   === sink,   "'D' channel sink changed with multibeat operation" + extra)
       assert (d.bits.addr_lo=== addr_lo,"'D' channel addr_lo changed with multibeat operation" + extra)
     }
-    when (d.fire()) {
+    when (d.fire() && d_first) {
-      counter := counter - UInt(1)
+      opcode  := d.bits.opcode
-      when (counter === UInt(0)) {
+      param   := d.bits.param
-        counter := edge.numBeats(d.bits) - UInt(1)
+      size    := d.bits.size
-        opcode  := d.bits.opcode
+      source  := d.bits.source
-        param   := d.bits.param
+      sink    := d.bits.sink
-        size    := d.bits.size
+      addr_lo := d.bits.addr_lo
-        source  := d.bits.source
-        sink    := d.bits.sink
-        addr_lo := d.bits.addr_lo
-      }
     }
   }
 
@@ -425,15 +409,8 @@ class TLMonitor(gen: () => TLBundleSnoop, edge: () => TLEdge, sourceInfo: Source
   def legalizeSourceUnique(bundle: TLBundleSnoop, edge: TLEdge)(implicit sourceInfo: SourceInfo) {
     val inflight = RegInit(UInt(0, width = edge.client.endSourceId))
 
-    val a_counter = RegInit(UInt(0, width = log2Up(edge.maxTransfer)))
+    val (_, a_last, _) = edge.firstlast(bundle.a.bits, bundle.a.fire())
-    val a_beats1 = edge.numBeats1(bundle.a.bits)
+    val (_, d_last, _) = edge.firstlast(bundle.d.bits, bundle.d.fire())
-    val a_first = a_counter === UInt(0)
-    val a_last = a_counter === UInt(1) || a_beats1 === UInt(0)
-
-    val d_counter = RegInit(UInt(0, width = log2Up(edge.maxTransfer)))
-    val d_beats1 = edge.numBeats1(bundle.d.bits)
-    val d_first = d_counter === UInt(0)
-    val d_last = d_counter === UInt(1) || d_beats1 === UInt(0)
 
     val bypass = bundle.a.bits.source === bundle.d.bits.source
     val a_bypass = bypass && bundle.d.valid && d_last
@@ -445,14 +422,12 @@ class TLMonitor(gen: () => TLBundleSnoop, edge: () => TLEdge, sourceInfo: Source
 
     val a_set = Wire(init = UInt(0, width = edge.client.endSourceId))
     when (bundle.a.fire()) {
-      a_counter := Mux(a_first, a_beats1, a_counter - UInt(1))
       when (a_last) { a_set := UIntToOH(bundle.a.bits.source) }
       assert(a_bypass || !inflight(bundle.a.bits.source), "'A' channel re-used a source ID" + extra)
     }
 
     val d_clr = Wire(init = UInt(0, width = edge.client.endSourceId))
     when (bundle.d.fire() && bundle.d.bits.opcode =/= TLMessages.ReleaseAck) {
-      d_counter := Mux(d_first, d_beats1, d_counter - UInt(1))
       when (d_last) { d_clr := UIntToOH(bundle.d.bits.source) }
       assert(d_bypass || inflight(bundle.d.bits.source), "'D' channel acknowledged for nothing inflight" + extra)
     }

src/main/scala/uncore/tilelink2/RAMModel.scala

@@ -110,13 +110,10 @@ class TLRAMModel extends LazyModule
     // Process A access requests
     val a = Reg(next = in.a.bits)
     val a_fire = Reg(next = in.a.fire(), init = Bool(false))
-    val a_beats1 = edge.numBeats1(a)
+    val (a_first, a_last, a_address_inc) = edge.firstlast(a, a_fire)
     val a_size = edge.size(a)
     val a_sizeOH = UIntToOH(a_size)
-    val a_counter = RegInit(UInt(0, width = maxLgBeats))
+    val a_addr_hi = a.addr_hi | a_address_inc
-    val a_counter1 = a_counter - UInt(1)
-    val a_first = a_counter === UInt(0)
-    val a_addr_hi = a.addr_hi | (a_beats1 & ~a_counter1)
     val a_base = edge.address(a)
     val a_mask = edge.mask(a_base, a_size)
     val a_fifo = edge.manager.hasFifoIdFast(a_base)
@@ -133,8 +130,6 @@ class TLRAMModel extends LazyModule
 
     when (a_fire) {
       // Record the request so we can handle it's response
-      a_counter := Mux(a_first, a_beats1, a_counter1)
-
       assert (a.opcode =/= TLMessages.Acquire)
 
       // Mark the operation as valid
@@ -199,15 +194,11 @@ class TLRAMModel extends LazyModule
     // Process D access responses
     val d = RegNext(out.d.bits)
     val d_fire = Reg(next = out.d.fire(), init = Bool(false))
-    val d_beats1 = edge.numBeats1(d)
+    val (d_first, d_last, d_address_inc) = edge.firstlast(d, d_fire)
     val d_size = edge.size(d)
     val d_sizeOH = UIntToOH(d_size)
-    val d_counter = RegInit(UInt(0, width = maxLgBeats))
-    val d_counter1 = d_counter - UInt(1)
-    val d_first = d_counter === UInt(0)
-    val d_last  = d_counter === UInt(1) || d_beats1 === UInt(0)
     val d_base = d_flight.base
-    val d_addr_hi = d_base >> shift | (d_beats1 & ~d_counter1)
+    val d_addr_hi = d_base >> shift | d_address_inc
     val d_mask = edge.mask(d_base, d_size)
     val d_fifo = edge.manager.hasFifoIdFast(d_flight.base)
 
@@ -224,8 +215,6 @@ class TLRAMModel extends LazyModule
     val d_valid = valid(d.source)
 
     when (d_fire) {
-      d_counter := Mux(d_first, d_beats1, d_counter1)
-
       // Check the response is correct
       assert (d_size === d_flight.size)
       assert (edge.manager.findIdStartFast(d_flight.base) <= d.sink)

src/main/scala/uncore/tilelink2/RegisterRouterTest.scala

@@ -229,20 +229,18 @@ trait RRTest1Module extends Module with HasRegMap
     val field = UInt(width = bits)
 
     val readCross = Module(new RegisterReadCrossing(field))
-    readCross.io.master_clock := clock
+    readCross.io.master_clock  := clock
-    readCross.io.master_reset := reset
+    readCross.io.master_reset  := reset
-    readCross.io.master_allow := Bool(true)
+    readCross.io.master_bypass := Bool(false)
-    readCross.io.slave_clock := clocks.io.clock_out
+    readCross.io.slave_clock   := clocks.io.clock_out
-    readCross.io.slave_reset := reset
+    readCross.io.slave_reset   := reset
-    readCross.io.slave_allow := Bool(true)
 
     val writeCross = Module(new RegisterWriteCrossing(field))
-    writeCross.io.master_clock := clock
+    writeCross.io.master_clock  := clock
-    writeCross.io.master_reset := reset
+    writeCross.io.master_reset  := reset
-    writeCross.io.master_allow := Bool(true)
+    writeCross.io.master_bypass := Bool(false)
-    writeCross.io.slave_clock := clocks.io.clock_out
+    writeCross.io.slave_clock   := clocks.io.clock_out
-    writeCross.io.slave_reset := reset
+    writeCross.io.slave_reset   := reset
-    writeCross.io.slave_allow := Bool(true)
 
     readCross.io.slave_register := writeCross.io.slave_register
     RegField(bits, readCross.io.master_port, writeCross.io.master_port)

src/main/scala/uncore/tilelink2/package.scala

@@ -5,9 +5,9 @@ import diplomacy._
 
 package object tilelink2
 {
-  type TLOutwardNode = OutwardNode[TLClientPortParameters, TLManagerPortParameters, TLBundle]
+  type TLOutwardNode = OutwardNodeHandle[TLClientPortParameters, TLManagerPortParameters, TLBundle]
-  type TLAsyncOutwardNode = OutwardNode[TLAsyncClientPortParameters, TLAsyncManagerPortParameters, TLAsyncBundle]
+  type TLAsyncOutwardNode = OutwardNodeHandle[TLAsyncClientPortParameters, TLAsyncManagerPortParameters, TLAsyncBundle]
-  type IntOutwardNode = OutwardNode[IntSourcePortParameters, IntSinkPortParameters, Vec[Bool]]
+  type IntOutwardNode = OutwardNodeHandle[IntSourcePortParameters, IntSinkPortParameters, Vec[Bool]]
   def OH1ToUInt(x: UInt) = OHToUInt((x << 1 | UInt(1)) ^ x)
   def UIntToOH1(x: UInt, width: Int) = ~(SInt(-1, width=width).asUInt << x)(width-1, 0)
   def trailingZeros(x: Int) = if (x > 0) Some(log2Ceil(x & -x)) else None

src/main/scala/util/AsyncQueue.scala

@@ -4,19 +4,19 @@ package util
 import Chisel._
 
 object GrayCounter {
-  def apply(bits: Int, increment: Bool = Bool(true), name: String = "binary"): UInt = {
+  def apply(bits: Int, increment: Bool = Bool(true), clear: Bool = Bool(false), name: String = "binary"): UInt = {
     val incremented = Wire(UInt(width=bits))
     val binary = AsyncResetReg(incremented, name)
-    incremented := binary + increment.asUInt()
+    incremented := Mux(clear, UInt(0), binary + increment.asUInt())
     incremented ^ (incremented >> UInt(1))
   }
 }
 
-object AsyncGrayCounter {
+object UIntSyncChain {
   def apply(in: UInt, sync: Int, name: String = "gray"): UInt = {
-    val syncv = List.tabulate(sync)(i =>
+    val syncv = List.tabulate(sync) { i =>
       Module (new AsyncResetRegVec(w = in.getWidth, 0)).suggestName(s"${name}_sync_${i}")
-    )
+    }
     syncv.last.io.d := in
     syncv.last.io.en := Bool(true)
       (syncv.init zip syncv.tail).foreach { case (sink, source) =>
@@ -31,27 +31,38 @@ class AsyncQueueSource[T <: Data](gen: T, depth: Int, sync: Int) extends Module
   val bits = log2Ceil(depth)
   val io = new Bundle {
     // These come from the source domain
-    val enq  = Decoupled(gen).flip()
+    val enq  = Decoupled(gen).flip
     // These cross to the sink clock domain
     val ridx = UInt(INPUT,  width = bits+1)
     val widx = UInt(OUTPUT, width = bits+1)
     val mem  = Vec(depth, gen).asOutput
+    // Reset for the other side
+    val sink_reset_n = Bool().flip
   }
 
+  // extend the sink reset to a full cycle (assertion latency <= 1 cycle)
+  val catch_sink_reset_n = AsyncResetReg(Bool(true), clock, !io.sink_reset_n, "catch_sink_reset_n")
+  // reset_n has a 1 cycle shorter path to ready than ridx does
+  val sink_reset_n = UIntSyncChain(catch_sink_reset_n.asUInt, sync, "sink_reset_n")(0)
+
   val mem = Reg(Vec(depth, gen)) //This does NOT need to be asynchronously reset.
-  val widx = GrayCounter(bits+1, io.enq.fire(), "widx_bin")
+  val widx = GrayCounter(bits+1, io.enq.fire(), !sink_reset_n, "widx_bin")
-  val ridx = AsyncGrayCounter(io.ridx, sync, "ridx_gray")
+  val ridx = UIntSyncChain(io.ridx, sync, "ridx_gray")
   val ready = widx =/= (ridx ^ UInt(depth | depth >> 1))
 
   val index = if (depth == 1) UInt(0) else io.widx(bits-1, 0) ^ (io.widx(bits, bits) << (bits-1))
   when (io.enq.fire()) { mem(index) := io.enq.bits }
-  val ready_reg = AsyncResetReg(ready, "ready")
+
-  io.enq.ready := ready_reg
+  val ready_reg = AsyncResetReg(ready.asUInt, "ready_reg")(0)
+  io.enq.ready := ready_reg && sink_reset_n
 
   val widx_reg = AsyncResetReg(widx, "widx_gray")
   io.widx := widx_reg
 
   io.mem := mem
+
+  // It is a fatal error to reset half a Queue while it still has data
+  assert (sink_reset_n || widx === ridx)
 }
 
 class AsyncQueueSink[T <: Data](gen: T, depth: Int, sync: Int) extends Module {
@@ -63,10 +74,17 @@ class AsyncQueueSink[T <: Data](gen: T, depth: Int, sync: Int) extends Module {
     val ridx = UInt(OUTPUT, width = bits+1)
     val widx = UInt(INPUT,  width = bits+1)
     val mem  = Vec(depth, gen).asInput
+    // Reset for the other side
+    val source_reset_n = Bool().flip
   }
 
-  val ridx = GrayCounter(bits+1, io.deq.fire(), "ridx_bin")
+  // extend the source reset to a full cycle (assertion latency <= 1 cycle)
-  val widx = AsyncGrayCounter(io.widx, sync, "widx_gray")
+  val catch_source_reset_n = AsyncResetReg(Bool(true), clock, !io.source_reset_n, "catch_source_reset_n")
+  // reset_n has a 1 cycle shorter path to valid than widx does
+  val source_reset_n = UIntSyncChain(catch_source_reset_n.asUInt, sync, "source_reset_n")(0)
+
+  val ridx = GrayCounter(bits+1, io.deq.fire(), !source_reset_n, "ridx_bin")
+  val widx = UIntSyncChain(io.widx, sync, "widx_gray")
   val valid = ridx =/= widx
 
   // The mux is safe because timing analysis ensures ridx has reached the register
@@ -76,12 +94,18 @@ class AsyncQueueSink[T <: Data](gen: T, depth: Int, sync: Int) extends Module {
   val index = if (depth == 1) UInt(0) else ridx(bits-1, 0) ^ (ridx(bits, bits) << (bits-1))
   // This register does not NEED to be reset, as its contents will not
   // be considered unless the asynchronously reset deq valid register is set.
-  val data = RegEnable(io.mem(index), valid)
+  // It is possible that bits latches when the source domain is reset / has power cut
-  io.deq.bits := data
+  // This is safe, because isolation gates brought mem low before the zeroed widx reached us
-    
+  io.deq.bits  := RegEnable(io.mem(index), valid)
-  io.deq.valid := AsyncResetReg(valid, "valid_reg")
 
-  io.ridx := AsyncResetReg(ridx, "ridx_gray")
+  val valid_reg = AsyncResetReg(valid.asUInt, "valid_reg")(0)
+  io.deq.valid := valid_reg && source_reset_n
+
+  val ridx_reg = AsyncResetReg(ridx, "ridx_gray")
+  io.ridx := ridx_reg
+
+  // It is a fatal error to reset half a Queue while it still has data
+  assert (source_reset_n || widx === ridx)
 }
 
 class AsyncQueue[T <: Data](gen: T, depth: Int = 8, sync: Int = 3) extends Crossing[T] {
@@ -97,6 +121,9 @@ class AsyncQueue[T <: Data](gen: T, depth: Int = 8, sync: Int = 3) extends Cross
   sink.clock := io.deq_clock
   sink.reset := io.deq_reset
 
+  source.io.sink_reset_n := !io.deq_reset
+  sink.io.source_reset_n := !io.enq_reset
+
   source.io.enq <> io.enq
   io.deq <> sink.io.deq
 

src/main/scala/util/Crossing.scala

@@ -7,7 +7,7 @@ class CrossingIO[T <: Data](gen: T) extends Bundle {
   // Enqueue clock domain
   val enq_clock = Clock(INPUT)
   val enq_reset = Bool(INPUT) // synchronously deasserted wrt. enq_clock
-  val enq = Decoupled(gen).flip()
+  val enq = Decoupled(gen).flip
   // Dequeue clock domain
   val deq_clock = Clock(INPUT)
   val deq_reset = Bool(INPUT) // synchronously deasserted wrt. deq_clock