diff --git a/src/main/scala/diplomacy/Nodes.scala b/src/main/scala/diplomacy/Nodes.scala index 3796f5ec..3e739048 100644 --- a/src/main/scala/diplomacy/Nodes.scala +++ b/src/main/scala/diplomacy/Nodes.scala @@ -50,8 +50,21 @@ abstract class BaseNode protected[diplomacy] def colour: String } -trait InwardNode[DI, UI, BI <: Data] extends BaseNode +case class NodeHandle[DI, UI, BI <: Data, DO, UO, BO <: Data] + (inward: InwardNode[DI, UI, BI], outward: OutwardNode[DO, UO, BO]) + extends Object with InwardNodeHandle[DI, UI, BI] with OutwardNodeHandle[DO, UO, BO] + +trait InwardNodeHandle[DI, UI, BI <: Data] { + val inward: InwardNode[DI, UI, BI] + def := (h: OutwardNodeHandle[DI, UI, BI])(implicit sourceInfo: SourceInfo): Option[LazyModule] = + inward.:=(h)(sourceInfo) +} + +trait InwardNode[DI, UI, BI <: Data] extends BaseNode with InwardNodeHandle[DI, UI, BI] +{ + val inward = this + protected[diplomacy] val numPI: Range.Inclusive require (!numPI.isEmpty, s"No number of inputs would be acceptable to ${name}${lazyModule.line}") require (numPI.start >= 0, s"${name} accepts a negative number of inputs${lazyModule.line}") @@ -75,8 +88,15 @@ trait InwardNode[DI, UI, BI <: Data] extends BaseNode protected[diplomacy] def iConnect: Vec[BI] } -trait OutwardNode[DO, UO, BO <: Data] extends BaseNode +trait OutwardNodeHandle[DO, UO, BO <: Data] { + val outward: OutwardNode[DO, UO, BO] +} + +trait OutwardNode[DO, UO, BO <: Data] extends BaseNode with OutwardNodeHandle[DO, UO, BO] +{ + val outward = this + protected[diplomacy] val numPO: Range.Inclusive require (!numPO.isEmpty, s"No number of outputs would be acceptable to ${name}${lazyModule.line}") require (numPO.start >= 0, s"${name} accepts a negative number of outputs${lazyModule.line}") @@ -136,8 +156,9 @@ class MixedNode[DI, UI, EI, BI <: Data, DO, UO, EO, BO <: Data]( def iConnect = bundleIn // connects the outward part of a node with the inward part of this node - def := (y: OutwardNode[DI, UI, BI])(implicit sourceInfo: SourceInfo): Option[LazyModule] = { + override def := (h: OutwardNodeHandle[DI, UI, BI])(implicit sourceInfo: SourceInfo): Option[LazyModule] = { val x = this // x := y + val y = h.outward val info = sourceLine(sourceInfo, " at ", "") require (!LazyModule.stack.isEmpty, s"${y.name} cannot be connected to ${x.name} outside of LazyModule scope" + info) val i = x.iPushed diff --git a/src/main/scala/regmapper/RegisterCrossing.scala b/src/main/scala/regmapper/RegisterCrossing.scala index 575d3a85..b0e7f4bd 100644 --- a/src/main/scala/regmapper/RegisterCrossing.scala +++ b/src/main/scala/regmapper/RegisterCrossing.scala @@ -7,47 +7,79 @@ import chisel3.util.{Irrevocable} import util.{AsyncQueue,AsyncResetRegVec} // A very simple flow control state machine, run in the specified clock domain -class BusyRegisterCrossing(clock: Clock, reset: Bool) - extends Module(_clock = clock, _reset = reset) { +class BusyRegisterCrossing extends Module { val io = new Bundle { - val progress = Bool(INPUT) - val request_valid = Bool(INPUT) - val response_ready = Bool(INPUT) - val busy = Bool(OUTPUT) + val bypass = Bool(INPUT) + val master_request_valid = Bool(INPUT) + val master_request_ready = Bool(OUTPUT) + val master_response_valid = Bool(OUTPUT) + val master_response_ready = Bool(INPUT) + val crossing_request_valid = Bool(OUTPUT) + val crossing_request_ready = Bool(INPUT) + // ... no crossing_response_ready; we are always ready } - val busy = RegInit(Bool(false)) - when (io.progress) { - busy := Mux(busy, !io.response_ready, io.request_valid) + val busy = RegInit(Bool(false)) + val bypass = Reg(Bool()) + + when (io.crossing_request_ready || Mux(busy, bypass, io.bypass)) { + busy := Mux(busy, !io.master_response_ready, io.master_request_valid) } - io.busy := busy + + when (io.master_request_valid && io.master_request_ready) { + bypass := io.bypass + } + + io.crossing_request_valid := io.master_request_valid && !io.bypass && !busy + io.master_request_ready := (io.bypass || io.crossing_request_ready) && !busy + io.master_response_valid := (bypass || io.crossing_request_ready) && busy +} + +class RegisterCrossingAssertion extends Module { + val io = new Bundle { + val master_bypass = Bool(INPUT) + val slave_reset = Bool(INPUT) + } + + assert (io.master_bypass || !io.slave_reset) } // RegField should support connecting to one of these class RegisterWriteIO[T <: Data](gen: T) extends Bundle { - val request = Decoupled(gen).flip() + val request = Decoupled(gen).flip val response = Irrevocable(Bool()) // ignore .bits } -// To turn on/off a domain: -// 1. lower allow on the other side -// 2. wait for inflight traffic to resolve -// 3. assert reset in the domain -// 4. turn off the domain -// 5. turn on the domain -// 6. deassert reset in the domain -// 7. raise allow on the other side +// To turn off=>on a domain: +// A. To turn disable the master domain +// 1. wait for all inflight traffic to resolve +// 2. assert master reset +// 3. (optional) stop the master clock +// --- YOU MAY NOT TURN OFF POWER --- +// 4. re-enable the clock +// 5. deassert reset +// B. To turn off the slave domain +// 1. assert bypass +// 2. wait for inflight traffic to resolve +// 3. assert slave reset +// 4. (optional) stop the slave clock +// --- YOU MAY NOT TURN OFF POWER --- +// 5. re-enable the clock +// 6. deassert reset +// 7. deassert bypass +// +// If you need to cut power, use something that support isolation gates. class RegisterWriteCrossingIO[T <: Data](gen: T) extends Bundle { // Master clock domain val master_clock = Clock(INPUT) val master_reset = Bool(INPUT) - val master_allow = Bool(INPUT) // actually wait for the slave val master_port = new RegisterWriteIO(gen) + // Bypass requests from the master to be noops + val master_bypass = Bool(INPUT) // Slave clock domain val slave_clock = Clock(INPUT) val slave_reset = Bool(INPUT) - val slave_allow = Bool(INPUT) // honour requests from the master val slave_register = gen.asOutput val slave_valid = Bool(OUTPUT) // is high on 1st cycle slave_register has a new value } @@ -55,37 +87,40 @@ class RegisterWriteCrossingIO[T <: Data](gen: T) extends Bundle { class RegisterWriteCrossing[T <: Data](gen: T, sync: Int = 3) extends Module { val io = new RegisterWriteCrossingIO(gen) // The crossing must only allow one item inflight at a time + val control = Module(new BusyRegisterCrossing) val crossing = Module(new AsyncQueue(gen, 1, sync)) - // We can just randomly reset one-side of a single entry AsyncQueue. - // If the enq side is reset, at worst deq.bits is reassigned from mem(0), which stays fixed. - // If the deq side is reset, at worst the master rewrites mem(0) once, deq.bits stays fixed. + control.clock := io.master_clock + control.reset := io.master_reset crossing.io.enq_clock := io.master_clock crossing.io.enq_reset := io.master_reset crossing.io.deq_clock := io.slave_clock crossing.io.deq_reset := io.slave_reset + control.io.bypass := io.master_bypass + control.io.master_request_valid := io.master_port.request.valid + control.io.master_response_ready := io.master_port.response.ready + io.master_port.request.ready := control.io.master_request_ready + io.master_port.response.valid := control.io.master_response_valid + + control.io.crossing_request_ready := crossing.io.enq.ready + crossing.io.enq.valid := control.io.crossing_request_valid crossing.io.enq.bits := io.master_port.request.bits - io.slave_register := crossing.io.deq.bits - io.slave_valid := crossing.io.deq.valid - - // If the slave is not operational, just drop the write. - val progress = crossing.io.enq.ready || !io.master_allow - - val control = Module(new BusyRegisterCrossing(io.master_clock, io.master_reset)) - control.io.progress := progress - control.io.request_valid := io.master_port.request.valid - control.io.response_ready := io.master_port.response.ready crossing.io.deq.ready := Bool(true) - crossing.io.enq.valid := io.master_port.request.valid && !control.io.busy - io.master_port.request.ready := progress && !control.io.busy - io.master_port.response.valid := progress && control.io.busy + io.slave_valid := crossing.io.deq.valid + io.slave_register := crossing.io.deq.bits + + val assertion = Module(new RegisterCrossingAssertion) + assertion.clock := io.master_clock + assertion.reset := io.master_reset + assertion.io.master_bypass := io.master_bypass + assertion.io.slave_reset := io.slave_reset } // RegField should support connecting to one of these class RegisterReadIO[T <: Data](gen: T) extends Bundle { - val request = Decoupled(Bool()).flip() // ignore .bits + val request = Decoupled(Bool()).flip // ignore .bits val response = Irrevocable(gen) } @@ -93,43 +128,46 @@ class RegisterReadCrossingIO[T <: Data](gen: T) extends Bundle { // Master clock domain val master_clock = Clock(INPUT) val master_reset = Bool(INPUT) - val master_allow = Bool(INPUT) // actually wait for the slave val master_port = new RegisterReadIO(gen) + // Bypass requests from the master to be noops + val master_bypass = Bool(INPUT) // Slave clock domain val slave_clock = Clock(INPUT) val slave_reset = Bool(INPUT) - val slave_allow = Bool(INPUT) // honour requests from the master val slave_register = gen.asInput } class RegisterReadCrossing[T <: Data](gen: T, sync: Int = 3) extends Module { val io = new RegisterReadCrossingIO(gen) // The crossing must only allow one item inflight at a time + val control = Module(new BusyRegisterCrossing) val crossing = Module(new AsyncQueue(gen, 1, sync)) - // We can just randomly reset one-side of a single entry AsyncQueue. - // If the enq side is reset, at worst deq.bits is reassigned from mem(0), which stays fixed. - // If the deq side is reset, at worst the slave rewrites mem(0) once, deq.bits stays fixed. - crossing.io.enq_clock := io.slave_clock - crossing.io.enq_reset := io.slave_reset + control.clock := io.master_clock + control.reset := io.master_reset crossing.io.deq_clock := io.master_clock crossing.io.deq_reset := io.master_reset + crossing.io.enq_clock := io.slave_clock + crossing.io.enq_reset := io.slave_reset - crossing.io.enq.bits := io.slave_register + control.io.bypass := io.master_bypass + control.io.master_request_valid := io.master_port.request.valid + control.io.master_response_ready := io.master_port.response.ready + io.master_port.request.ready := control.io.master_request_ready + io.master_port.response.valid := control.io.master_response_valid + + control.io.crossing_request_ready := crossing.io.deq.valid + crossing.io.deq.ready := control.io.crossing_request_valid io.master_port.response.bits := crossing.io.deq.bits - // If the slave is not operational, just repeat the last value we saw. - val progress = crossing.io.deq.valid || !io.master_allow - - val control = Module(new BusyRegisterCrossing(io.master_clock, io.master_reset)) - control.io.progress := progress - control.io.request_valid := io.master_port.request.valid - control.io.response_ready := io.master_port.response.ready - - io.master_port.response.valid := progress && control.io.busy - io.master_port.request.ready := progress && !control.io.busy - crossing.io.deq.ready := io.master_port.request.valid && !control.io.busy crossing.io.enq.valid := Bool(true) + crossing.io.enq.bits := io.slave_register + + val assertion = Module(new RegisterCrossingAssertion) + assertion.clock := io.master_clock + assertion.reset := io.master_reset + assertion.io.master_bypass := io.master_bypass + assertion.io.slave_reset := io.slave_reset } /** Wrapper to create an @@ -151,8 +189,7 @@ object AsyncRWSlaveRegField { width: Int, init: Int, name: Option[String] = None, - master_allow: Bool = Bool(true), - slave_allow: Bool = Bool(true) + master_bypass: Bool = Bool(true) ): (UInt, RegField) = { val async_slave_reg = Module(new AsyncResetRegVec(width, init)) @@ -163,12 +200,11 @@ object AsyncRWSlaveRegField { val wr_crossing = Module (new RegisterWriteCrossing(UInt(width = width))) name.foreach(n => wr_crossing.suggestName(s"${n}_wcrossing")) - wr_crossing.io.master_clock := master_clock - wr_crossing.io.master_reset := master_reset - wr_crossing.io.master_allow := master_allow - wr_crossing.io.slave_clock := slave_clock - wr_crossing.io.slave_reset := slave_reset - wr_crossing.io.slave_allow := slave_allow + wr_crossing.io.master_clock := master_clock + wr_crossing.io.master_reset := master_reset + wr_crossing.io.master_bypass := master_bypass + wr_crossing.io.slave_clock := slave_clock + wr_crossing.io.slave_reset := slave_reset async_slave_reg.io.en := wr_crossing.io.slave_valid async_slave_reg.io.d := wr_crossing.io.slave_register @@ -176,12 +212,11 @@ object AsyncRWSlaveRegField { val rd_crossing = Module (new RegisterReadCrossing(UInt(width = width ))) name.foreach(n => rd_crossing.suggestName(s"${n}_rcrossing")) - rd_crossing.io.master_clock := master_clock - rd_crossing.io.master_reset := master_reset - rd_crossing.io.master_allow := master_allow - rd_crossing.io.slave_clock := slave_clock - rd_crossing.io.slave_reset := slave_reset - rd_crossing.io.slave_allow := slave_allow + rd_crossing.io.master_clock := master_clock + rd_crossing.io.master_reset := master_reset + rd_crossing.io.master_bypass := master_bypass + rd_crossing.io.slave_clock := slave_clock + rd_crossing.io.slave_reset := slave_reset rd_crossing.io.slave_register := async_slave_reg.io.q diff --git a/src/main/scala/uncore/axi4/package.scala b/src/main/scala/uncore/axi4/package.scala index 4ea7c9ab..6a382b6d 100644 --- a/src/main/scala/uncore/axi4/package.scala +++ b/src/main/scala/uncore/axi4/package.scala @@ -5,5 +5,5 @@ import diplomacy._ package object axi4 { - type AXI4OutwardNode = OutwardNode[AXI4MasterPortParameters, AXI4SlavePortParameters, AXI4Bundle] + type AXI4OutwardNode = OutwardNodeHandle[AXI4MasterPortParameters, AXI4SlavePortParameters, AXI4Bundle] } diff --git a/src/main/scala/uncore/tilelink2/Bundles.scala b/src/main/scala/uncore/tilelink2/Bundles.scala index 32deff95..b90c4a2c 100644 --- a/src/main/scala/uncore/tilelink2/Bundles.scala +++ b/src/main/scala/uncore/tilelink2/Bundles.scala @@ -226,6 +226,9 @@ final class AsyncBundle[T <: Data](val depth: Int, gen: T) extends Bundle val ridx = UInt(width = log2Up(depth)+1).flip val widx = UInt(width = log2Up(depth)+1) val mem = Vec(depth, gen) + val source_reset_n = Bool() + val sink_reset_n = Bool().flip + override def cloneType: this.type = new AsyncBundle(depth, gen).asInstanceOf[this.type] } @@ -236,6 +239,8 @@ object FromAsyncBundle x.ridx := sink.io.ridx sink.io.widx := x.widx sink.io.mem := x.mem + sink.io.source_reset_n := x.source_reset_n + x.sink_reset_n := !sink.reset val out = Wire(Irrevocable(x.mem(0))) out.valid := sink.io.deq.valid out.bits := sink.io.deq.bits @@ -255,6 +260,8 @@ object ToAsyncBundle source.io.ridx := out.ridx out.mem := source.io.mem out.widx := source.io.widx + source.io.sink_reset_n := out.sink_reset_n + out.source_reset_n := !source.reset out } } diff --git a/src/main/scala/uncore/tilelink2/Crossing.scala b/src/main/scala/uncore/tilelink2/Crossing.scala index 394fbcb1..3e81d926 100644 --- a/src/main/scala/uncore/tilelink2/Crossing.scala +++ b/src/main/scala/uncore/tilelink2/Crossing.scala @@ -18,16 +18,22 @@ class TLAsyncCrossingSource(sync: Int = 3) extends LazyModule } ((io.in zip io.out) zip (node.edgesIn zip node.edgesOut)) foreach { case ((in, out), (edgeIn, edgeOut)) => + val sink_reset_n = out.a.sink_reset_n val bce = edgeIn.manager.anySupportAcquire && edgeIn.client.anySupportProbe val depth = edgeOut.manager.depth out.a <> ToAsyncBundle(in.a, depth, sync) in.d <> FromAsyncBundle(out.d, sync) + assert (!in.a.valid || sink_reset_n, "A channel request sent to a missing manager") + if (bce) { in.b <> FromAsyncBundle(out.b, sync) out.c <> ToAsyncBundle(in.c, depth, sync) out.e <> ToAsyncBundle(in.e, depth, sync) + + assert (!in.c.valid || sink_reset_n, "C channel response sent to a missing manager") + assert (!in.e.valid || sink_reset_n, "E channel response sent to a missing manager") } else { in.b.valid := Bool(false) in.c.ready := Bool(true) @@ -51,15 +57,20 @@ class TLAsyncCrossingSink(depth: Int = 8, sync: Int = 3) extends LazyModule } ((io.in zip io.out) zip (node.edgesIn zip node.edgesOut)) foreach { case ((in, out), (edgeIn, edgeOut)) => + val source_reset_n = in.a.source_reset_n val bce = edgeOut.manager.anySupportAcquire && edgeOut.client.anySupportProbe out.a <> FromAsyncBundle(in.a, sync) in.d <> ToAsyncBundle(out.d, depth, sync) + assert (!out.d.valid || source_reset_n, "D channel respose sent to missing client") + if (bce) { in.b <> ToAsyncBundle(out.b, depth, sync) out.c <> FromAsyncBundle(in.c, sync) out.e <> FromAsyncBundle(in.e, sync) + + assert (!out.b.valid || source_reset_n, "B channel request sent to missing client") } else { in.b.widx := UInt(0) in.c.ridx := UInt(0) @@ -96,6 +107,7 @@ class TLAsyncCrossing(depth: Int = 8, sync: Int = 3) extends LazyModule { val nodeIn = TLInputNode() val nodeOut = TLOutputNode() + val node = NodeHandle(nodeIn, nodeOut) val source = LazyModule(new TLAsyncCrossingSource(sync)) val sink = LazyModule(new TLAsyncCrossingSink(depth, sync)) @@ -140,8 +152,8 @@ class TLRAMCrossing extends LazyModule { val cross = LazyModule(new TLAsyncCrossing) model.node := fuzz.node - cross.nodeIn := TLFragmenter(4, 256)(model.node) - val monitor = (ram.node := cross.nodeOut) + cross.node := TLFragmenter(4, 256)(model.node) + val monitor = (ram.node := cross.node) lazy val module = new LazyModuleImp(this) with HasUnitTestIO { io.finished := fuzz.module.io.finished diff --git a/src/main/scala/uncore/tilelink2/Edges.scala b/src/main/scala/uncore/tilelink2/Edges.scala index aac8338c..ee5a8bcb 100644 --- a/src/main/scala/uncore/tilelink2/Edges.scala +++ b/src/main/scala/uncore/tilelink2/Edges.scala @@ -4,6 +4,7 @@ package uncore.tilelink2 import Chisel._ import chisel3.internal.sourceinfo.SourceInfo +import chisel3.util.IrrevocableIO import diplomacy._ class TLEdge( @@ -218,6 +219,20 @@ class TLEdge( } } } + + def firstlast(bits: TLChannel, fire: Bool): (Bool, Bool, UInt) = { + val beats1 = numBeats1(bits) + val counter = RegInit(UInt(0, width = log2Up(maxTransfer / manager.beatBytes))) + val counter1 = counter - UInt(1) + val first = counter === UInt(0) + val last = counter === UInt(1) || beats1 === UInt(0) + when (fire) { + counter := Mux(first, beats1, counter1) + } + (first, last, beats1 & ~counter1) + } + + def firstlast(x: IrrevocableIO[TLChannel]): (Bool, Bool, UInt) = firstlast(x.bits, x.fire()) } class TLEdgeOut( diff --git a/src/main/scala/uncore/tilelink2/Fuzzer.scala b/src/main/scala/uncore/tilelink2/Fuzzer.scala index 33e4a13e..4063985d 100644 --- a/src/main/scala/uncore/tilelink2/Fuzzer.scala +++ b/src/main/scala/uncore/tilelink2/Fuzzer.scala @@ -113,19 +113,11 @@ class TLFuzzer( // Progress within each operation val a = out.a.bits - val a_beats1 = edge.numBeats1(a) - val a_counter = RegInit(UInt(0, width = maxLgBeats)) - val a_counter1 = a_counter - UInt(1) - val a_first = a_counter === UInt(0) - val a_last = a_counter === UInt(1) || a_beats1 === UInt(0) + val (a_first, a_last, _) = edge.firstlast(out.a) val req_done = out.a.fire() && a_last val d = out.d.bits - val d_beats1 = edge.numBeats1(d) - val d_counter = RegInit(UInt(0, width = maxLgBeats)) - val d_counter1 = d_counter - UInt(1) - val d_first = d_counter === UInt(0) - val d_last = d_counter === UInt(1) || d_beats1 === UInt(0) + val (d_first, d_last, _) = edge.firstlast(out.d) val resp_done = out.d.fire() && d_last // Source ID generation @@ -199,14 +191,12 @@ class TLFuzzer( inc := !legal || req_done inc_beat := !legal || out.a.fire() - when (out.a.fire()) { - a_counter := Mux(a_first, a_beats1, a_counter1) - when(a_last) { num_reqs := num_reqs - UInt(1) } + when (out.a.fire() && a_last) { + num_reqs := num_reqs - UInt(1) } - when (out.d.fire()) { - d_counter := Mux(d_first, d_beats1, d_counter1) - when(d_last) { num_resps := num_resps - UInt(1) } + when (out.d.fire() && d_last) { + num_resps := num_resps - UInt(1) } } } @@ -229,8 +219,8 @@ class TLFuzzRAM extends LazyModule xbar2.node := TLAtomicAutomata()(model.node) ram2.node := TLFragmenter(16, 256)(xbar2.node) xbar.node := TLWidthWidget(16)(TLHintHandler()(xbar2.node)) - cross.nodeIn := TLFragmenter(4, 256)(TLBuffer()(xbar.node)) - val monitor = (ram.node := cross.nodeOut) + cross.node := TLFragmenter(4, 256)(TLBuffer()(xbar.node)) + val monitor = (ram.node := cross.node) gpio.node := TLFragmenter(4, 32)(TLBuffer()(xbar.node)) lazy val module = new LazyModuleImp(this) with HasUnitTestIO { diff --git a/src/main/scala/uncore/tilelink2/HintHandler.scala b/src/main/scala/uncore/tilelink2/HintHandler.scala index bda7f2e4..61161f9c 100644 --- a/src/main/scala/uncore/tilelink2/HintHandler.scala +++ b/src/main/scala/uncore/tilelink2/HintHandler.scala @@ -33,38 +33,20 @@ class TLHintHandler(supportManagers: Boolean = true, supportClients: Boolean = f val smartClients = edgeIn.client.clients.map(_.supportsHint.max == edgeIn.client.maxTransfer).reduce(_&&_) val smartManagers = edgeOut.manager.managers.map(_.supportsHint.max == edgeOut.manager.maxTransfer).reduce(_&&_) - if (supportManagers && !smartManagers) { - // State of the Hint bypass - val counter = RegInit(UInt(0, width = log2Up(edgeOut.manager.maxTransfer/edgeOut.manager.beatBytes))) - val hintHoldsD = RegInit(Bool(false)) - val outerHoldsD = counter =/= UInt(0) - // Only one of them can hold it - assert (!hintHoldsD || !outerHoldsD) - - // Count outer D beats - val beats1 = edgeOut.numBeats1(out.d.bits) - when (out.d.fire()) { counter := Mux(outerHoldsD, counter - UInt(1), beats1) } - - // Who wants what? + if (supportManagers && !(passthrough && smartManagers)) { val address = edgeIn.address(in.a.bits) val handleA = if (passthrough) !edgeOut.manager.supportsHintFast(address, edgeIn.size(in.a.bits)) else Bool(true) val hintBitsAtA = handleA && in.a.bits.opcode === TLMessages.Hint - val hintWantsD = in.a.valid && hintBitsAtA - val outerWantsD = out.d.valid + val hint = Wire(out.d) - // Prioritize existing D traffic over HintAck (and finish multibeat xfers) - val hintWinsD = hintHoldsD || (!outerHoldsD && !outerWantsD) - hintHoldsD := hintWantsD && hintWinsD && !in.d.ready - // Hint can only hold D b/c it still wants it from last cycle - assert (!hintHoldsD || hintWantsD) - - in.d.valid := Mux(hintWinsD, hintWantsD, outerWantsD) - in.d.bits := Mux(hintWinsD, edgeIn.HintAck(in.a.bits, edgeOut.manager.findIdStartFast(address)), out.d.bits) - out.d.ready := in.d.ready && !hintHoldsD - - in.a.ready := Mux(hintBitsAtA, hintWinsD && in.d.ready, out.a.ready) + hint.valid := in.a.valid && hintBitsAtA out.a.valid := in.a.valid && !hintBitsAtA - out.a.bits := in.a.bits + in.a.ready := Mux(hintBitsAtA, hint.ready, out.a.ready) + + hint.bits := edgeIn.HintAck(in.a.bits, edgeOut.manager.findIdStartFast(address)) + out.a.bits := in.a.bits + + TLArbiter(TLArbiter.lowestIndexFirst)(in.d, (edgeOut.numBeats(out.d.bits), out.d), (UInt(1), hint)) } else { out.a.valid := in.a.valid in.a.ready := out.a.ready @@ -75,37 +57,19 @@ class TLHintHandler(supportManagers: Boolean = true, supportClients: Boolean = f in.d.bits := out.d.bits } - if (supportClients && !smartClients) { - // State of the Hint bypass - val counter = RegInit(UInt(0, width = log2Up(edgeIn.client.maxTransfer/edgeIn.manager.beatBytes))) - val hintHoldsC = RegInit(Bool(false)) - val innerHoldsC = counter =/= UInt(0) - // Only one of them can hold it - assert (!hintHoldsC || !innerHoldsC) - - // Count inner C beats - val beats1 = edgeIn.numBeats1(in.c.bits) - when (in.c.fire()) { counter := Mux(innerHoldsC, counter - UInt(1), beats1) } - - // Who wants what? + if (supportClients && !(passthrough && smartClients)) { val handleB = if (passthrough) !edgeIn.client.supportsHint(out.b.bits.source, edgeOut.size(out.b.bits)) else Bool(true) val hintBitsAtB = handleB && out.b.bits.opcode === TLMessages.Hint - val hintWantsC = out.b.valid && hintBitsAtB - val innerWantsC = in.c.valid + val hint = Wire(in.c) - // Prioritize existing C traffic over HintAck (and finish multibeat xfers) - val hintWinsC = hintHoldsC || (!innerHoldsC && !innerWantsC) - hintHoldsC := hintWantsC && hintWinsC && !out.c.ready - // Hint can only hold C b/c it still wants it from last cycle - assert (!hintHoldsC || hintWantsC) + hint.valid := out.b.valid && hintBitsAtB + in.b.valid := out.b.valid && !hintBitsAtB + out.b.ready := Mux(hintBitsAtB, hint.ready, in.b.ready) - out.c.valid := Mux(hintWinsC, hintWantsC, innerWantsC) - out.c.bits := Mux(hintWinsC, edgeOut.HintAck(out.b.bits), in.c.bits) - in.c.ready := out.c.ready && !hintHoldsC + hint.bits := edgeOut.HintAck(out.b.bits) + in.b.bits := out.b.bits - out.b.ready := Mux(hintBitsAtB, hintWinsC && out.c.ready, in.b.ready) - in.b.valid := out.b.valid && !hintBitsAtB - in.b.bits := out.b.bits + TLArbiter(TLArbiter.lowestIndexFirst)(out.c, (edgeIn.numBeats(in.c.bits), in.c), (UInt(1), hint)) } else if (bce) { in.b.valid := out.b.valid out.b.ready := in.b.ready diff --git a/src/main/scala/uncore/tilelink2/Isolation.scala b/src/main/scala/uncore/tilelink2/Isolation.scala index 3e9d0757..3f02053e 100644 --- a/src/main/scala/uncore/tilelink2/Isolation.scala +++ b/src/main/scala/uncore/tilelink2/Isolation.scala @@ -6,6 +6,7 @@ import Chisel._ import chisel3.internal.sourceinfo.SourceInfo import diplomacy._ +// READ the comments in the TLIsolation object before you instantiate this module class TLIsolation(fOut: (Bool, UInt) => UInt, fIn: (Bool, UInt) => UInt) extends LazyModule { val node = TLAsyncIdentityNode() @@ -30,6 +31,11 @@ class TLIsolation(fOut: (Bool, UInt) => UInt, fIn: (Bool, UInt) => UInt) extends in .d.widx := ISOi(out.d.widx) in .d.mem := ISOi(out.d.mem) + out.a.source_reset_n := ISOo(in .a.source_reset_n) + in .a.sink_reset_n := ISOi(out.a.sink_reset_n) + out.d.sink_reset_n := ISOo(in .d.sink_reset_n) + in .d.source_reset_n := ISOi(out.d.source_reset_n) + if (edgeOut.manager.base.anySupportAcquire && edgeOut.client.base.anySupportProbe) { in .b.widx := ISOi(out.b.widx) in .c.ridx := ISOi(out.c.ridx) @@ -40,6 +46,13 @@ class TLIsolation(fOut: (Bool, UInt) => UInt, fIn: (Bool, UInt) => UInt) extends in .b.mem := ISOi(out.b.mem) out.c.mem := ISOo(in .c.mem) out.e.mem := ISOo(in .e.mem) + + out.b.sink_reset_n := ISOo(in .b.sink_reset_n) + in .b.source_reset_n := ISOi(out.b.source_reset_n) + out.c.source_reset_n := ISOo(in .c.source_reset_n) + in .c.sink_reset_n := ISOi(out.c.sink_reset_n) + out.e.source_reset_n := ISOo(in .e.source_reset_n) + in .e.sink_reset_n := ISOi(out.e.sink_reset_n) } else { in .b.widx := UInt(0) in .c.ridx := UInt(0) @@ -55,9 +68,10 @@ class TLIsolation(fOut: (Bool, UInt) => UInt, fIn: (Bool, UInt) => UInt) extends object TLIsolation { // applied to the TL source node; y.node := TLIsolation(fOut, fIn)(x.node) - // f should insert an isolation gate between the input UInt and its result - // fOut is applied for data flowing from client to manager - // fIn is applied for data flowing from manager to client + // f* should insert an isolation gate between the input UInt and its result + // fOut is applied to data flowing from client to manager + // fIn is applied to data flowing from manager to client + // **** WARNING: the isolation functions must bring the values to 0 **** def apply(fOut: (Bool, UInt) => UInt, fIn: (Bool, UInt) => UInt)(x: TLAsyncOutwardNode)(implicit sourceInfo: SourceInfo): (TLAsyncOutwardNode, () => (Bool, Bool)) = { val iso = LazyModule(new TLIsolation(fOut, fIn)) iso.node := x diff --git a/src/main/scala/uncore/tilelink2/Monitor.scala b/src/main/scala/uncore/tilelink2/Monitor.scala index c0fc51a8..e6dbab2b 100644 --- a/src/main/scala/uncore/tilelink2/Monitor.scala +++ b/src/main/scala/uncore/tilelink2/Monitor.scala @@ -282,68 +282,60 @@ class TLMonitor(gen: () => TLBundleSnoop, edge: () => TLEdge, sourceInfo: Source } def legalizeMultibeatA(a: IrrevocableSnoop[TLBundleA], edge: TLEdge)(implicit sourceInfo: SourceInfo) { - val counter = RegInit(UInt(0, width = log2Up(edge.maxTransfer))) + val (a_first, _, _) = edge.firstlast(a.bits, a.fire()) val opcode = Reg(UInt()) val param = Reg(UInt()) val size = Reg(UInt()) val source = Reg(UInt()) val addr_hi = Reg(UInt()) - when (a.valid && counter =/= UInt(0)) { + when (a.valid && !a_first) { assert (a.bits.opcode === opcode, "'A' channel opcode changed within multibeat operation" + extra) assert (a.bits.param === param, "'A' channel param changed within multibeat operation" + extra) assert (a.bits.size === size, "'A' channel size changed within multibeat operation" + extra) assert (a.bits.source === source, "'A' channel source changed within multibeat operation" + extra) assert (a.bits.addr_hi=== addr_hi,"'A' channel addr_hi changed with multibeat operation" + extra) } - when (a.fire()) { - counter := counter - UInt(1) - when (counter === UInt(0)) { - counter := edge.numBeats(a.bits) - UInt(1) - opcode := a.bits.opcode - param := a.bits.param - size := a.bits.size - source := a.bits.source - addr_hi := a.bits.addr_hi - } + when (a.fire() && a_first) { + opcode := a.bits.opcode + param := a.bits.param + size := a.bits.size + source := a.bits.source + addr_hi := a.bits.addr_hi } } def legalizeMultibeatB(b: IrrevocableSnoop[TLBundleB], edge: TLEdge)(implicit sourceInfo: SourceInfo) { - val counter = RegInit(UInt(0, width = log2Up(edge.maxTransfer))) + val (b_first, _, _) = edge.firstlast(b.bits, b.fire()) val opcode = Reg(UInt()) val param = Reg(UInt()) val size = Reg(UInt()) val source = Reg(UInt()) val addr_hi = Reg(UInt()) - when (b.valid && counter =/= UInt(0)) { + when (b.valid && !b_first) { assert (b.bits.opcode === opcode, "'B' channel opcode changed within multibeat operation" + extra) assert (b.bits.param === param, "'B' channel param changed within multibeat operation" + extra) assert (b.bits.size === size, "'B' channel size changed within multibeat operation" + extra) assert (b.bits.source === source, "'B' channel source changed within multibeat operation" + extra) assert (b.bits.addr_hi=== addr_hi,"'B' channel addr_hi changed with multibeat operation" + extra) } - when (b.fire()) { - counter := counter - UInt(1) - when (counter === UInt(0)) { - counter := edge.numBeats(b.bits) - UInt(1) - opcode := b.bits.opcode - param := b.bits.param - size := b.bits.size - source := b.bits.source - addr_hi := b.bits.addr_hi - } + when (b.fire() && b_first) { + opcode := b.bits.opcode + param := b.bits.param + size := b.bits.size + source := b.bits.source + addr_hi := b.bits.addr_hi } } def legalizeMultibeatC(c: IrrevocableSnoop[TLBundleC], edge: TLEdge)(implicit sourceInfo: SourceInfo) { - val counter = RegInit(UInt(0, width = log2Up(edge.maxTransfer))) + val (c_first, _, _) = edge.firstlast(c.bits, c.fire()) val opcode = Reg(UInt()) val param = Reg(UInt()) val size = Reg(UInt()) val source = Reg(UInt()) val addr_hi = Reg(UInt()) val addr_lo = Reg(UInt()) - when (c.valid && counter =/= UInt(0)) { + when (c.valid && !c_first) { assert (c.bits.opcode === opcode, "'C' channel opcode changed within multibeat operation" + extra) assert (c.bits.param === param, "'C' channel param changed within multibeat operation" + extra) assert (c.bits.size === size, "'C' channel size changed within multibeat operation" + extra) @@ -351,29 +343,25 @@ class TLMonitor(gen: () => TLBundleSnoop, edge: () => TLEdge, sourceInfo: Source assert (c.bits.addr_hi=== addr_hi,"'C' channel addr_hi changed with multibeat operation" + extra) assert (c.bits.addr_lo=== addr_lo,"'C' channel addr_lo changed with multibeat operation" + extra) } - when (c.fire()) { - counter := counter - UInt(1) - when (counter === UInt(0)) { - counter := edge.numBeats(c.bits) - UInt(1) - opcode := c.bits.opcode - param := c.bits.param - size := c.bits.size - source := c.bits.source - addr_hi := c.bits.addr_hi - addr_lo := c.bits.addr_lo - } + when (c.fire() && c_first) { + opcode := c.bits.opcode + param := c.bits.param + size := c.bits.size + source := c.bits.source + addr_hi := c.bits.addr_hi + addr_lo := c.bits.addr_lo } } def legalizeMultibeatD(d: IrrevocableSnoop[TLBundleD], edge: TLEdge)(implicit sourceInfo: SourceInfo) { - val counter = RegInit(UInt(0, width = log2Up(edge.maxTransfer))) + val (d_first, _, _) = edge.firstlast(d.bits, d.fire()) val opcode = Reg(UInt()) val param = Reg(UInt()) val size = Reg(UInt()) val source = Reg(UInt()) val sink = Reg(UInt()) val addr_lo = Reg(UInt()) - when (d.valid && counter =/= UInt(0)) { + when (d.valid && !d_first) { assert (d.bits.opcode === opcode, "'D' channel opcode changed within multibeat operation" + extra) assert (d.bits.param === param, "'D' channel param changed within multibeat operation" + extra) assert (d.bits.size === size, "'D' channel size changed within multibeat operation" + extra) @@ -381,17 +369,13 @@ class TLMonitor(gen: () => TLBundleSnoop, edge: () => TLEdge, sourceInfo: Source assert (d.bits.sink === sink, "'D' channel sink changed with multibeat operation" + extra) assert (d.bits.addr_lo=== addr_lo,"'D' channel addr_lo changed with multibeat operation" + extra) } - when (d.fire()) { - counter := counter - UInt(1) - when (counter === UInt(0)) { - counter := edge.numBeats(d.bits) - UInt(1) - opcode := d.bits.opcode - param := d.bits.param - size := d.bits.size - source := d.bits.source - sink := d.bits.sink - addr_lo := d.bits.addr_lo - } + when (d.fire() && d_first) { + opcode := d.bits.opcode + param := d.bits.param + size := d.bits.size + source := d.bits.source + sink := d.bits.sink + addr_lo := d.bits.addr_lo } } @@ -425,15 +409,8 @@ class TLMonitor(gen: () => TLBundleSnoop, edge: () => TLEdge, sourceInfo: Source def legalizeSourceUnique(bundle: TLBundleSnoop, edge: TLEdge)(implicit sourceInfo: SourceInfo) { val inflight = RegInit(UInt(0, width = edge.client.endSourceId)) - val a_counter = RegInit(UInt(0, width = log2Up(edge.maxTransfer))) - val a_beats1 = edge.numBeats1(bundle.a.bits) - val a_first = a_counter === UInt(0) - val a_last = a_counter === UInt(1) || a_beats1 === UInt(0) - - val d_counter = RegInit(UInt(0, width = log2Up(edge.maxTransfer))) - val d_beats1 = edge.numBeats1(bundle.d.bits) - val d_first = d_counter === UInt(0) - val d_last = d_counter === UInt(1) || d_beats1 === UInt(0) + val (_, a_last, _) = edge.firstlast(bundle.a.bits, bundle.a.fire()) + val (_, d_last, _) = edge.firstlast(bundle.d.bits, bundle.d.fire()) val bypass = bundle.a.bits.source === bundle.d.bits.source val a_bypass = bypass && bundle.d.valid && d_last @@ -445,14 +422,12 @@ class TLMonitor(gen: () => TLBundleSnoop, edge: () => TLEdge, sourceInfo: Source val a_set = Wire(init = UInt(0, width = edge.client.endSourceId)) when (bundle.a.fire()) { - a_counter := Mux(a_first, a_beats1, a_counter - UInt(1)) when (a_last) { a_set := UIntToOH(bundle.a.bits.source) } assert(a_bypass || !inflight(bundle.a.bits.source), "'A' channel re-used a source ID" + extra) } val d_clr = Wire(init = UInt(0, width = edge.client.endSourceId)) when (bundle.d.fire() && bundle.d.bits.opcode =/= TLMessages.ReleaseAck) { - d_counter := Mux(d_first, d_beats1, d_counter - UInt(1)) when (d_last) { d_clr := UIntToOH(bundle.d.bits.source) } assert(d_bypass || inflight(bundle.d.bits.source), "'D' channel acknowledged for nothing inflight" + extra) } diff --git a/src/main/scala/uncore/tilelink2/RAMModel.scala b/src/main/scala/uncore/tilelink2/RAMModel.scala index b8d46c61..cfeb8cce 100644 --- a/src/main/scala/uncore/tilelink2/RAMModel.scala +++ b/src/main/scala/uncore/tilelink2/RAMModel.scala @@ -110,13 +110,10 @@ class TLRAMModel extends LazyModule // Process A access requests val a = Reg(next = in.a.bits) val a_fire = Reg(next = in.a.fire(), init = Bool(false)) - val a_beats1 = edge.numBeats1(a) + val (a_first, a_last, a_address_inc) = edge.firstlast(a, a_fire) val a_size = edge.size(a) val a_sizeOH = UIntToOH(a_size) - val a_counter = RegInit(UInt(0, width = maxLgBeats)) - val a_counter1 = a_counter - UInt(1) - val a_first = a_counter === UInt(0) - val a_addr_hi = a.addr_hi | (a_beats1 & ~a_counter1) + val a_addr_hi = a.addr_hi | a_address_inc val a_base = edge.address(a) val a_mask = edge.mask(a_base, a_size) val a_fifo = edge.manager.hasFifoIdFast(a_base) @@ -133,8 +130,6 @@ class TLRAMModel extends LazyModule when (a_fire) { // Record the request so we can handle it's response - a_counter := Mux(a_first, a_beats1, a_counter1) - assert (a.opcode =/= TLMessages.Acquire) // Mark the operation as valid @@ -199,15 +194,11 @@ class TLRAMModel extends LazyModule // Process D access responses val d = RegNext(out.d.bits) val d_fire = Reg(next = out.d.fire(), init = Bool(false)) - val d_beats1 = edge.numBeats1(d) + val (d_first, d_last, d_address_inc) = edge.firstlast(d, d_fire) val d_size = edge.size(d) val d_sizeOH = UIntToOH(d_size) - val d_counter = RegInit(UInt(0, width = maxLgBeats)) - val d_counter1 = d_counter - UInt(1) - val d_first = d_counter === UInt(0) - val d_last = d_counter === UInt(1) || d_beats1 === UInt(0) val d_base = d_flight.base - val d_addr_hi = d_base >> shift | (d_beats1 & ~d_counter1) + val d_addr_hi = d_base >> shift | d_address_inc val d_mask = edge.mask(d_base, d_size) val d_fifo = edge.manager.hasFifoIdFast(d_flight.base) @@ -224,8 +215,6 @@ class TLRAMModel extends LazyModule val d_valid = valid(d.source) when (d_fire) { - d_counter := Mux(d_first, d_beats1, d_counter1) - // Check the response is correct assert (d_size === d_flight.size) assert (edge.manager.findIdStartFast(d_flight.base) <= d.sink) diff --git a/src/main/scala/uncore/tilelink2/RegisterRouterTest.scala b/src/main/scala/uncore/tilelink2/RegisterRouterTest.scala index a7823ea3..25f87100 100644 --- a/src/main/scala/uncore/tilelink2/RegisterRouterTest.scala +++ b/src/main/scala/uncore/tilelink2/RegisterRouterTest.scala @@ -229,20 +229,18 @@ trait RRTest1Module extends Module with HasRegMap val field = UInt(width = bits) val readCross = Module(new RegisterReadCrossing(field)) - readCross.io.master_clock := clock - readCross.io.master_reset := reset - readCross.io.master_allow := Bool(true) - readCross.io.slave_clock := clocks.io.clock_out - readCross.io.slave_reset := reset - readCross.io.slave_allow := Bool(true) + readCross.io.master_clock := clock + readCross.io.master_reset := reset + readCross.io.master_bypass := Bool(false) + readCross.io.slave_clock := clocks.io.clock_out + readCross.io.slave_reset := reset val writeCross = Module(new RegisterWriteCrossing(field)) - writeCross.io.master_clock := clock - writeCross.io.master_reset := reset - writeCross.io.master_allow := Bool(true) - writeCross.io.slave_clock := clocks.io.clock_out - writeCross.io.slave_reset := reset - writeCross.io.slave_allow := Bool(true) + writeCross.io.master_clock := clock + writeCross.io.master_reset := reset + writeCross.io.master_bypass := Bool(false) + writeCross.io.slave_clock := clocks.io.clock_out + writeCross.io.slave_reset := reset readCross.io.slave_register := writeCross.io.slave_register RegField(bits, readCross.io.master_port, writeCross.io.master_port) diff --git a/src/main/scala/uncore/tilelink2/package.scala b/src/main/scala/uncore/tilelink2/package.scala index a0490d7e..e996f2ba 100644 --- a/src/main/scala/uncore/tilelink2/package.scala +++ b/src/main/scala/uncore/tilelink2/package.scala @@ -5,9 +5,9 @@ import diplomacy._ package object tilelink2 { - type TLOutwardNode = OutwardNode[TLClientPortParameters, TLManagerPortParameters, TLBundle] - type TLAsyncOutwardNode = OutwardNode[TLAsyncClientPortParameters, TLAsyncManagerPortParameters, TLAsyncBundle] - type IntOutwardNode = OutwardNode[IntSourcePortParameters, IntSinkPortParameters, Vec[Bool]] + type TLOutwardNode = OutwardNodeHandle[TLClientPortParameters, TLManagerPortParameters, TLBundle] + type TLAsyncOutwardNode = OutwardNodeHandle[TLAsyncClientPortParameters, TLAsyncManagerPortParameters, TLAsyncBundle] + type IntOutwardNode = OutwardNodeHandle[IntSourcePortParameters, IntSinkPortParameters, Vec[Bool]] def OH1ToUInt(x: UInt) = OHToUInt((x << 1 | UInt(1)) ^ x) def UIntToOH1(x: UInt, width: Int) = ~(SInt(-1, width=width).asUInt << x)(width-1, 0) def trailingZeros(x: Int) = if (x > 0) Some(log2Ceil(x & -x)) else None diff --git a/src/main/scala/util/AsyncQueue.scala b/src/main/scala/util/AsyncQueue.scala index 9eef5a14..a347b62f 100644 --- a/src/main/scala/util/AsyncQueue.scala +++ b/src/main/scala/util/AsyncQueue.scala @@ -4,19 +4,19 @@ package util import Chisel._ object GrayCounter { - def apply(bits: Int, increment: Bool = Bool(true), name: String = "binary"): UInt = { + def apply(bits: Int, increment: Bool = Bool(true), clear: Bool = Bool(false), name: String = "binary"): UInt = { val incremented = Wire(UInt(width=bits)) val binary = AsyncResetReg(incremented, name) - incremented := binary + increment.asUInt() + incremented := Mux(clear, UInt(0), binary + increment.asUInt()) incremented ^ (incremented >> UInt(1)) } } -object AsyncGrayCounter { +object UIntSyncChain { def apply(in: UInt, sync: Int, name: String = "gray"): UInt = { - val syncv = List.tabulate(sync)(i => + val syncv = List.tabulate(sync) { i => Module (new AsyncResetRegVec(w = in.getWidth, 0)).suggestName(s"${name}_sync_${i}") - ) + } syncv.last.io.d := in syncv.last.io.en := Bool(true) (syncv.init zip syncv.tail).foreach { case (sink, source) => @@ -31,27 +31,38 @@ class AsyncQueueSource[T <: Data](gen: T, depth: Int, sync: Int) extends Module val bits = log2Ceil(depth) val io = new Bundle { // These come from the source domain - val enq = Decoupled(gen).flip() + val enq = Decoupled(gen).flip // These cross to the sink clock domain val ridx = UInt(INPUT, width = bits+1) val widx = UInt(OUTPUT, width = bits+1) val mem = Vec(depth, gen).asOutput + // Reset for the other side + val sink_reset_n = Bool().flip } + // extend the sink reset to a full cycle (assertion latency <= 1 cycle) + val catch_sink_reset_n = AsyncResetReg(Bool(true), clock, !io.sink_reset_n, "catch_sink_reset_n") + // reset_n has a 1 cycle shorter path to ready than ridx does + val sink_reset_n = UIntSyncChain(catch_sink_reset_n.asUInt, sync, "sink_reset_n")(0) + val mem = Reg(Vec(depth, gen)) //This does NOT need to be asynchronously reset. - val widx = GrayCounter(bits+1, io.enq.fire(), "widx_bin") - val ridx = AsyncGrayCounter(io.ridx, sync, "ridx_gray") + val widx = GrayCounter(bits+1, io.enq.fire(), !sink_reset_n, "widx_bin") + val ridx = UIntSyncChain(io.ridx, sync, "ridx_gray") val ready = widx =/= (ridx ^ UInt(depth | depth >> 1)) val index = if (depth == 1) UInt(0) else io.widx(bits-1, 0) ^ (io.widx(bits, bits) << (bits-1)) when (io.enq.fire()) { mem(index) := io.enq.bits } - val ready_reg = AsyncResetReg(ready, "ready") - io.enq.ready := ready_reg + + val ready_reg = AsyncResetReg(ready.asUInt, "ready_reg")(0) + io.enq.ready := ready_reg && sink_reset_n val widx_reg = AsyncResetReg(widx, "widx_gray") io.widx := widx_reg io.mem := mem + + // It is a fatal error to reset half a Queue while it still has data + assert (sink_reset_n || widx === ridx) } class AsyncQueueSink[T <: Data](gen: T, depth: Int, sync: Int) extends Module { @@ -63,10 +74,17 @@ class AsyncQueueSink[T <: Data](gen: T, depth: Int, sync: Int) extends Module { val ridx = UInt(OUTPUT, width = bits+1) val widx = UInt(INPUT, width = bits+1) val mem = Vec(depth, gen).asInput + // Reset for the other side + val source_reset_n = Bool().flip } - val ridx = GrayCounter(bits+1, io.deq.fire(), "ridx_bin") - val widx = AsyncGrayCounter(io.widx, sync, "widx_gray") + // extend the source reset to a full cycle (assertion latency <= 1 cycle) + val catch_source_reset_n = AsyncResetReg(Bool(true), clock, !io.source_reset_n, "catch_source_reset_n") + // reset_n has a 1 cycle shorter path to valid than widx does + val source_reset_n = UIntSyncChain(catch_source_reset_n.asUInt, sync, "source_reset_n")(0) + + val ridx = GrayCounter(bits+1, io.deq.fire(), !source_reset_n, "ridx_bin") + val widx = UIntSyncChain(io.widx, sync, "widx_gray") val valid = ridx =/= widx // The mux is safe because timing analysis ensures ridx has reached the register @@ -76,12 +94,18 @@ class AsyncQueueSink[T <: Data](gen: T, depth: Int, sync: Int) extends Module { val index = if (depth == 1) UInt(0) else ridx(bits-1, 0) ^ (ridx(bits, bits) << (bits-1)) // This register does not NEED to be reset, as its contents will not // be considered unless the asynchronously reset deq valid register is set. - val data = RegEnable(io.mem(index), valid) - io.deq.bits := data - - io.deq.valid := AsyncResetReg(valid, "valid_reg") + // It is possible that bits latches when the source domain is reset / has power cut + // This is safe, because isolation gates brought mem low before the zeroed widx reached us + io.deq.bits := RegEnable(io.mem(index), valid) - io.ridx := AsyncResetReg(ridx, "ridx_gray") + val valid_reg = AsyncResetReg(valid.asUInt, "valid_reg")(0) + io.deq.valid := valid_reg && source_reset_n + + val ridx_reg = AsyncResetReg(ridx, "ridx_gray") + io.ridx := ridx_reg + + // It is a fatal error to reset half a Queue while it still has data + assert (source_reset_n || widx === ridx) } class AsyncQueue[T <: Data](gen: T, depth: Int = 8, sync: Int = 3) extends Crossing[T] { @@ -97,6 +121,9 @@ class AsyncQueue[T <: Data](gen: T, depth: Int = 8, sync: Int = 3) extends Cross sink.clock := io.deq_clock sink.reset := io.deq_reset + source.io.sink_reset_n := !io.deq_reset + sink.io.source_reset_n := !io.enq_reset + source.io.enq <> io.enq io.deq <> sink.io.deq diff --git a/src/main/scala/util/Crossing.scala b/src/main/scala/util/Crossing.scala index ade91747..f466dc11 100644 --- a/src/main/scala/util/Crossing.scala +++ b/src/main/scala/util/Crossing.scala @@ -7,7 +7,7 @@ class CrossingIO[T <: Data](gen: T) extends Bundle { // Enqueue clock domain val enq_clock = Clock(INPUT) val enq_reset = Bool(INPUT) // synchronously deasserted wrt. enq_clock - val enq = Decoupled(gen).flip() + val enq = Decoupled(gen).flip // Dequeue clock domain val deq_clock = Clock(INPUT) val deq_reset = Bool(INPUT) // synchronously deasserted wrt. deq_clock