rocketchip: fix all clock crossings

src/main/scala/coreplex/BaseCoreplex.scala

@@ -68,16 +68,11 @@ abstract class BareCoreplex(implicit val p: Parameters) extends LazyModule with
 abstract class BareCoreplexBundle[+L <: BareCoreplex](val outer: L) extends Bundle with HasCoreplexParameters {
   implicit val p = outer.p
 
-  val master = new Bundle {
   val mem = Vec(nMemChannels, new ClientUncachedTileLinkIO()(outerMemParams))
   val mmio = outer.mmio.bundleOut
-  }
-
   val slave = Vec(nSlaves, new ClientUncachedTileLinkIO()(innerParams)).flip
   val resetVector = UInt(INPUT, p(XLen))
   val success = Bool(OUTPUT) // used for testing
-
-  override def cloneType = this.getClass.getConstructors.head.newInstance(outer).asInstanceOf[this.type]
 }
 
 abstract class BareCoreplexModule[+L <: BareCoreplex, +B <: BareCoreplexBundle[L]](val outer: L, val io: B) extends LazyModuleImp(outer) with HasCoreplexParameters {
@@ -134,7 +129,7 @@ abstract class BareCoreplexModule[+L <: BareCoreplex, +B <: BareCoreplexBundle[L
       icPort <> TileLinkIOUnwrapper(enqueued)
     }
 
-    io.master.mem <> mem_ic.io.out
+    io.mem <> mem_ic.io.out
   }
 
   for ((tile, i) <- (uncoreTileIOs zipWithIndex)) {
@@ -150,7 +145,6 @@ trait CoreplexPeripherals extends HasCoreplexParameters {
   val module: CoreplexPeripheralsModule
   val l1tol2: TLXbar
   val legacy: TLLegacy
-  val lazyTiles: Seq[LazyTile]
 
   val cbus  = LazyModule(new TLXbar)
   val debug = LazyModule(new TLDebugModule())
@@ -166,10 +160,6 @@ trait CoreplexPeripherals extends HasCoreplexParameters {
   debug.node := TLFragmenter(p(XLen)/8, legacy.tlDataBeats * legacy.tlDataBytes)(cbus.node)
   plic.node  := TLFragmenter(p(XLen)/8, legacy.tlDataBeats * legacy.tlDataBytes)(cbus.node)
   clint.node := TLFragmenter(p(XLen)/8, legacy.tlDataBeats * legacy.tlDataBytes)(cbus.node)
-
-  lazyTiles.map(_.slave).flatten.foreach { scratch =>
-    scratch  := TLFragmenter(p(XLen)/8, legacy.tlDataBeats * legacy.tlDataBytes)(cbus.node)
-  }
 }
 
 trait CoreplexPeripheralsBundle extends HasCoreplexParameters {

src/main/scala/coreplex/Coreplex.scala

@@ -3,12 +3,23 @@ package coreplex
 import Chisel._
 import cde.{Parameters, Field}
 import junctions._
+import diplomacy._
 import uncore.tilelink._
+import uncore.tilelink2._
 import uncore.util._
 import util._
 import rocket._
 
 trait DirectConnection {
+  implicit val p: Parameters
+  val lazyTiles: Seq[LazyTile]
+  val legacy: TLLegacy
+  val cbus: TLXbar
+
+  lazyTiles.map(_.slave).flatten.foreach { scratch => scratch := cbus.node }
+}
+
+trait DirectConnectionModule {
   val tiles: Seq[TileImp]
   val uncoreTileIOs: Seq[TileIO]
 
@@ -18,7 +29,6 @@ trait DirectConnection {
   (tiles zip uncoreTileIOs) foreach { case (tile, uncore) =>
     (uncore.cached zip tile.io.cached) foreach { case (u, t) => u <> TileLinkEnqueuer(t, tlBuffering) }
     (uncore.uncached zip tile.io.uncached) foreach { case (u, t) => u <> TileLinkEnqueuer(t, ultBuffering) }
-// !!!    tile.io.slave.foreach { _ <> TileLinkEnqueuer(uncore.slave.get, 1) }
 
     tile.io.interrupts <> uncore.interrupts
 
@@ -27,18 +37,19 @@ trait DirectConnection {
   }
 }
 
-class DefaultCoreplex(implicit p: Parameters) extends BaseCoreplex {
+class DefaultCoreplex(implicit p: Parameters) extends BaseCoreplex
+    with DirectConnection {
   override lazy val module = new DefaultCoreplexModule(this, new DefaultCoreplexBundle(this))
 }
 
 class DefaultCoreplexBundle[+L <: DefaultCoreplex](outer: L) extends BaseCoreplexBundle(outer)
 
 class DefaultCoreplexModule[+L <: DefaultCoreplex, +B <: DefaultCoreplexBundle[L]](outer: L, io: B) extends BaseCoreplexModule(outer, io)
-    with DirectConnection
+    with DirectConnectionModule
 
 /////
 
-trait TileClockResetBundle extends HasCoreplexParameters {
+trait TileClockResetBundle extends Bundle with HasCoreplexParameters {
   val tcrs = Vec(nTiles, new Bundle {
     val clock = Clock(INPUT)
     val reset = Bool(INPUT)
@@ -46,9 +57,37 @@ trait TileClockResetBundle extends HasCoreplexParameters {
 }
 
 trait AsyncConnection {
+  implicit val p: Parameters
+  val lazyTiles: Seq[LazyTile]
+  val legacy: TLLegacy
+  val cbus: TLXbar
+
+  val crossings = lazyTiles.map(_.slave).map(_.map { scratch =>
+    val crossing = LazyModule(new TLAsyncCrossing)
+    crossing.node := cbus.node
+    val monitor = (scratch := crossing.node)
+    (crossing, monitor)
+  })
+}
+
+trait AsyncConnectionModule extends Module {
   val io: TileClockResetBundle
   val tiles: Seq[TileImp]
   val uncoreTileIOs: Seq[TileIO]
+  val outer: AsyncConnection
+
+  (outer.crossings zip io.tcrs) foreach { case (slaves, tcr) =>
+    slaves.foreach { case (crossing, monitor) =>
+      crossing.module.io.in_clock  := clock
+      crossing.module.io.in_reset  := reset
+      crossing.module.io.out_clock := tcr.clock
+      crossing.module.io.out_reset := tcr.reset
+      monitor.foreach { m =>
+        m.module.clock := tcr.clock
+        m.module.reset := tcr.reset
+      }
+    }
+  }
 
   (tiles, uncoreTileIOs, io.tcrs).zipped foreach { case (tile, uncore, tcr) =>
     tile.clock := tcr.clock
@@ -56,7 +95,6 @@ trait AsyncConnection {
 
     (uncore.cached zip tile.io.cached) foreach { case (u, t) => u <> AsyncTileLinkFrom(tcr.clock, tcr.reset, t) }
     (uncore.uncached zip tile.io.uncached) foreach { case (u, t) => u <> AsyncUTileLinkFrom(tcr.clock, tcr.reset, t) }
-// !!!    tile.io.slave.foreach { _ <> AsyncUTileLinkTo(tcr.clock, tcr.reset, uncore.slave.get)}
 
     val ti = tile.io.interrupts
     val ui = uncore.interrupts
@@ -71,7 +109,8 @@ trait AsyncConnection {
   }
 }
 
-class MultiClockCoreplex(implicit p: Parameters) extends BaseCoreplex {
+class MultiClockCoreplex(implicit p: Parameters) extends BaseCoreplex
+    with AsyncConnection {
   override lazy val module = new MultiClockCoreplexModule(this, new MultiClockCoreplexBundle(this))
 }
 
@@ -79,4 +118,4 @@ class MultiClockCoreplexBundle[+L <: MultiClockCoreplex](outer: L) extends BaseC
     with TileClockResetBundle
 
 class MultiClockCoreplexModule[+L <: MultiClockCoreplex, +B <: MultiClockCoreplexBundle[L]](outer: L, io: B) extends BaseCoreplexModule(outer, io)
-    with AsyncConnection
+    with AsyncConnectionModule

src/main/scala/groundtest/Coreplex.scala

@@ -4,13 +4,14 @@ import Chisel._
 import cde.{Parameters}
 import coreplex._
 
-class GroundTestCoreplex(implicit p: Parameters) extends BaseCoreplex {
+class GroundTestCoreplex(implicit p: Parameters) extends BaseCoreplex
+    with DirectConnection {
   override lazy val module = new GroundTestCoreplexModule(this, new GroundTestCoreplexBundle(this))
 }
 
 class GroundTestCoreplexBundle[+L <: GroundTestCoreplex](outer: L) extends BaseCoreplexBundle(outer)
 
 class GroundTestCoreplexModule[+L <: GroundTestCoreplex, +B <: GroundTestCoreplexBundle[L]](outer: L, io: B) extends BaseCoreplexModule(outer, io)
-    with DirectConnection {
+    with DirectConnectionModule {
   io.success := tiles.flatMap(_.io.elements get "success").map(_.asInstanceOf[Bool]).reduce(_&&_)
 }

src/main/scala/rocketchip/BaseTop.scala

@@ -54,8 +54,6 @@ abstract class BaseTop[+C <: BaseCoreplex](buildCoreplex: Parameters => C)(impli
     TLWidthWidget(p(SOCBusKey).beatBytes)(
     socBus.node)))
 
-  socBus.node := coreplex.mmio
-
   TopModule.contents = Some(this)
 }
 
@@ -66,7 +64,11 @@ abstract class BaseTopBundle[+L <: BaseTop[BaseCoreplex]](val outer: L) extends
 
 abstract class BaseTopModule[+L <: BaseTop[BaseCoreplex], +B <: BaseTopBundle[L]](val outer: L, val io: B) extends LazyModuleImp(outer) {
   implicit val p = outer.p
-  val coreplexIO = Wire(outer.coreplex.module.io)
+
+  val coreplexMem        : Vec[ClientUncachedTileLinkIO] = Wire(outer.coreplex.module.io.mem)
+  val coreplexSlave      : Vec[ClientUncachedTileLinkIO] = Wire(outer.coreplex.module.io.slave)
+  val coreplexDebug      : DebugBusIO                    = Wire(outer.coreplex.module.io.debug)
+  val coreplexInterrupts : Vec[Bool]                     = Wire(outer.coreplex.module.io.interrupts)
 
   println("Generated Address Map")
   for (entry <- p(GlobalAddrMap).flatten) {
@@ -88,12 +90,26 @@ abstract class BaseTopModule[+L <: BaseTop[BaseCoreplex], +B <: BaseTopBundle[L]
   println(p(ConfigString))
   ConfigStringOutput.contents = Some(p(ConfigString))
 
-  io.success := coreplexIO.success
+  io.success := outer.coreplex.module.io.success
 }
 
 trait DirectConnection {
-  val coreplexIO: BaseCoreplexBundle[BaseCoreplex]
+  val coreplex: BaseCoreplex
+  val socBus: TLXbar
+
+  socBus.node := coreplex.mmio
+}
+
+trait DirectConnectionModule {
   val outer: BaseTop[BaseCoreplex]
 
-  coreplexIO <> outer.coreplex.module.io
+  val coreplexMem        : Vec[ClientUncachedTileLinkIO]
+  val coreplexSlave      : Vec[ClientUncachedTileLinkIO]
+  val coreplexDebug      : DebugBusIO
+  val coreplexInterrupts : Vec[Bool]
+
+  coreplexMem        <> outer.coreplex.module.io.mem
+  coreplexInterrupts <> outer.coreplex.module.io.interrupts
+  outer.coreplex.module.io.slave <> coreplexSlave
+  outer.coreplex.module.io.debug <> coreplexDebug
 }

src/main/scala/rocketchip/ExampleTop.scala

@@ -15,7 +15,8 @@ class ExampleTop[+C <: BaseCoreplex](buildCoreplex: Parameters => C)(implicit p:
     with PeripheryExtInterrupts
     with PeripheryMasterMem
     with PeripheryMasterAXI4MMIO
-    with PeripherySlave {
+    with PeripherySlave
+    with DirectConnection {
   override lazy val module = new ExampleTopModule(this, new ExampleTopBundle(this))
 }
 
@@ -35,7 +36,7 @@ class ExampleTopModule[+L <: ExampleTop[BaseCoreplex], +B <: ExampleTopBundle[L]
     with PeripheryMasterAXI4MMIOModule
     with PeripherySlaveModule
     with HardwiredResetVector
-    with DirectConnection
+    with DirectConnectionModule
 
 /** Example Top with TestRAM */
 class ExampleTopWithTestRAM[+C <: BaseCoreplex](buildCoreplex: Parameters => C)(implicit p: Parameters) extends ExampleTop(buildCoreplex)

src/main/scala/rocketchip/Periphery.scala

@@ -110,16 +110,16 @@ trait PeripheryDebugModule {
   implicit val p: Parameters
   val outer: PeripheryDebug
   val io: PeripheryDebugBundle
-  val coreplexIO: BaseCoreplexBundle[BaseCoreplex]
+  val coreplexDebug: DebugBusIO
 
   if (p(IncludeJtagDTM)) {
     // JtagDTMWithSync is a wrapper which
     // handles the synchronization as well.
     val dtm = Module (new JtagDTMWithSync()(p))
     dtm.io.jtag <> io.jtag.get
-    coreplexIO.debug <> dtm.io.debug
+    coreplexDebug <> dtm.io.debug
   } else {
-    coreplexIO.debug <>
+    coreplexDebug <>
       (if (p(AsyncDebugBus)) AsyncDebugBusFrom(io.debug_clk.get, io.debug_rst.get, io.debug.get)
       else io.debug.get)
   }
@@ -143,12 +143,12 @@ trait PeripheryExtInterruptsModule {
   implicit val p: Parameters
   val outer: PeripheryExtInterrupts
   val io: PeripheryExtInterruptsBundle
-  val coreplexIO: BaseCoreplexBundle[BaseCoreplex]
+  val coreplexInterrupts: Vec[Bool]
 
   {
     val r = outer.pInterrupts.range("ext")
     ((r._1 until r._2) zipWithIndex) foreach { case (c, i) =>
-      coreplexIO.interrupts(c) := io.interrupts(i)
+      coreplexInterrupts(c) := io.interrupts(i)
     }
   }
 }
@@ -172,9 +172,9 @@ trait PeripheryMasterMemModule extends HasPeripheryParameters {
   implicit val p: Parameters
   val outer: PeripheryMasterMem
   val io: PeripheryMasterMemBundle
-  val coreplexIO: BaseCoreplexBundle[BaseCoreplex]
+  val coreplexMem: Vec[ClientUncachedTileLinkIO]
 
-  val edgeMem = coreplexIO.master.mem.map(TileLinkWidthAdapter(_, edgeMemParams))
+  val edgeMem = coreplexMem.map(TileLinkWidthAdapter(_, edgeMemParams))
 
   // Abuse the fact that zip takes the shorter of the two lists
   ((io.mem_axi zip edgeMem) zipWithIndex) foreach { case ((axi, mem), idx) =>
@@ -254,7 +254,7 @@ trait PeripherySlaveModule extends HasPeripheryParameters {
   implicit val p: Parameters
   val outer: PeripherySlave
   val io: PeripherySlaveBundle
-  val coreplexIO: BaseCoreplexBundle[BaseCoreplex]
+  val coreplexSlave: Vec[ClientUncachedTileLinkIO]
 
   if (p(NExtBusAXIChannels) > 0) {
     val arb = Module(new NastiArbiter(p(NExtBusAXIChannels)))
@@ -269,7 +269,7 @@ trait PeripherySlaveModule extends HasPeripheryParameters {
 
     val (r_start, r_end) = outer.pBusMasters.range("ext")
     require(r_end - r_start == 1, "RangeManager should return 1 slot")
-    TileLinkWidthAdapter(coreplexIO.slave(r_start), conv.io.tl)
+    TileLinkWidthAdapter(coreplexSlave(r_start), conv.io.tl)
   }
 }
 
@@ -340,6 +340,7 @@ trait PeripheryTestBusMasterModule {
 /////
 
 trait HardwiredResetVector {
-  val coreplexIO: BaseCoreplexBundle[BaseCoreplex]
+  val outer: BaseTop[BaseCoreplex]
-  coreplexIO.resetVector := UInt(0x1000) // boot ROM
+
+  outer.coreplex.module.io.resetVector := UInt(0x1000) // boot ROM
 }