From 90266464593f29f9ab8bfc10696741a857bbeca7 Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Mon, 9 Oct 2017 23:43:18 -0700
Subject: [PATCH 01/11] coreplex: first cut at using RocketCrossingParams

---
 src/main/scala/coreplex/Configs.scala        | 12 ++++--
 src/main/scala/coreplex/RocketCoreplex.scala | 43 +++++++++++++++-----
 src/main/scala/coreplex/SystemBus.scala      | 13 +++---
 src/main/scala/groundtest/Coreplex.scala     |  2 +-
 src/main/scala/groundtest/Tile.scala         |  4 +-
 src/main/scala/rocket/DCache.scala           | 16 ++++----
 src/main/scala/rocket/HellaCache.scala       | 12 +++---
 src/main/scala/tile/RocketTile.scala         |  4 +-
 8 files changed, 65 insertions(+), 41 deletions(-)

diff --git a/src/main/scala/coreplex/Configs.scala b/src/main/scala/coreplex/Configs.scala
index 0d932c04..c9f6c797 100644
--- a/src/main/scala/coreplex/Configs.scala
+++ b/src/main/scala/coreplex/Configs.scala
@@ -242,15 +242,21 @@ class WithBootROMFile(bootROMFile: String) extends Config((site, here, up) => {
 })
 
 class WithSynchronousRocketTiles extends Config((site, here, up) => {
-  case RocketCrossing => SynchronousCrossing()
+  case RocketCrossingKey => up(RocketCrossingKey, site) map { r =>
+    r.copy(crossingType = SynchronousCrossing())
+  }
 })
 
 class WithAynchronousRocketTiles(depth: Int, sync: Int) extends Config((site, here, up) => {
-  case RocketCrossing => AsynchronousCrossing(depth, sync)
+  case RocketCrossingKey => up(RocketCrossingKey, site) map { r =>
+    r.copy(crossingType = AsynchronousCrossing(depth, sync))
+  }
 })
 
 class WithRationalRocketTiles extends Config((site, here, up) => {
-  case RocketCrossing => RationalCrossing()
+  case RocketCrossingKey => up(RocketCrossingKey, site) map { r =>
+    r.copy(crossingType = RationalCrossing())
+  }
 })
 
 class WithEdgeDataBits(dataBits: Int) extends Config((site, here, up) => {
diff --git a/src/main/scala/coreplex/RocketCoreplex.scala b/src/main/scala/coreplex/RocketCoreplex.scala
index fa315ddb..93e6544e 100644
--- a/src/main/scala/coreplex/RocketCoreplex.scala
+++ b/src/main/scala/coreplex/RocketCoreplex.scala
@@ -11,8 +11,23 @@ import freechips.rocketchip.tile._
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.util._
 
+// TODO: how specific are these to RocketTiles?
+case class TilePortParams(
+  addBuffers: Int = 0,
+  blockerCtrlAddr: Option[BigInt] = None)
+
+case class RocketCrossingParams(
+    crossingType: CoreplexClockCrossing = SynchronousCrossing(),
+    master: TilePortParams = TilePortParams(),
+    slave: TilePortParams = TilePortParams()) {
+  def knownRatio: Option[Int] = crossingType match {
+    case RationalCrossing(_) => Some(2)
+    case _ => None
+  }
+}
+
 case object RocketTilesKey extends Field[Seq[RocketTileParams]](Nil)
-case object RocketCrossing extends Field[CoreplexClockCrossing](SynchronousCrossing())
+case object RocketCrossingKey extends Field[Seq[RocketCrossingParams]](List(RocketCrossingParams()))
 
 trait HasRocketTiles extends HasTiles
     with HasPeripheryBus
@@ -21,35 +36,43 @@ trait HasRocketTiles extends HasTiles
     with HasPeripheryDebug {
   val module: HasRocketTilesModuleImp
 
-  private val crossing = p(RocketCrossing)
   protected val tileParams = p(RocketTilesKey)
+  private val NumRocketTiles = tileParams.size
+  private val crossingParams = p(RocketCrossingKey)
+  private val crossings = crossingParams.size match {
+    case 1 => List.fill(NumRocketTiles) { crossingParams.head }
+    case NumRocketTiles => crossingParams
+    case _ => throw new Exception("RocketCrossingKey.size must == 1 or == RocketTilesKey.size")
+  }
 
   // Make a wrapper for each tile that will wire it to coreplex devices and crossbars,
   // according to the specified type of clock crossing.
-  val tiles: Seq[BaseTile] = localIntNodes.zip(tileParams).map { case (lip, tp) =>
+  private val crossingTuples = localIntNodes.zip(tileParams).zip(crossings)
+  val tiles: Seq[BaseTile] = crossingTuples.map { case ((lip, tp), crossing) =>
     val pWithExtra = p.alterPartial {
       case TileKey => tp
       case BuildRoCC => tp.rocc
       case SharedMemoryTLEdge => sharedMemoryTLEdge
+      case RocketCrossingKey => List(crossing)
     }
 
-    val wrapper = crossing match {
+    val wrapper = crossing.crossingType match {
       case SynchronousCrossing(params) => {
         val wrapper = LazyModule(new SyncRocketTile(tp)(pWithExtra))
-        sbus.fromSyncTiles(params, tp.externalMasterBuffers, tp.name) :=* wrapper.masterNode
-        FlipRendering { implicit p => wrapper.slaveNode :*= pbus.toSyncSlaves(tp.name, tp.externalSlaveBuffers) }
+        sbus.fromSyncTiles(params, crossing.master, tp.name) :=* wrapper.masterNode
+        FlipRendering { implicit p => wrapper.slaveNode :*= pbus.toSyncSlaves(tp.name, crossing.slave.addBuffers) }
         wrapper
       }
       case AsynchronousCrossing(depth, sync) => {
         val wrapper = LazyModule(new AsyncRocketTile(tp)(pWithExtra))
-        sbus.fromAsyncTiles(depth, sync, tp.externalMasterBuffers, tp.name) :=* wrapper.masterNode
-        FlipRendering { implicit p => wrapper.slaveNode :*= pbus.toAsyncSlaves(sync, tp.name, tp.externalSlaveBuffers) }
+        sbus.fromAsyncTiles(depth, sync, crossing.master, tp.name) :=* wrapper.masterNode
+        FlipRendering { implicit p => wrapper.slaveNode :*= pbus.toAsyncSlaves(sync, tp.name, crossing.slave.addBuffers) }
         wrapper
       }
       case RationalCrossing(direction) => {
         val wrapper = LazyModule(new RationalRocketTile(tp)(pWithExtra))
-        sbus.fromRationalTiles(direction, tp.externalMasterBuffers, tp.name) :=* wrapper.masterNode
-        FlipRendering { implicit p => wrapper.slaveNode :*= pbus.toRationalSlaves(tp.name, tp.externalSlaveBuffers) }
+        sbus.fromRationalTiles(direction, crossing.master, tp.name) :=* wrapper.masterNode
+        FlipRendering { implicit p => wrapper.slaveNode :*= pbus.toRationalSlaves(tp.name, crossing.slave.addBuffers) }
         wrapper
       }
     }
diff --git a/src/main/scala/coreplex/SystemBus.scala b/src/main/scala/coreplex/SystemBus.scala
index 7096b3e5..a755662e 100644
--- a/src/main/scala/coreplex/SystemBus.scala
+++ b/src/main/scala/coreplex/SystemBus.scala
@@ -55,30 +55,31 @@ class SystemBus(params: SystemBusParams)(implicit p: Parameters) extends TLBusWr
 
   def fromFrontBus: TLInwardNode = master_splitter.node
 
-  def fromSyncTiles(params: BufferParams, addBuffers: Int = 0, name: Option[String] = None): TLInwardNode = {
+  def fromSyncTiles(params: BufferParams, port: TilePortParams, name: Option[String] = None): TLInwardNode = {
     val tile_buf = LazyModule(new TLBuffer(params))
     name.foreach { n => tile_buf.suggestName(s"${busName}_${n}_TLBuffer") }
-    val (in, out) = bufferChain(addBuffers, name = name)
+    val (in, out) = bufferChain(port.addBuffers, name = name)
 
     tile_fixer.node :=* out
     in :=* tile_buf.node
     tile_buf.node
   }
 
-  def fromRationalTiles(dir: RationalDirection, addBuffers: Int = 0, name: Option[String] = None): TLRationalInwardNode = {
+  def fromRationalTiles(dir: RationalDirection, port: TilePortParams, name: Option[String] = None): TLRationalInwardNode = {
+    // TODO val tile_blocker = port.blockerCtrlAddr.map(a => LazyModule(new BusBlocker(BusBlockerParams(a, , ))))
     val tile_sink = LazyModule(new TLRationalCrossingSink(direction = dir))
     name.foreach { n => tile_sink.suggestName(s"${busName}_${n}_TLRationalCrossingSink") }
-    val (in, out) = bufferChain(addBuffers, name = name)
+    val (in, out) = bufferChain(port.addBuffers, name = name)
 
     tile_fixer.node :=* out
     in :=* tile_sink.node
     tile_sink.node
   }
 
-  def fromAsyncTiles(depth: Int, sync: Int, addBuffers: Int = 0, name: Option[String] = None): TLAsyncInwardNode = {
+  def fromAsyncTiles(depth: Int, sync: Int, port: TilePortParams, name: Option[String] = None): TLAsyncInwardNode = {
     val tile_sink = LazyModule(new TLAsyncCrossingSink(depth, sync))
     name.foreach { n => tile_sink.suggestName(s"${busName}_${n}_TLAsyncCrossingSink") }
-    val (in, out) = bufferChain(addBuffers, name = name)
+    val (in, out) = bufferChain(port.addBuffers, name = name)
 
     tile_fixer.node :=* out
     in :=* tile_sink.node
diff --git a/src/main/scala/groundtest/Coreplex.scala b/src/main/scala/groundtest/Coreplex.scala
index d6dab576..e45dbc3f 100644
--- a/src/main/scala/groundtest/Coreplex.scala
+++ b/src/main/scala/groundtest/Coreplex.scala
@@ -25,7 +25,7 @@ class GroundTestCoreplex(implicit p: Parameters) extends BaseCoreplex
     })
   )}
 
-  tiles.flatMap(_.dcacheOpt).foreach { sbus.fromSyncTiles(BufferParams.default) :=* _.node }
+  tiles.flatMap(_.dcacheOpt).foreach { sbus.fromSyncTiles(BufferParams.default, TilePortParams()) :=* _.node }
 
   val pbusRAM = LazyModule(new TLRAM(AddressSet(testRamAddr, 0xffff), false, pbus.beatBytes))
   pbusRAM.node := pbus.toVariableWidthSlaves
diff --git a/src/main/scala/groundtest/Tile.scala b/src/main/scala/groundtest/Tile.scala
index 573e71c6..ccb8b853 100644
--- a/src/main/scala/groundtest/Tile.scala
+++ b/src/main/scala/groundtest/Tile.scala
@@ -7,7 +7,7 @@ import Chisel._
 import freechips.rocketchip.config._
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.coreplex._
-import freechips.rocketchip.rocket.{HellaCache, RocketCoreParams}
+import freechips.rocketchip.rocket.{DCache, RocketCoreParams}
 import freechips.rocketchip.tile._
 import scala.collection.mutable.ListBuffer
 
@@ -30,7 +30,7 @@ case object GroundTestTilesKey extends Field[Seq[GroundTestTileParams]]
 
 abstract class GroundTestTile(params: GroundTestTileParams)(implicit p: Parameters) extends BaseTile(params)(p) {
   val slave = None
-  val dcacheOpt = params.dcache.map { dc => LazyModule(HellaCache(0, dc.nMSHRs == 0)) }
+  val dcacheOpt = params.dcache.map { dc => LazyModule(new DCache(0)) }
 
   override lazy val module = new GroundTestTileModule(this, () => new GroundTestTileBundle(this))
 }
diff --git a/src/main/scala/rocket/DCache.scala b/src/main/scala/rocket/DCache.scala
index e3daa973..ed872a2a 100644
--- a/src/main/scala/rocket/DCache.scala
+++ b/src/main/scala/rocket/DCache.scala
@@ -5,7 +5,7 @@ package freechips.rocketchip.rocket
 import Chisel._
 import Chisel.ImplicitConversions._
 import freechips.rocketchip.config.Parameters
-import freechips.rocketchip.coreplex.{RationalCrossing, RocketCrossing, RocketTilesKey}
+import freechips.rocketchip.coreplex.{RocketTilesKey}
 import freechips.rocketchip.diplomacy.{AddressSet, RegionType}
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.util._
@@ -62,7 +62,7 @@ class DCacheMetadataReq(implicit p: Parameters) extends L1HellaCacheBundle()(p)
   val data = new L1Metadata
 }
 
-class DCache(hartid: Int, val scratch: () => Option[AddressSet] = () => None)(implicit p: Parameters) extends HellaCache(hartid)(p) {
+class DCache(hartid: Int, val scratch: () => Option[AddressSet] = () => None, val bufferUncachedRequests: Option[Int] = None)(implicit p: Parameters) extends HellaCache(hartid)(p) {
   override lazy val module = new DCacheModule(this) 
 }
 
@@ -89,14 +89,12 @@ class DCacheModule(outer: DCache) extends HellaCacheModule(outer) {
   dataArb.io.out.ready := true
   metaArb.io.out.ready := true
 
-  val rational = p(RocketCrossing) match {
-    case RationalCrossing(_) => true
-    case _ => false
-  }
-
-  val q_depth = if (rational) (2 min maxUncachedInFlight-1) else 0
   val tl_out_a = Wire(tl_out.a)
-  tl_out.a <> (if (q_depth == 0) tl_out_a else Queue(tl_out_a, q_depth, flow = true))
+  tl_out.a <> outer.bufferUncachedRequests
+                .map(_ min maxUncachedInFlight-1)
+                .map(Queue(tl_out_a, _, flow = true))
+                .getOrElse(tl_out_a)
+
   val (tl_out_c, release_queue_empty) =
     if (cacheParams.acquireBeforeRelease) {
       val q = Module(new Queue(tl_out.c.bits, cacheDataBeats, flow = true))
diff --git a/src/main/scala/rocket/HellaCache.scala b/src/main/scala/rocket/HellaCache.scala
index 558e0259..4648a4b9 100644
--- a/src/main/scala/rocket/HellaCache.scala
+++ b/src/main/scala/rocket/HellaCache.scala
@@ -192,12 +192,6 @@ class HellaCacheModule(outer: HellaCache) extends LazyModuleImp(outer)
   }
 }
 
-object HellaCache {
-  def apply(hartid: Int, blocking: Boolean, scratch: () => Option[AddressSet] = () => None)(implicit p: Parameters) = {
-    if (blocking) new DCache(hartid, scratch) else new NonBlockingDCache(hartid)
-  }
-}
-
 /** Mix-ins for constructing tiles that have a HellaCache */
 
 trait HasHellaCache extends HasTileLinkMasterPort with HasTileParameters {
@@ -206,7 +200,11 @@ trait HasHellaCache extends HasTileLinkMasterPort with HasTileParameters {
   def findScratchpadFromICache: Option[AddressSet]
   val hartid: Int
   var nDCachePorts = 0
-  val dcache = LazyModule(HellaCache(hartid, tileParams.dcache.get.nMSHRs == 0, findScratchpadFromICache _))
+  val dcache: HellaCache = LazyModule(
+    if(tileParams.dcache.get.nMSHRs == 0) {
+      new DCache(hartid, findScratchpadFromICache _, p(RocketCrossingKey).head.knownRatio)
+    } else { new NonBlockingDCache(hartid) })
+
   tileBus.node := dcache.node
 }
 
diff --git a/src/main/scala/tile/RocketTile.scala b/src/main/scala/tile/RocketTile.scala
index 89eda305..fde37ab6 100644
--- a/src/main/scala/tile/RocketTile.scala
+++ b/src/main/scala/tile/RocketTile.scala
@@ -22,9 +22,7 @@ case class RocketTileParams(
     trace: Boolean = false,
     hcfOnUncorrectable: Boolean = false,
     name: Option[String] = Some("tile"),
-    hartid: Int = 0,
-    externalMasterBuffers: Int = 0,
-    externalSlaveBuffers: Int = 0) extends TileParams {
+    hartid: Int = 0) extends TileParams {
   require(icache.isDefined)
   require(dcache.isDefined)
 }

From 660355004e8d48959425f7f7e4961fe8387ca244 Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Tue, 10 Oct 2017 15:02:50 -0700
Subject: [PATCH 02/11] coreplex: TileMasterPortParams inject adapters into
 SBus

---
 src/main/scala/coreplex/RocketCoreplex.scala | 65 +++++++++++++++++---
 src/main/scala/coreplex/SystemBus.scala      | 44 ++++++-------
 src/main/scala/groundtest/Coreplex.scala     |  4 +-
 3 files changed, 80 insertions(+), 33 deletions(-)

diff --git a/src/main/scala/coreplex/RocketCoreplex.scala b/src/main/scala/coreplex/RocketCoreplex.scala
index 93e6544e..66a66acf 100644
--- a/src/main/scala/coreplex/RocketCoreplex.scala
+++ b/src/main/scala/coreplex/RocketCoreplex.scala
@@ -11,15 +11,64 @@ import freechips.rocketchip.tile._
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.util._
 
+case class TLNodeChain(in: TLInwardNode, out: TLOutwardNode)
+
 // TODO: how specific are these to RocketTiles?
-case class TilePortParams(
-  addBuffers: Int = 0,
-  blockerCtrlAddr: Option[BigInt] = None)
+case class TileMasterPortParams(
+    addBuffers: Int = 0,
+    blockerCtrlAddr: Option[BigInt] = None,
+    cork: Boolean = false) {
+  def adapterChain(coreplex: HasPeripheryBus)
+                  (implicit p: Parameters): () => TLNodeChain = {
+
+    val blockerParams = blockerCtrlAddr.map(BusBlockerParams(_, coreplex.pbus.beatBytes, coreplex.sbus.beatBytes, 1))
+
+    val tile_master_cork = cork.option(LazyModule(new TLCacheCork))
+    val tile_master_blocker = blockerParams.map(bp => LazyModule(new BusBlocker(bp)))
+    val tile_master_fixer = LazyModule(new TLFIFOFixer(TLFIFOFixer.allUncacheable))
+    val tile_master_buffer = LazyModule(new TLBufferChain(addBuffers))
+
+    val nodes = List(
+      Some(tile_master_buffer.node),
+      Some(tile_master_fixer.node),
+      tile_master_blocker.map(_.node),
+      tile_master_cork.map(_.node)).flatMap(b=>b)
+
+    nodes.init zip nodes.tail foreach { case(front, back) => front :=* back }
+
+    tile_master_blocker.foreach { _.controlNode := coreplex.pbus.toVariableWidthSlaves }
+
+    () => TLNodeChain(nodes.last, nodes.head)
+  }
+}
+
+case class TileSlavePortParams(
+    addBuffers: Int = 0,
+    blockerCtrlAddr: Option[BigInt] = None) {
+  def adapterChain(coreplex: HasPeripheryBus)
+                  (implicit p: Parameters): () => TLNodeChain = {
+
+    val blockerParams = blockerCtrlAddr.map(BusBlockerParams(_, coreplex.pbus.beatBytes, coreplex.sbus.beatBytes, 1))
+
+    val tile_slave_blocker = blockerParams.map(bp => LazyModule(new BusBlocker(bp)))
+    val tile_slave_buffer = LazyModule(new TLBufferChain(addBuffers))
+
+    val nodes = List(
+      Some(tile_slave_buffer.node),
+      tile_slave_blocker.map(_.node)).flatMap(b=>b)
+
+    nodes.init zip nodes.tail foreach { case(front, back) => front :=* back }
+
+    tile_slave_blocker.foreach { _.controlNode := coreplex.pbus.toVariableWidthSlaves }
+
+    () => TLNodeChain(nodes.last, nodes.head)
+  }
+}
 
 case class RocketCrossingParams(
     crossingType: CoreplexClockCrossing = SynchronousCrossing(),
-    master: TilePortParams = TilePortParams(),
-    slave: TilePortParams = TilePortParams()) {
+    master: TileMasterPortParams = TileMasterPortParams(),
+    slave: TileSlavePortParams = TileSlavePortParams()) {
   def knownRatio: Option[Int] = crossingType match {
     case RationalCrossing(_) => Some(2)
     case _ => None
@@ -59,19 +108,19 @@ trait HasRocketTiles extends HasTiles
     val wrapper = crossing.crossingType match {
       case SynchronousCrossing(params) => {
         val wrapper = LazyModule(new SyncRocketTile(tp)(pWithExtra))
-        sbus.fromSyncTiles(params, crossing.master, tp.name) :=* wrapper.masterNode
+        sbus.fromSyncTiles(params, crossing.master.adapterChain(this), tp.name) :=* wrapper.masterNode
         FlipRendering { implicit p => wrapper.slaveNode :*= pbus.toSyncSlaves(tp.name, crossing.slave.addBuffers) }
         wrapper
       }
       case AsynchronousCrossing(depth, sync) => {
         val wrapper = LazyModule(new AsyncRocketTile(tp)(pWithExtra))
-        sbus.fromAsyncTiles(depth, sync, crossing.master, tp.name) :=* wrapper.masterNode
+        sbus.fromAsyncTiles(depth, sync, crossing.master.adapterChain(this), tp.name) :=* wrapper.masterNode
         FlipRendering { implicit p => wrapper.slaveNode :*= pbus.toAsyncSlaves(sync, tp.name, crossing.slave.addBuffers) }
         wrapper
       }
       case RationalCrossing(direction) => {
         val wrapper = LazyModule(new RationalRocketTile(tp)(pWithExtra))
-        sbus.fromRationalTiles(direction, crossing.master, tp.name) :=* wrapper.masterNode
+        sbus.fromRationalTiles(direction, crossing.master.adapterChain(this), tp.name) :=* wrapper.masterNode
         FlipRendering { implicit p => wrapper.slaveNode :*= pbus.toRationalSlaves(tp.name, crossing.slave.addBuffers) }
         wrapper
       }
diff --git a/src/main/scala/coreplex/SystemBus.scala b/src/main/scala/coreplex/SystemBus.scala
index a755662e..c07138f9 100644
--- a/src/main/scala/coreplex/SystemBus.scala
+++ b/src/main/scala/coreplex/SystemBus.scala
@@ -27,9 +27,6 @@ class SystemBus(params: SystemBusParams)(implicit p: Parameters) extends TLBusWr
   protected def inwardSplitNode: TLInwardNode = master_splitter.node
   protected def outwardSplitNode: TLOutwardNode = master_splitter.node
 
-  private val tile_fixer = LazyModule(new TLFIFOFixer(TLFIFOFixer.allUncacheable))
-  tile_fixer.suggestName(s"${busName}_tile_TLFIFOFixer")
-  master_splitter.node :=* tile_fixer.node
 
   private val port_fixer = LazyModule(new TLFIFOFixer(TLFIFOFixer.all))
   port_fixer.suggestName(s"${busName}_port_TLFIFOFixer")
@@ -55,34 +52,33 @@ class SystemBus(params: SystemBusParams)(implicit p: Parameters) extends TLBusWr
 
   def fromFrontBus: TLInwardNode = master_splitter.node
 
-  def fromSyncTiles(params: BufferParams, port: TilePortParams, name: Option[String] = None): TLInwardNode = {
-    val tile_buf = LazyModule(new TLBuffer(params))
-    name.foreach { n => tile_buf.suggestName(s"${busName}_${n}_TLBuffer") }
-    val (in, out) = bufferChain(port.addBuffers, name = name)
+  def fromSyncTiles(params: BufferParams, adapt: () => TLNodeChain, name: Option[String] = None): TLInwardNode = {
+    val adapters = adapt() // wanted to be called inside SystemBus scope
+    val tile_sink = LazyModule(new TLBuffer(params))
+    name.foreach { n => tile_sink.suggestName(s"${busName}_${n}_TLBuffer") }
 
-    tile_fixer.node :=* out
-    in :=* tile_buf.node
-    tile_buf.node
-  }
-
-  def fromRationalTiles(dir: RationalDirection, port: TilePortParams, name: Option[String] = None): TLRationalInwardNode = {
-    // TODO val tile_blocker = port.blockerCtrlAddr.map(a => LazyModule(new BusBlocker(BusBlockerParams(a, , ))))
-    val tile_sink = LazyModule(new TLRationalCrossingSink(direction = dir))
-    name.foreach { n => tile_sink.suggestName(s"${busName}_${n}_TLRationalCrossingSink") }
-    val (in, out) = bufferChain(port.addBuffers, name = name)
-
-    tile_fixer.node :=* out
-    in :=* tile_sink.node
+    adapters.in :=* tile_sink.node
+    master_splitter.node :=* adapters.out
     tile_sink.node
   }
 
-  def fromAsyncTiles(depth: Int, sync: Int, port: TilePortParams, name: Option[String] = None): TLAsyncInwardNode = {
+  def fromRationalTiles(dir: RationalDirection, adapt: () => TLNodeChain, name: Option[String] = None): TLRationalInwardNode = {
+    val adapters = adapt() // wanted to be called inside SystemBus scope
+    val tile_sink = LazyModule(new TLRationalCrossingSink(direction = dir))
+    name.foreach { n => tile_sink.suggestName(s"${busName}_${n}_TLRationalCrossingSink") }
+
+    adapters.in :=* tile_sink.node
+    master_splitter.node :=* adapters.out
+    tile_sink.node
+  }
+
+  def fromAsyncTiles(depth: Int, sync: Int, adapt: () => TLNodeChain, name: Option[String] = None): TLAsyncInwardNode = {
+    val adapters = adapt() // wanted to be called inside SystemBus scope
     val tile_sink = LazyModule(new TLAsyncCrossingSink(depth, sync))
     name.foreach { n => tile_sink.suggestName(s"${busName}_${n}_TLAsyncCrossingSink") }
-    val (in, out) = bufferChain(port.addBuffers, name = name)
 
-    tile_fixer.node :=* out
-    in :=* tile_sink.node
+    adapters.in :=* tile_sink.node
+    master_splitter.node :=* adapters.out
     tile_sink.node
   }
 
diff --git a/src/main/scala/groundtest/Coreplex.scala b/src/main/scala/groundtest/Coreplex.scala
index e45dbc3f..b9ee65cf 100644
--- a/src/main/scala/groundtest/Coreplex.scala
+++ b/src/main/scala/groundtest/Coreplex.scala
@@ -25,7 +25,9 @@ class GroundTestCoreplex(implicit p: Parameters) extends BaseCoreplex
     })
   )}
 
-  tiles.flatMap(_.dcacheOpt).foreach { sbus.fromSyncTiles(BufferParams.default, TilePortParams()) :=* _.node }
+  tiles.flatMap(_.dcacheOpt).foreach {
+    sbus.fromSyncTiles(BufferParams.default, TileMasterPortParams().adapterChain(this)) :=* _.node
+  }
 
   val pbusRAM = LazyModule(new TLRAM(AddressSet(testRamAddr, 0xffff), false, pbus.beatBytes))
   pbusRAM.node := pbus.toVariableWidthSlaves

From 8f5f80f958f35348371490c9323e7f14066565a4 Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Tue, 10 Oct 2017 15:25:08 -0700
Subject: [PATCH 03/11] coreplex: TileSlavePortParams inject adapters into PBus

---
 src/main/scala/coreplex/PeripheryBus.scala   | 24 ++++++++++++++++++++
 src/main/scala/coreplex/RocketCoreplex.scala |  6 ++---
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/src/main/scala/coreplex/PeripheryBus.scala b/src/main/scala/coreplex/PeripheryBus.scala
index 3272a3b1..d7ac4a0e 100644
--- a/src/main/scala/coreplex/PeripheryBus.scala
+++ b/src/main/scala/coreplex/PeripheryBus.scala
@@ -31,6 +31,30 @@ class PeripheryBus(params: PeripheryBusParams)(implicit p: Parameters) extends T
     TLFragmenter(params.beatBytes, maxXferBytes)(outwardBufNode)
   }
 
+  def toSyncSlaves(adapt: () => TLNodeChain, name: Option[String]): TLOutwardNode = SinkCardinality { implicit p =>
+    val adapters = adapt()
+    adapters.in :=? outwardBufNode
+    adapters.out
+  }
+
+  def toAsyncSlaves(sync: Int, adapt: () => TLNodeChain, name: Option[String]): TLAsyncOutwardNode = SinkCardinality { implicit p =>
+    val adapters = adapt()
+    val source = LazyModule(new TLAsyncCrossingSource(sync))
+    name.foreach{ n => source.suggestName(s"${busName}_${n}_TLAsyncCrossingSource")}
+    adapters.in :=? outwardNode
+    source.node :=? adapters.out
+    source.node
+  }
+
+  def toRationalSlaves(adapt: () => TLNodeChain, name: Option[String]): TLRationalOutwardNode = SinkCardinality { implicit p =>
+    val adapters = adapt()
+    val source = LazyModule(new TLRationalCrossingSource())
+    name.foreach{ n => source.suggestName(s"${busName}_${n}_TLRationalCrossingSource")}
+    adapters.in :=? outwardNode
+    source.node :=? adapters.out
+    source.node
+  }
+
   val fromSystemBus: TLInwardNode = {
     val atomics = LazyModule(new TLAtomicAutomata(arithmetic = params.arithmetic))
     inwardBufNode := atomics.node
diff --git a/src/main/scala/coreplex/RocketCoreplex.scala b/src/main/scala/coreplex/RocketCoreplex.scala
index 66a66acf..e8301f22 100644
--- a/src/main/scala/coreplex/RocketCoreplex.scala
+++ b/src/main/scala/coreplex/RocketCoreplex.scala
@@ -109,19 +109,19 @@ trait HasRocketTiles extends HasTiles
       case SynchronousCrossing(params) => {
         val wrapper = LazyModule(new SyncRocketTile(tp)(pWithExtra))
         sbus.fromSyncTiles(params, crossing.master.adapterChain(this), tp.name) :=* wrapper.masterNode
-        FlipRendering { implicit p => wrapper.slaveNode :*= pbus.toSyncSlaves(tp.name, crossing.slave.addBuffers) }
+        FlipRendering { implicit p => wrapper.slaveNode :*= pbus.toSyncSlaves(crossing.slave.adapterChain(this), tp.name) }
         wrapper
       }
       case AsynchronousCrossing(depth, sync) => {
         val wrapper = LazyModule(new AsyncRocketTile(tp)(pWithExtra))
         sbus.fromAsyncTiles(depth, sync, crossing.master.adapterChain(this), tp.name) :=* wrapper.masterNode
-        FlipRendering { implicit p => wrapper.slaveNode :*= pbus.toAsyncSlaves(sync, tp.name, crossing.slave.addBuffers) }
+        FlipRendering { implicit p => wrapper.slaveNode :*= pbus.toAsyncSlaves(sync, crossing.slave.adapterChain(this), tp.name) }
         wrapper
       }
       case RationalCrossing(direction) => {
         val wrapper = LazyModule(new RationalRocketTile(tp)(pWithExtra))
         sbus.fromRationalTiles(direction, crossing.master.adapterChain(this), tp.name) :=* wrapper.masterNode
-        FlipRendering { implicit p => wrapper.slaveNode :*= pbus.toRationalSlaves(tp.name, crossing.slave.addBuffers) }
+        FlipRendering { implicit p => wrapper.slaveNode :*= pbus.toRationalSlaves(crossing.slave.adapterChain(this), tp.name) }
         wrapper
       }
     }

From 37406706b4a5a2290e24f2841051ae393f3a2f27 Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Tue, 10 Oct 2017 16:24:32 -0700
Subject: [PATCH 04/11] coreplex: move CacheCork in front of SBus

Continue to not allow caches to cache ROMs.
Update TinyConfig and WithStatelessBridge.
---
 src/main/scala/coreplex/Configs.scala        | 55 ++++++++++----------
 src/main/scala/coreplex/RocketCoreplex.scala |  4 +-
 src/main/scala/system/Configs.scala          |  2 +-
 src/main/scala/tilelink/CacheCork.scala      |  6 +--
 4 files changed, 33 insertions(+), 34 deletions(-)

diff --git a/src/main/scala/coreplex/Configs.scala b/src/main/scala/coreplex/Configs.scala
index c9f6c797..0caf7f65 100644
--- a/src/main/scala/coreplex/Configs.scala
+++ b/src/main/scala/coreplex/Configs.scala
@@ -71,31 +71,32 @@ class WithNSmallCores(n: Int) extends Config((site, here, up) => {
   }
 })
 
-class WithNTinyCores(n: Int) extends Config((site, here, up) => {
-    case XLen => 32
-    case RocketTilesKey => {
-      val tiny = RocketTileParams(
-        core = RocketCoreParams(
-          useVM = false,
-          fpu = None,
-          mulDiv = Some(MulDivParams(mulUnroll = 8))),
-        btb = None,
-        dcache = Some(DCacheParams(
-          rowBits = site(SystemBusKey).beatBits,
-          nSets = 256, // 16Kb scratchpad
-          nWays = 1,
-          nTLBEntries = 4,
-          nMSHRs = 0,
-          blockBytes = site(CacheBlockBytes),
-          scratch = Some(0x80000000L))),
-        icache = Some(ICacheParams(
-          rowBits = site(SystemBusKey).beatBits,
-          nSets = 64,
-          nWays = 1,
-          nTLBEntries = 4,
-          blockBytes = site(CacheBlockBytes))))
-    List.tabulate(n)(i => tiny.copy(hartid = i))
-  }
+class With1TinyCore extends Config((site, here, up) => {
+  case XLen => 32
+  case RocketTilesKey => List(RocketTileParams(
+      core = RocketCoreParams(
+        useVM = false,
+        fpu = None,
+        mulDiv = Some(MulDivParams(mulUnroll = 8))),
+      btb = None,
+      dcache = Some(DCacheParams(
+        rowBits = site(SystemBusKey).beatBits,
+        nSets = 256, // 16Kb scratchpad
+        nWays = 1,
+        nTLBEntries = 4,
+        nMSHRs = 0,
+        blockBytes = site(CacheBlockBytes),
+        scratch = Some(0x80000000L))),
+      icache = Some(ICacheParams(
+        rowBits = site(SystemBusKey).beatBits,
+        nSets = 64,
+        nWays = 1,
+        nTLBEntries = 4,
+        blockBytes = site(CacheBlockBytes)))))
+  case RocketCrossingKey => List(RocketCrossingParams(
+    crossingType = SynchronousCrossing(),
+    master = TileMasterPortParams(cork = Some(true))
+  ))
 })
 
 class WithNBanksPerMemChannel(n: Int) extends Config((site, here, up) => {
@@ -153,10 +154,8 @@ class WithBufferlessBroadcastHub extends Config((site, here, up) => {
 class WithStatelessBridge extends Config((site, here, up) => {
   case BankedL2Key => up(BankedL2Key, site).copy(coherenceManager = { coreplex =>
     implicit val p = coreplex.p
-    val cork = LazyModule(new TLCacheCork(unsafe = true))
     val ww = LazyModule(new TLWidthWidget(coreplex.sbusBeatBytes))
-    ww.node :*= cork.node
-    (cork.node, ww.node, () => None)
+    (ww.node, ww.node, () => None)
   })
 })
 
diff --git a/src/main/scala/coreplex/RocketCoreplex.scala b/src/main/scala/coreplex/RocketCoreplex.scala
index e8301f22..7287928c 100644
--- a/src/main/scala/coreplex/RocketCoreplex.scala
+++ b/src/main/scala/coreplex/RocketCoreplex.scala
@@ -17,13 +17,13 @@ case class TLNodeChain(in: TLInwardNode, out: TLOutwardNode)
 case class TileMasterPortParams(
     addBuffers: Int = 0,
     blockerCtrlAddr: Option[BigInt] = None,
-    cork: Boolean = false) {
+    cork: Option[Boolean] = None) {
   def adapterChain(coreplex: HasPeripheryBus)
                   (implicit p: Parameters): () => TLNodeChain = {
 
     val blockerParams = blockerCtrlAddr.map(BusBlockerParams(_, coreplex.pbus.beatBytes, coreplex.sbus.beatBytes, 1))
 
-    val tile_master_cork = cork.option(LazyModule(new TLCacheCork))
+    val tile_master_cork = cork.map(u => (LazyModule(new TLCacheCork(unsafe = u))))
     val tile_master_blocker = blockerParams.map(bp => LazyModule(new BusBlocker(bp)))
     val tile_master_fixer = LazyModule(new TLFIFOFixer(TLFIFOFixer.allUncacheable))
     val tile_master_buffer = LazyModule(new TLBufferChain(addBuffers))
diff --git a/src/main/scala/system/Configs.scala b/src/main/scala/system/Configs.scala
index 1d5fe758..ee8ce660 100644
--- a/src/main/scala/system/Configs.scala
+++ b/src/main/scala/system/Configs.scala
@@ -66,7 +66,7 @@ class DualCoreConfig extends Config(
 class TinyConfig extends Config(
   new WithNMemoryChannels(0) ++
   new WithStatelessBridge ++
-  new WithNTinyCores(1) ++
+  new With1TinyCore ++
   new BaseConfig)
 
 class DefaultFPGAConfig extends Config(new BaseConfig)
diff --git a/src/main/scala/tilelink/CacheCork.scala b/src/main/scala/tilelink/CacheCork.scala
index 063c58a4..5307d7e9 100644
--- a/src/main/scala/tilelink/CacheCork.scala
+++ b/src/main/scala/tilelink/CacheCork.scala
@@ -19,9 +19,9 @@ class TLCacheCork(unsafe: Boolean = false)(implicit p: Parameters) extends LazyM
     managerFn = { case mp =>
       mp.copy(
         endSinkId = 1,
-        managers = mp.managers.map { m => m.copy(
-          supportsAcquireB = if (m.regionType == RegionType.UNCACHED) m.supportsGet     else m.supportsAcquireB,
-          supportsAcquireT = if (m.regionType == RegionType.UNCACHED) m.supportsPutFull else m.supportsAcquireT)})})
+        managers = mp.managers.map { m => m.copy( // Rocket requires m.supportsAcquireT || !m.supportsAcquireB, so, don't cache ROMs
+          supportsAcquireB = if (m.regionType == RegionType.UNCACHED && m.supportsPutFull) m.supportsGet     else m.supportsAcquireB,
+          supportsAcquireT = if (m.regionType == RegionType.UNCACHED)                      m.supportsPutFull else m.supportsAcquireT)})})
 
   lazy val module = new LazyModuleImp(this) {
     (node.in zip node.out) foreach { case ((in, edgeIn), (out, edgeOut)) =>

From 1867a5b226f29604f4c724d664cac18f485fb51f Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Tue, 10 Oct 2017 18:06:58 -0700
Subject: [PATCH 05/11] rocket: only cache when AcquireT is possible

---
 src/main/scala/rocket/DCache.scala         | 6 +++---
 src/main/scala/rocket/TLB.scala            | 2 +-
 src/main/scala/rocket/TLBPermissions.scala | 1 -
 src/main/scala/tilelink/CacheCork.scala    | 6 +++---
 4 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/main/scala/rocket/DCache.scala b/src/main/scala/rocket/DCache.scala
index ed872a2a..db5cd49e 100644
--- a/src/main/scala/rocket/DCache.scala
+++ b/src/main/scala/rocket/DCache.scala
@@ -381,7 +381,7 @@ class DCacheModule(outer: DCache) extends HellaCacheModule(outer) {
   val access_address = s2_req.addr
   val a_size = mtSize(s2_req.typ)
   val a_data = Fill(beatWords, pstore1_data)
-  val acquire = if (edge.manager.anySupportAcquireB) {
+  val acquire = if (edge.manager.anySupportAcquireT) {
     edge.AcquireBlock(UInt(0), acquire_address, lgCacheBlockBytes, s2_grow_param)._2 // Cacheability checked by tlb
   } else {
     Wire(new TLBundleA(edge.bundle))
@@ -583,7 +583,7 @@ class DCacheModule(outer: DCache) extends HellaCacheModule(outer) {
     when (releaseDone) { release_state := s_probe_write_meta }
   }
   when (release_state.isOneOf(s_voluntary_writeback, s_voluntary_write_meta)) {
-    if (edge.manager.anySupportAcquireB)
+    if (edge.manager.anySupportAcquireT)
       tl_out_c.bits := edge.Release(fromSource = 0.U,
                                     toAddress = 0.U,
                                     lgSize = lgCacheBlockBytes,
@@ -696,7 +696,7 @@ class DCacheModule(outer: DCache) extends HellaCacheModule(outer) {
   metaArb.io.in(5).bits.way_en := ~UInt(0, nWays)
   metaArb.io.in(5).bits.data := metaArb.io.in(4).bits.data
   // Only flush D$ on FENCE.I if some cached executable regions are untracked.
-  if (!edge.manager.managers.forall(m => !m.supportsAcquireB || !m.executable || m.regionType >= RegionType.TRACKED)) {
+  if (!edge.manager.managers.forall(m => !m.supportsAcquireT || !m.executable || m.regionType >= RegionType.TRACKED)) {
     when (tl_out_a.fire() && !s2_uncached) { flushed := false }
     when (flushing) {
       s1_victim_way := flushCounter >> log2Up(nSets)
diff --git a/src/main/scala/rocket/TLB.scala b/src/main/scala/rocket/TLB.scala
index 47619484..4b6f72fe 100644
--- a/src/main/scala/rocket/TLB.scala
+++ b/src/main/scala/rocket/TLB.scala
@@ -112,7 +112,7 @@ class TLB(instruction: Boolean, lgMaxSize: Int, nEntries: Int)(implicit edge: TL
   val legal_address = edge.manager.findSafe(mpu_physaddr).reduce(_||_)
   def fastCheck(member: TLManagerParameters => Boolean) =
     legal_address && edge.manager.fastProperty(mpu_physaddr, member, (b:Boolean) => Bool(b))
-  val cacheable = fastCheck(_.supportsAcquireB) && (instruction || !usingDataScratchpad)
+  val cacheable = fastCheck(_.supportsAcquireT) && (instruction || !usingDataScratchpad)
   val homogeneous = TLBPageLookup(edge.manager.managers, xLen, p(CacheBlockBytes), BigInt(1) << pgIdxBits)(mpu_physaddr).homogeneous
   val prot_r = fastCheck(_.supportsGet) && pmp.io.r
   val prot_w = fastCheck(_.supportsPutFull) && pmp.io.w
diff --git a/src/main/scala/rocket/TLBPermissions.scala b/src/main/scala/rocket/TLBPermissions.scala
index b9cfb430..7196aff3 100644
--- a/src/main/scala/rocket/TLBPermissions.scala
+++ b/src/main/scala/rocket/TLBPermissions.scala
@@ -46,7 +46,6 @@ object TLBPageLookup
       require (!m.supportsAcquireT   || m.supportsAcquireT  .contains(xferSizes), s"MemoryMap region ${m.name} only supports ${m.supportsAcquireT} AcquireT, but must support ${xferSizes}")
       require (!m.supportsLogical    || m.supportsLogical   .contains(amoSizes),  s"MemoryMap region ${m.name} only supports ${m.supportsLogical} Logical, but must support ${amoSizes}")
       require (!m.supportsArithmetic || m.supportsArithmetic.contains(amoSizes),  s"MemoryMap region ${m.name} only supports ${m.supportsArithmetic} Arithmetic, but must support ${amoSizes}")
-      require (m.supportsAcquireT || !m.supportsAcquireB, s"MemoryMap region ${m.name} supports AcquireB (cached read) but not AcquireT (cached write)... and rocket assumes this")
 
       (m.address, TLBFixedPermissions(
         e = Seq(RegionType.PUT_EFFECTS, RegionType.GET_EFFECTS) contains m.regionType,
diff --git a/src/main/scala/tilelink/CacheCork.scala b/src/main/scala/tilelink/CacheCork.scala
index 5307d7e9..063c58a4 100644
--- a/src/main/scala/tilelink/CacheCork.scala
+++ b/src/main/scala/tilelink/CacheCork.scala
@@ -19,9 +19,9 @@ class TLCacheCork(unsafe: Boolean = false)(implicit p: Parameters) extends LazyM
     managerFn = { case mp =>
       mp.copy(
         endSinkId = 1,
-        managers = mp.managers.map { m => m.copy( // Rocket requires m.supportsAcquireT || !m.supportsAcquireB, so, don't cache ROMs
-          supportsAcquireB = if (m.regionType == RegionType.UNCACHED && m.supportsPutFull) m.supportsGet     else m.supportsAcquireB,
-          supportsAcquireT = if (m.regionType == RegionType.UNCACHED)                      m.supportsPutFull else m.supportsAcquireT)})})
+        managers = mp.managers.map { m => m.copy(
+          supportsAcquireB = if (m.regionType == RegionType.UNCACHED) m.supportsGet     else m.supportsAcquireB,
+          supportsAcquireT = if (m.regionType == RegionType.UNCACHED) m.supportsPutFull else m.supportsAcquireT)})})
 
   lazy val module = new LazyModuleImp(this) {
     (node.in zip node.out) foreach { case ((in, edgeIn), (out, edgeOut)) =>

From 1240cb275c14712b196dcac4d2161e14f492e614 Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Wed, 11 Oct 2017 00:29:11 -0700
Subject: [PATCH 06/11] coreplex: TilePortParams formatting

---
 src/main/scala/coreplex/RocketCoreplex.scala | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/main/scala/coreplex/RocketCoreplex.scala b/src/main/scala/coreplex/RocketCoreplex.scala
index 7287928c..96583fb7 100644
--- a/src/main/scala/coreplex/RocketCoreplex.scala
+++ b/src/main/scala/coreplex/RocketCoreplex.scala
@@ -28,17 +28,18 @@ case class TileMasterPortParams(
     val tile_master_fixer = LazyModule(new TLFIFOFixer(TLFIFOFixer.allUncacheable))
     val tile_master_buffer = LazyModule(new TLBufferChain(addBuffers))
 
+    tile_master_blocker.foreach { _.controlNode := coreplex.pbus.toVariableWidthSlaves }
+
     val nodes = List(
       Some(tile_master_buffer.node),
       Some(tile_master_fixer.node),
       tile_master_blocker.map(_.node),
-      tile_master_cork.map(_.node)).flatMap(b=>b)
+      tile_master_cork.map(_.node)
+    ).flatMap(b=>b)
 
     nodes.init zip nodes.tail foreach { case(front, back) => front :=* back }
 
-    tile_master_blocker.foreach { _.controlNode := coreplex.pbus.toVariableWidthSlaves }
-
-    () => TLNodeChain(nodes.last, nodes.head)
+    () => TLNodeChain(in = nodes.last, out = nodes.head)
   }
 }
 
@@ -53,15 +54,16 @@ case class TileSlavePortParams(
     val tile_slave_blocker = blockerParams.map(bp => LazyModule(new BusBlocker(bp)))
     val tile_slave_buffer = LazyModule(new TLBufferChain(addBuffers))
 
+    tile_slave_blocker.foreach { _.controlNode := coreplex.pbus.toVariableWidthSlaves }
+
     val nodes = List(
       Some(tile_slave_buffer.node),
-      tile_slave_blocker.map(_.node)).flatMap(b=>b)
+      tile_slave_blocker.map(_.node)
+    ).flatMap(b=>b)
 
     nodes.init zip nodes.tail foreach { case(front, back) => front :=* back }
 
-    tile_slave_blocker.foreach { _.controlNode := coreplex.pbus.toVariableWidthSlaves }
-
-    () => TLNodeChain(nodes.last, nodes.head)
+    () => TLNodeChain(in = nodes.last, out = nodes.head)
   }
 }
 

From 329a5c35d426c1d48634f3dbc854686fae6e666a Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Wed, 11 Oct 2017 00:30:51 -0700
Subject: [PATCH 07/11] tilelink: unsafe cache cork discards outer d.sink

---
 src/main/scala/tilelink/CacheCork.scala | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/scala/tilelink/CacheCork.scala b/src/main/scala/tilelink/CacheCork.scala
index 063c58a4..3ee2aa18 100644
--- a/src/main/scala/tilelink/CacheCork.scala
+++ b/src/main/scala/tilelink/CacheCork.scala
@@ -98,6 +98,7 @@ class TLCacheCork(unsafe: Boolean = false)(implicit p: Parameters) extends LazyM
       val d_d = Wire(in.d)
       d_d <> out.d
       d_d.bits.source := out.d.bits.source >> 1
+      if (unsafe) { d_d.bits.sink := UInt(0) }
 
       when (out.d.bits.opcode === AccessAckData && out.d.bits.source(0)) {
         d_d.bits.opcode := GrantData

From ec056535dc678d9997444c08fcc8f97f1d60b5c1 Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Wed, 11 Oct 2017 12:44:23 -0700
Subject: [PATCH 08/11] tilelink: add DeadlockDevice

---
 src/main/scala/devices/tilelink/Error.scala | 34 ++++++++++++++++-----
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/src/main/scala/devices/tilelink/Error.scala b/src/main/scala/devices/tilelink/Error.scala
index cdbb95e0..3c9131bc 100644
--- a/src/main/scala/devices/tilelink/Error.scala
+++ b/src/main/scala/devices/tilelink/Error.scala
@@ -13,17 +13,13 @@ import scala.math.min
 case class ErrorParams(address: Seq[AddressSet], maxTransfer: Int = 4096)
 case object ErrorParams extends Field[ErrorParams]
 
-/** Adds a /dev/null slave that generates TL error response messages */
-class TLError(params: ErrorParams, beatBytes: Int = 4)(implicit p: Parameters) extends LazyModule
-{
-  val address = params.address
-
-  val device = new SimpleDevice("error-device", Seq("sifive,error0"))
-
+abstract class DevNullDevice(params: ErrorParams, beatBytes: Int = 4)
+                            (device: SimpleDevice)
+                            (implicit p: Parameters) extends LazyModule {
   val xfer = TransferSizes(1, params.maxTransfer)
   val node = TLManagerNode(Seq(TLManagerPortParameters(
     Seq(TLManagerParameters(
-      address            = address,
+      address            = params.address,
       resources          = device.reg("mem"),
       regionType         = RegionType.UNCACHED,
       supportsAcquireT   = xfer,
@@ -38,7 +34,12 @@ class TLError(params: ErrorParams, beatBytes: Int = 4)(implicit p: Parameters) e
     beatBytes  = beatBytes,
     endSinkId  = 1, // can receive GrantAck
     minLatency = 1))) // no bypass needed for this device
+}
 
+/** Adds a /dev/null slave that generates TL error response messages */
+class TLError(params: ErrorParams, beatBytes: Int = 4)(implicit p: Parameters)
+    extends DevNullDevice(params, beatBytes)(new SimpleDevice("error-device", Seq("sifive,error0")))
+{
   lazy val module = new LazyModuleImp(this) {
     import TLMessages._
     import TLPermissions._
@@ -88,6 +89,23 @@ class TLError(params: ErrorParams, beatBytes: Int = 4)(implicit p: Parameters) e
   }
 }
 
+/** Adds a /dev/null slave that does not raise ready for any incoming traffic.
+  * !!! WARNING: This device WILL cause your bus to deadlock for as long as you
+  *              continue to send traffic to it !!!
+  */
+class DeadlockDevice(params: ErrorParams, beatBytes: Int = 4)(implicit p: Parameters)
+    extends DevNullDevice(params, beatBytes)(new SimpleDevice("deadlock-device", Seq("sifive,deadlock0")))
+{
+  lazy val module = new LazyModuleImp(this) {
+    val (in, _) = node.in(0)
+    in.a.ready := Bool(false)
+    in.b.valid := Bool(false)
+    in.c.ready := Bool(false)
+    in.d.valid := Bool(false)
+    in.e.ready := Bool(false)
+  }
+}
+
 trait HasSystemErrorSlave extends HasSystemBus {
   private val params = p(ErrorParams)
   val error = LazyModule(new TLError(params, sbus.beatBytes))

From 9f8e3d8879fd6fdd4caf2ed7ee60e6579f70311b Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Wed, 11 Oct 2017 12:45:36 -0700
Subject: [PATCH 09/11] tilelink: BusBypass can be sent to DeadlockDevice

---
 src/main/scala/devices/tilelink/BusBypass.scala | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/scala/devices/tilelink/BusBypass.scala b/src/main/scala/devices/tilelink/BusBypass.scala
index 4621074c..38c378a3 100644
--- a/src/main/scala/devices/tilelink/BusBypass.scala
+++ b/src/main/scala/devices/tilelink/BusBypass.scala
@@ -10,7 +10,7 @@ import freechips.rocketchip.tilelink._
 import freechips.rocketchip.util._
 import scala.math.min
 
-abstract class TLBusBypassBase(beatBytes: Int)(implicit p: Parameters) extends LazyModule
+abstract class TLBusBypassBase(beatBytes: Int, deadlock: Boolean = false)(implicit p: Parameters) extends LazyModule
 {
   protected val nodeIn = TLIdentityNode()
   protected val nodeOut = TLIdentityNode()
@@ -18,7 +18,8 @@ abstract class TLBusBypassBase(beatBytes: Int)(implicit p: Parameters) extends L
 
   protected val bar = LazyModule(new TLBusBypassBar)
   protected val everything = Seq(AddressSet(0, BigInt("ffffffffffffffffffffffffffffffff", 16))) // 128-bit
-  protected val error = LazyModule(new TLError(ErrorParams(everything), beatBytes))
+  protected val error = if (deadlock) LazyModule(new DeadlockDevice(ErrorParams(everything), beatBytes))
+                        else LazyModule(new TLError(ErrorParams(everything), beatBytes))
 
   // order matters
   bar.node := nodeIn

From 2dbe882e58ce281e7bc790879b9f3ae6aba881c5 Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Wed, 11 Oct 2017 13:16:50 -0700
Subject: [PATCH 10/11] tilelink: add BasicBusBlocker device

---
 .../scala/devices/tilelink/BusBlocker.scala   | 73 ++++++++++++++-----
 .../scala/devices/tilelink/BusBypass.scala    |  3 +
 2 files changed, 59 insertions(+), 17 deletions(-)

diff --git a/src/main/scala/devices/tilelink/BusBlocker.scala b/src/main/scala/devices/tilelink/BusBlocker.scala
index b057ab8d..3ff76d19 100644
--- a/src/main/scala/devices/tilelink/BusBlocker.scala
+++ b/src/main/scala/devices/tilelink/BusBlocker.scala
@@ -9,23 +9,6 @@ import freechips.rocketchip.regmapper._
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.util._
 
-case class BusBlockerParams(
-  controlAddress:   BigInt,
-  controlBeatBytes: Int,
-  deviceBeatBytes:  Int,
-  pmpRegisters:     Int)
-{
-  val page = 4096
-  val pageBits = log2Ceil(page)
-  val size = (((pmpRegisters * 8) + page - 1) / page) * page
-
-  require (pmpRegisters > 0)
-  require (controlAddress > 0)
-  require (controlAddress % size == 0)
-  require (controlBeatBytes > 0 && isPow2(controlBeatBytes))
-  require (deviceBeatBytes  > 0 && isPow2(deviceBeatBytes))
-}
-
 case class DevicePMPParams(addressBits: Int, pageBits: Int)
 class DevicePMP(params: DevicePMPParams) extends GenericParameterizedBundle(params)
 {
@@ -71,6 +54,28 @@ object DevicePMP
   }
 }
 
+/** BusBlocker uses a set of DevicePMP registers to control whether
+  * accesses of certain types are allowed to proceed or bypassed to
+  * a /dev/null device.
+  */
+
+case class BusBlockerParams(
+  controlAddress:   BigInt,
+  controlBeatBytes: Int,
+  deviceBeatBytes:  Int,
+  pmpRegisters:     Int)
+{
+  val page = 4096
+  val pageBits = log2Ceil(page)
+  val size = (((pmpRegisters * 8) + page - 1) / page) * page
+
+  require (pmpRegisters > 0)
+  require (controlAddress > 0)
+  require (controlAddress % size == 0)
+  require (controlBeatBytes > 0 && isPow2(controlBeatBytes))
+  require (deviceBeatBytes  > 0 && isPow2(deviceBeatBytes))
+}
+
 class BusBlocker(params: BusBlockerParams)(implicit p: Parameters) extends TLBusBypassBase(params.deviceBeatBytes)
 {
   val device = new SimpleDevice("bus-blocker", Seq("sifive,bus-blocker0"))
@@ -100,3 +105,37 @@ class BusBlocker(params: BusBlockerParams)(implicit p: Parameters) extends TLBus
     bar.module.io.bypass := !allow
   }
 }
+
+/** BasicBusBlocker uses a single bit register to control whether
+  * accesses of all types are allowed to proceed or bypassed to
+  * a /dev/null device. It has a second bit register to report
+  * whether any requests are pending on either path.
+  */
+
+case class BasicBusBlockerParams(
+  controlAddress:   BigInt,
+  controlBeatBytes: Int,
+  deviceBeatBytes:  Int,
+  deadlock: Boolean = false)
+
+class BasicBusBlocker(params: BasicBusBlockerParams)(implicit p: Parameters)
+    extends TLBusBypassBase(params.deviceBeatBytes, params.deadlock)
+{
+  val device = new SimpleDevice("basic-bus-blocker", Seq("sifive,basic-bus-blocker0"))
+
+  val controlNode = TLRegisterNode(
+    address   = Seq(AddressSet(params.controlAddress, 0xFFF)),
+    device    = device,
+    beatBytes = params.controlBeatBytes)
+
+  lazy val module = new LazyModuleImp(this) {
+    val allow = RegInit(true.B)
+    val pending = RegNext(bar.module.io.pending)
+
+    controlNode.regmap(
+      0 -> Seq(RegField  (32, allow)),
+      4 -> Seq(RegField.r(32, pending)))
+
+    bar.module.io.bypass := !allow
+  }
+}
diff --git a/src/main/scala/devices/tilelink/BusBypass.scala b/src/main/scala/devices/tilelink/BusBypass.scala
index 38c378a3..08d61418 100644
--- a/src/main/scala/devices/tilelink/BusBypass.scala
+++ b/src/main/scala/devices/tilelink/BusBypass.scala
@@ -49,6 +49,7 @@ class TLBusBypassBar(implicit p: Parameters) extends LazyModule
   lazy val module = new LazyModuleImp(this) {
     val io = IO(new Bundle {
       val bypass = Bool(INPUT)
+      val pending = Bool(OUTPUT)
     })
 
     val (in, edge) = node.in(0)
@@ -60,6 +61,8 @@ class TLBusBypassBar(implicit p: Parameters) extends LazyModule
     val flight = RegInit(UInt(0, width = log2Ceil(3*edge.client.endSourceId+1)))
     val bypass = RegInit(io.bypass) // synchronous reset required
 
+    io.pending := (flight > 0.U)
+
     val (a_first, a_last, _) = edge.firstlast(in.a)
     val (b_first, b_last, _) = edge.firstlast(in.b)
     val (c_first, c_last, _) = edge.firstlast(in.c)

From 60934ac622a127fe5eef5a98458bed00759ba465 Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Wed, 11 Oct 2017 13:35:06 -0700
Subject: [PATCH 11/11] coreplex: TilePortParams use BasicBusBlockers

---
 src/main/scala/coreplex/RocketCoreplex.scala | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/scala/coreplex/RocketCoreplex.scala b/src/main/scala/coreplex/RocketCoreplex.scala
index 96583fb7..eca82f1e 100644
--- a/src/main/scala/coreplex/RocketCoreplex.scala
+++ b/src/main/scala/coreplex/RocketCoreplex.scala
@@ -21,10 +21,10 @@ case class TileMasterPortParams(
   def adapterChain(coreplex: HasPeripheryBus)
                   (implicit p: Parameters): () => TLNodeChain = {
 
-    val blockerParams = blockerCtrlAddr.map(BusBlockerParams(_, coreplex.pbus.beatBytes, coreplex.sbus.beatBytes, 1))
+    val blockerParams = blockerCtrlAddr.map(BasicBusBlockerParams(_, coreplex.pbus.beatBytes, coreplex.sbus.beatBytes, deadlock = true))
 
     val tile_master_cork = cork.map(u => (LazyModule(new TLCacheCork(unsafe = u))))
-    val tile_master_blocker = blockerParams.map(bp => LazyModule(new BusBlocker(bp)))
+    val tile_master_blocker = blockerParams.map(bp => LazyModule(new BasicBusBlocker(bp)))
     val tile_master_fixer = LazyModule(new TLFIFOFixer(TLFIFOFixer.allUncacheable))
     val tile_master_buffer = LazyModule(new TLBufferChain(addBuffers))
 
@@ -49,9 +49,9 @@ case class TileSlavePortParams(
   def adapterChain(coreplex: HasPeripheryBus)
                   (implicit p: Parameters): () => TLNodeChain = {
 
-    val blockerParams = blockerCtrlAddr.map(BusBlockerParams(_, coreplex.pbus.beatBytes, coreplex.sbus.beatBytes, 1))
+    val blockerParams = blockerCtrlAddr.map(BasicBusBlockerParams(_, coreplex.pbus.beatBytes, coreplex.sbus.beatBytes))
 
-    val tile_slave_blocker = blockerParams.map(bp => LazyModule(new BusBlocker(bp)))
+    val tile_slave_blocker = blockerParams.map(bp => LazyModule(new BasicBusBlocker(bp)))
     val tile_slave_buffer = LazyModule(new TLBufferChain(addBuffers))
 
     tile_slave_blocker.foreach { _.controlNode := coreplex.pbus.toVariableWidthSlaves }